@tanstack/db 0.5.10 → 0.5.12

package/src/query/live/collection-subscriber.ts

@@ -1,19 +1,19 @@
-import { MultiSet } from "@tanstack/db-ivm"
+import { MultiSet } from '@tanstack/db-ivm'
 import {
   normalizeExpressionPaths,
   normalizeOrderByPaths,
-} from "../compiler/expressions.js"
-import type { MultiSetArray, RootStreamBuilder } from "@tanstack/db-ivm"
-import type { Collection } from "../../collection/index.js"
-import type { ChangeMessage } from "../../types.js"
-import type { Context, GetResult } from "../builder/types.js"
-import type { BasicExpression } from "../ir.js"
-import type { OrderByOptimizationInfo } from "../compiler/order-by.js"
-import type { CollectionConfigBuilder } from "./collection-config-builder.js"
-import type { CollectionSubscription } from "../../collection/subscription.js"
+} from '../compiler/expressions.js'
+import type { MultiSetArray, RootStreamBuilder } from '@tanstack/db-ivm'
+import type { Collection } from '../../collection/index.js'
+import type { ChangeMessage } from '../../types.js'
+import type { Context, GetResult } from '../builder/types.js'
+import type { BasicExpression } from '../ir.js'
+import type { OrderByOptimizationInfo } from '../compiler/order-by.js'
+import type { CollectionConfigBuilder } from './collection-config-builder.js'
+import type { CollectionSubscription } from '../../collection/subscription.js'
 
 const loadMoreCallbackSymbol = Symbol.for(
-  `@tanstack/db.collection-config-builder`
+  `@tanstack/db.collection-config-builder`,
 )
 
 export class CollectionSubscriber<
@@ -33,7 +33,7 @@ export class CollectionSubscriber<
     private alias: string,
     private collectionId: string,
     private collection: Collection,
-    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>
+    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>,
   ) {}
 
   subscribe(): CollectionSubscription {
@@ -53,17 +53,17 @@ export class CollectionSubscriber<
     if (orderByInfo) {
       subscription = this.subscribeToOrderedChanges(
         whereExpression,
-        orderByInfo
+        orderByInfo,
       )
     } else {
       // If the source alias is lazy then we should not include the initial state
       const includeInitialState = !this.collectionConfigBuilder.isLazyAlias(
-        this.alias
+        this.alias,
       )
 
       subscription = this.subscribeToMatchingChanges(
         whereExpression,
-        includeInitialState
+        includeInitialState,
       )
     }
 
@@ -79,7 +79,7 @@ export class CollectionSubscriber<
           resolve: resolve!,
         })
         this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
-          promise
+          promise,
         )
       }
     }
@@ -120,14 +120,14 @@ export class CollectionSubscriber<
     // currentSyncState is always defined when subscribe() is called
     // (called during sync session setup)
     this.collectionConfigBuilder.currentSyncState!.unsubscribeCallbacks.add(
-      unsubscribe
+      unsubscribe,
     )
     return subscription
   }
 
   private sendChangesToPipeline(
     changes: Iterable<ChangeMessage<any, string | number>>,
-    callback?: () => boolean
+    callback?: () => boolean,
   ) {
     // currentSyncState and input are always defined when this method is called
     // (only called from active subscriptions during a sync session)
@@ -136,7 +136,7 @@ export class CollectionSubscriber<
     const sentChanges = sendChangesToInput(
       input,
       changes,
-      this.collection.config.getKey
+      this.collection.config.getKey,
     )
 
     // Do not provide the callback that loads more data
@@ -154,10 +154,10 @@ export class CollectionSubscriber<
 
   private subscribeToMatchingChanges(
     whereExpression: BasicExpression<boolean> | undefined,
-    includeInitialState: boolean = false
+    includeInitialState: boolean = false,
   ) {
     const sendChanges = (
-      changes: Array<ChangeMessage<any, string | number>>
+      changes: Array<ChangeMessage<any, string | number>>,
     ) => {
       this.sendChangesToPipeline(changes)
     }
@@ -172,12 +172,12 @@ export class CollectionSubscriber<
 
   private subscribeToOrderedChanges(
     whereExpression: BasicExpression<boolean> | undefined,
-    orderByInfo: OrderByOptimizationInfo
+    orderByInfo: OrderByOptimizationInfo,
   ) {
     const { orderBy, offset, limit, index } = orderByInfo
 
     const sendChangesInRange = (
-      changes: Iterable<ChangeMessage<any, string | number>>
+      changes: Iterable<ChangeMessage<any, string | number>>,
     ) => {
       // Split live updates into a delete of the old value and an insert of the new value
       const splittedChanges = splitUpdates(changes)
@@ -190,17 +190,41 @@ export class CollectionSubscriber<
       whereExpression,
     })
 
-    subscription.setOrderByIndex(index)
+    // Listen for truncate events to reset cursor tracking state
+    // This ensures that after a must-refetch/truncate, we don't use stale cursor data
+    const truncateUnsubscribe = this.collection.on(`truncate`, () => {
+      this.biggest = undefined
+    })
+
+    // Clean up truncate listener when subscription is unsubscribed
+    subscription.on(`unsubscribed`, () => {
+      truncateUnsubscribe()
+    })
 
     // Normalize the orderBy clauses such that the references are relative to the collection
     const normalizedOrderBy = normalizeOrderByPaths(orderBy, this.alias)
 
-    // Load the first `offset + limit` values from the index
-    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[
-    subscription.requestLimitedSnapshot({
-      limit: offset + limit,
-      orderBy: normalizedOrderBy,
-    })
+    if (index) {
+      // We have an index on the first orderBy column - use lazy loading optimization
+      // This works for both single-column and multi-column orderBy:
+      // - Single-column: index provides exact ordering
+      // - Multi-column: index provides ordering on first column, secondary sort in memory
+      subscription.setOrderByIndex(index)
+
+      // Load the first `offset + limit` values from the index
+      // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[
+      subscription.requestLimitedSnapshot({
+        limit: offset + limit,
+        orderBy: normalizedOrderBy,
+      })
+    } else {
+      // No index available (e.g., non-ref expression): pass orderBy/limit to loadSubset
+      // so the sync layer can optimize if the backend supports it
+      subscription.requestSnapshot({
+        orderBy: normalizedOrderBy,
+        limit: offset + limit,
+      })
+    }
 
     return subscription
   }
@@ -220,11 +244,10 @@ export class CollectionSubscriber<
     const { dataNeeded } = orderByInfo
 
     if (!dataNeeded) {
-      // This should never happen because the topK operator should always set the size callback
-      // which in turn should lead to the orderBy operator setting the dataNeeded callback
-      throw new Error(
-        `Missing dataNeeded callback for collection ${this.collectionId}`
-      )
+      // dataNeeded is not set when there's no index (e.g., non-ref expression).
+      // In this case, we've already loaded all data via requestSnapshot
+      // and don't need to lazily load more.
+      return true
     }
 
     // `dataNeeded` probes the orderBy operator to see if it needs more data
@@ -238,7 +261,7 @@ export class CollectionSubscriber<
 
   private sendChangesToPipelineWithTracking(
     changes: Iterable<ChangeMessage<any, string | number>>,
-    subscription: CollectionSubscription
+    subscription: CollectionSubscription,
   ) {
     const orderByInfo = this.getOrderByInfo()
     if (!orderByInfo) {
@@ -262,7 +285,7 @@ export class CollectionSubscriber<
 
     this.sendChangesToPipeline(
       trackedChanges,
-      subscriptionWithLoader[loadMoreCallbackSymbol]
+      subscriptionWithLoader[loadMoreCallbackSymbol],
     )
   }
 
@@ -273,20 +296,33 @@ export class CollectionSubscriber<
     if (!orderByInfo) {
       return
     }
-    const { orderBy, valueExtractorForRawRow } = orderByInfo
+    const { orderBy, valueExtractorForRawRow, offset } = orderByInfo
     const biggestSentRow = this.biggest
-    const biggestSentValue = biggestSentRow
+
+    // Extract all orderBy column values from the biggest sent row
+    // For single-column: returns single value, for multi-column: returns array
+    const extractedValues = biggestSentRow
       ? valueExtractorForRawRow(biggestSentRow)
-      : biggestSentRow
+      : undefined
+
+    // Normalize to array format for minValues
+    const minValues =
+      extractedValues !== undefined
+        ? Array.isArray(extractedValues)
+          ? extractedValues
+          : [extractedValues]
+        : undefined
 
     // Normalize the orderBy clauses such that the references are relative to the collection
     const normalizedOrderBy = normalizeOrderByPaths(orderBy, this.alias)
 
     // Take the `n` items after the biggest sent value
+    // Pass the current window offset to ensure proper deduplication
     subscription.requestLimitedSnapshot({
       orderBy: normalizedOrderBy,
       limit: n,
-      minValue: biggestSentValue,
+      minValues,
+      offset,
     })
   }
 
@@ -312,13 +348,16 @@ export class CollectionSubscriber<
 
   private *trackSentValues(
     changes: Iterable<ChangeMessage<any, string | number>>,
-    comparator: (a: any, b: any) => number
+    comparator: (a: any, b: any) => number,
   ) {
     for (const change of changes) {
-      if (!this.biggest) {
-        this.biggest = change.value
-      } else if (comparator(this.biggest, change.value) < 0) {
-        this.biggest = change.value
+      // Only track inserts/updates for cursor positioning, not deletes
+      if (change.type !== `delete`) {
+        if (!this.biggest) {
+          this.biggest = change.value
+        } else if (comparator(this.biggest, change.value) < 0) {
+          this.biggest = change.value
+        }
       }
 
       yield change
@@ -332,7 +371,7 @@ export class CollectionSubscriber<
 function sendChangesToInput(
   input: RootStreamBuilder<unknown>,
   changes: Iterable<ChangeMessage>,
-  getKey: (item: ChangeMessage[`value`]) => any
+  getKey: (item: ChangeMessage[`value`]) => any,
 ): number {
   const multiSetArray: MultiSetArray<unknown> = []
   for (const change of changes) {
@@ -360,7 +399,7 @@ function* splitUpdates<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
 >(
-  changes: Iterable<ChangeMessage<T, TKey>>
+  changes: Iterable<ChangeMessage<T, TKey>>,
 ): Generator<ChangeMessage<T, TKey>> {
   for (const change of changes) {
     if (change.type === `update`) {

package/src/query/live/internal.ts

@@ -1,4 +1,4 @@
-import type { CollectionConfigBuilder } from "./collection-config-builder.js"
+import type { CollectionConfigBuilder } from './collection-config-builder.js'
 
 /**
  * Symbol for accessing internal utilities that should not be part of the public API

package/src/query/live/types.ts

@@ -1,11 +1,11 @@
-import type { D2, RootStreamBuilder } from "@tanstack/db-ivm"
+import type { D2, RootStreamBuilder } from '@tanstack/db-ivm'
 import type {
   CollectionConfig,
   ResultStream,
   StringCollationConfig,
-} from "../../types.js"
-import type { InitialQueryBuilder, QueryBuilder } from "../builder/index.js"
-import type { Context, GetResult } from "../builder/types.js"
+} from '../../types.js'
+import type { InitialQueryBuilder, QueryBuilder } from '../builder/index.js'
+import type { Context, GetResult } from '../builder/types.js'
 
 export type Changes<T> = {
   deletes: number

package/src/query/live-query-collection.ts

@@ -1,21 +1,21 @@
-import { createCollection } from "../collection/index.js"
-import { CollectionConfigBuilder } from "./live/collection-config-builder.js"
+import { createCollection } from '../collection/index.js'
+import { CollectionConfigBuilder } from './live/collection-config-builder.js'
 import {
   getBuilderFromConfig,
   registerCollectionBuilder,
-} from "./live/collection-registry.js"
-import type { LiveQueryCollectionUtils } from "./live/collection-config-builder.js"
-import type { LiveQueryCollectionConfig } from "./live/types.js"
-import type { InitialQueryBuilder, QueryBuilder } from "./builder/index.js"
-import type { Collection } from "../collection/index.js"
+} from './live/collection-registry.js'
+import type { LiveQueryCollectionUtils } from './live/collection-config-builder.js'
+import type { LiveQueryCollectionConfig } from './live/types.js'
+import type { InitialQueryBuilder, QueryBuilder } from './builder/index.js'
+import type { Collection } from '../collection/index.js'
 import type {
   CollectionConfig,
   CollectionConfigSingleRowOption,
   NonSingleResult,
   SingleResult,
   UtilsRecord,
-} from "../types.js"
-import type { Context, GetResult } from "./builder/types.js"
+} from '../types.js'
+import type { Context, GetResult } from './builder/types.js'
 
 type CollectionConfigForContext<
   TContext extends Context,
@@ -63,7 +63,7 @@ export function liveQueryCollectionOptions<
   TContext extends Context,
   TResult extends object = GetResult<TContext>,
 >(
-  config: LiveQueryCollectionConfig<TContext, TResult>
+  config: LiveQueryCollectionConfig<TContext, TResult>,
 ): CollectionConfigForContext<TContext, TResult> & {
   utils: LiveQueryCollectionUtils
 } {
@@ -116,7 +116,7 @@ export function createLiveQueryCollection<
   TContext extends Context,
   TResult extends object = GetResult<TContext>,
 >(
-  query: (q: InitialQueryBuilder) => QueryBuilder<TContext>
+  query: (q: InitialQueryBuilder) => QueryBuilder<TContext>,
 ): CollectionForContext<TContext, TResult> & {
   utils: LiveQueryCollectionUtils
 }
@@ -127,7 +127,7 @@ export function createLiveQueryCollection<
   TResult extends object = GetResult<TContext>,
   TUtils extends UtilsRecord = {},
 >(
-  config: LiveQueryCollectionConfig<TContext, TResult> & { utils?: TUtils }
+  config: LiveQueryCollectionConfig<TContext, TResult> & { utils?: TUtils },
 ): CollectionForContext<TContext, TResult> & {
   utils: LiveQueryCollectionUtils & TUtils
 }
@@ -140,7 +140,7 @@ export function createLiveQueryCollection<
 >(
   configOrQuery:
     | (LiveQueryCollectionConfig<TContext, TResult> & { utils?: TUtils })
-    | ((q: InitialQueryBuilder) => QueryBuilder<TContext>)
+    | ((q: InitialQueryBuilder) => QueryBuilder<TContext>),
 ): CollectionForContext<TContext, TResult> & {
   utils: LiveQueryCollectionUtils & TUtils
 } {
@@ -149,7 +149,7 @@ export function createLiveQueryCollection<
     // Simple query function case
     const config: LiveQueryCollectionConfig<TContext, TResult> = {
       query: configOrQuery as (
-        q: InitialQueryBuilder
+        q: InitialQueryBuilder,
       ) => QueryBuilder<TContext>,
     }
     const options = liveQueryCollectionOptions<TContext, TResult>(config)
@@ -185,7 +185,7 @@ function bridgeToCreateCollection<
   TResult extends object,
   TUtils extends UtilsRecord = {},
 >(
-  options: CollectionConfig<TResult> & { utils: TUtils }
+  options: CollectionConfig<TResult> & { utils: TUtils },
 ): Collection<TResult, string | number, TUtils> {
   const collection = createCollection(options as any) as unknown as Collection<
     TResult,

package/src/query/optimizer.ts

@@ -120,8 +120,8 @@
  * transformed into a D2Mini pipeline.
  */
 
-import { deepEquals } from "../utils.js"
-import { CannotCombineEmptyExpressionListError } from "../errors.js"
+import { deepEquals } from '../utils.js'
+import { CannotCombineEmptyExpressionListError } from '../errors.js'
 import {
   CollectionRef as CollectionRefClass,
   Func,
@@ -130,8 +130,8 @@ import {
   createResidualWhere,
   getWhereExpression,
   isResidualWhere,
-} from "./ir.js"
-import type { BasicExpression, From, QueryIR, Select, Where } from "./ir.js"
+} from './ir.js'
+import type { BasicExpression, From, QueryIR, Select, Where } from './ir.js'
 
 /**
  * Represents a WHERE clause after source analysis
@@ -226,7 +226,7 @@ export function optimizeQuery(query: QueryIR): OptimizationResult {
  * @returns Map of source aliases to their WHERE clauses
  */
 function extractSourceWhereClauses(
-  query: QueryIR
+  query: QueryIR,
 ): Map<string, BasicExpression<boolean>> {
   const sourceWhereClauses = new Map<string, BasicExpression<boolean>>()
 
@@ -240,7 +240,7 @@ function extractSourceWhereClauses(
 
   // Analyze each WHERE clause to determine which sources it touches
   const analyzedClauses = splitWhereClauses.map((clause) =>
-    analyzeWhereClause(clause)
+    analyzeWhereClause(clause),
   )
 
   // Group clauses by single-source vs multi-source
@@ -297,7 +297,7 @@ function applyRecursiveOptimization(query: QueryIR): QueryIR {
       query.from.type === `queryRef`
         ? new QueryRefClass(
             applyRecursiveOptimization(query.from.query),
-            query.from.alias
+            query.from.alias,
           )
         : query.from,
     join: query.join?.map((joinClause) => ({
@@ -306,7 +306,7 @@ function applyRecursiveOptimization(query: QueryIR): QueryIR {
         joinClause.from.type === `queryRef`
           ? new QueryRefClass(
               applyRecursiveOptimization(joinClause.from.query),
-              joinClause.from.alias
+              joinClause.from.alias,
             )
           : joinClause.from,
     })),
@@ -346,7 +346,7 @@ function applySingleLevelOptimization(query: QueryIR): QueryIR {
 
   // Filter out residual WHERE clauses to prevent them from being optimized again
   const nonResidualWhereClauses = query.where.filter(
-    (where) => !isResidualWhere(where)
+    (where) => !isResidualWhere(where),
   )
 
   // Step 1: Split all AND clauses at the root level for granular optimization
@@ -354,7 +354,7 @@ function applySingleLevelOptimization(query: QueryIR): QueryIR {
 
   // Step 2: Analyze each WHERE clause to determine which sources it touches
   const analyzedClauses = splitWhereClauses.map((clause) =>
-    analyzeWhereClause(clause)
+    analyzeWhereClause(clause),
   )
 
   // Step 3: Group clauses by single-source vs multi-source
@@ -365,7 +365,7 @@ function applySingleLevelOptimization(query: QueryIR): QueryIR {
 
   // Add back any residual WHERE clauses that were filtered out
   const residualWhereClauses = query.where.filter((where) =>
-    isResidualWhere(where)
+    isResidualWhere(where),
   )
   if (residualWhereClauses.length > 0) {
     optimizedQuery.where = [
@@ -461,7 +461,7 @@ function isRedundantSubquery(query: QueryIR): boolean {
  * ```
  */
 function splitAndClauses(
-  whereClauses: Array<Where>
+  whereClauses: Array<Where>,
 ): Array<BasicExpression<boolean>> {
   const result: Array<BasicExpression<boolean>> = []
 
@@ -475,7 +475,7 @@ function splitAndClauses(
 
 // Helper function for recursive splitting of BasicExpression arrays
 function splitAndClausesRecursive(
-  clause: BasicExpression<boolean>
+  clause: BasicExpression<boolean>,
 ): Array<BasicExpression<boolean>> {
   if (clause.type === `func` && clause.name === `and`) {
     // Recursively split nested AND clauses to handle complex expressions
@@ -515,7 +515,7 @@ function splitAndClausesRecursive(
  * ```
  */
 function analyzeWhereClause(
-  clause: BasicExpression<boolean>
+  clause: BasicExpression<boolean>,
 ): AnalyzedWhereClause {
   // Track which table aliases this WHERE clause touches
   const touchedSources = new Set<string>()
@@ -580,7 +580,7 @@ function analyzeWhereClause(
  * @returns Grouped clauses ready for optimization
  */
 function groupWhereClauses(
-  analyzedClauses: Array<AnalyzedWhereClause>
+  analyzedClauses: Array<AnalyzedWhereClause>,
 ): GroupedWhereClauses {
   const singleSource = new Map<string, Array<BasicExpression<boolean>>>()
   const multiSource: Array<BasicExpression<boolean>> = []
@@ -630,7 +630,7 @@ function groupWhereClauses(
  */
 function applyOptimizations(
   query: QueryIR,
-  groupedClauses: GroupedWhereClauses
+  groupedClauses: GroupedWhereClauses,
 ): QueryIR {
   // Track which single-source clauses were actually optimized
   const actuallyOptimized = new Set<string>()
@@ -639,7 +639,7 @@ function applyOptimizations(
   const optimizedFrom = optimizeFromWithTracking(
     query.from,
     groupedClauses.singleSource,
-    actuallyOptimized
+    actuallyOptimized,
   )
 
   // Optimize JOIN clauses and track what was optimized
@@ -649,7 +649,7 @@ function applyOptimizations(
         from: optimizeFromWithTracking(
           joinClause.from,
           groupedClauses.singleSource,
-          actuallyOptimized
+          actuallyOptimized,
         ),
       }))
     : undefined
@@ -667,7 +667,7 @@ function applyOptimizations(
     query.join &&
     query.join.some(
       (join) =>
-        join.type === `left` || join.type === `right` || join.type === `full`
+        join.type === `left` || join.type === `right` || join.type === `full`,
     )
 
   // Add single-source clauses
@@ -690,8 +690,8 @@ function applyOptimizations(
       ? [
           combineWithAnd(
             remainingWhereClauses.flatMap((clause) =>
-              splitAndClausesRecursive(getWhereExpression(clause))
-            )
+              splitAndClausesRecursive(getWhereExpression(clause)),
+            ),
           ),
         ]
       : remainingWhereClauses
@@ -749,11 +749,11 @@ function deepCopyQuery(query: QueryIR): QueryIR {
             joinClause.from.type === `collectionRef`
               ? new CollectionRefClass(
                   joinClause.from.collection,
-                  joinClause.from.alias
+                  joinClause.from.alias,
                 )
               : new QueryRefClass(
                   deepCopyQuery(joinClause.from.query),
-                  joinClause.from.alias
+                  joinClause.from.alias,
                 ),
         }))
       : undefined,
@@ -780,7 +780,7 @@ function deepCopyQuery(query: QueryIR): QueryIR {
 function optimizeFromWithTracking(
   from: From,
   singleSourceClauses: Map<string, BasicExpression<boolean>>,
-  actuallyOptimized: Set<string>
+  actuallyOptimized: Set<string>,
 ): From {
   const whereClause = singleSourceClauses.get(from.alias)
 
@@ -833,7 +833,7 @@ function optimizeFromWithTracking(
 function unsafeSelect(
   query: QueryIR,
   whereClause: BasicExpression<boolean>,
-  outerAlias: string
+  outerAlias: string,
 ): boolean {
   if (!query.select) return false
 
@@ -870,7 +870,7 @@ function unsafeFnSelect(query: QueryIR) {
 function isSafeToPushIntoExistingSubquery(
   query: QueryIR,
   whereClause: BasicExpression<boolean>,
-  outerAlias: string
+  outerAlias: string,
 ): boolean {
   return !(
     unsafeSelect(query, whereClause, outerAlias) ||
@@ -945,7 +945,7 @@ function collectRefs(expr: any): Array<PropRef> {
 function whereReferencesComputedSelectFields(
   select: Select,
   whereClause: BasicExpression<boolean>,
-  outerAlias: string
+  outerAlias: string,
 ): boolean {
   // Build a set of computed field names at the top-level of the subquery output
   const computed = new Set<string>()
@@ -979,7 +979,7 @@ function whereReferencesComputedSelectFields(
 function referencesAliasWithRemappedSelect(
   subquery: QueryIR,
   whereClause: BasicExpression<boolean>,
-  outerAlias: string
+  outerAlias: string,
 ): boolean {
   const refs = collectRefs(whereClause)
   // Only care about clauses that actually reference the outer alias.
@@ -1046,7 +1046,7 @@ function referencesAliasWithRemappedSelect(
  * @throws Error if the expressions array is empty
  */
 function combineWithAnd(
-  expressions: Array<BasicExpression<boolean>>
+  expressions: Array<BasicExpression<boolean>>,
 ): BasicExpression<boolean> {
   if (expressions.length === 0) {
     throw new CannotCombineEmptyExpressionListError()