@tanstack/db 0.5.11 → 0.5.12

package/src/query/compiler/order-by.ts

@@ -1,22 +1,22 @@
-import { orderByWithFractionalIndex } from "@tanstack/db-ivm"
-import { defaultComparator, makeComparator } from "../../utils/comparison.js"
-import { PropRef, followRef } from "../ir.js"
-import { ensureIndexForField } from "../../indexes/auto-index.js"
-import { findIndexForField } from "../../utils/index-optimization.js"
-import { compileExpression } from "./evaluators.js"
-import { replaceAggregatesByRefs } from "./group-by.js"
-import type { CompareOptions } from "../builder/types.js"
-import type { WindowOptions } from "./types.js"
-import type { CompiledSingleRowExpression } from "./evaluators.js"
-import type { OrderBy, OrderByClause, QueryIR, Select } from "../ir.js"
+import { orderByWithFractionalIndex } from '@tanstack/db-ivm'
+import { defaultComparator, makeComparator } from '../../utils/comparison.js'
+import { PropRef, followRef } from '../ir.js'
+import { ensureIndexForField } from '../../indexes/auto-index.js'
+import { findIndexForField } from '../../utils/index-optimization.js'
+import { compileExpression } from './evaluators.js'
+import { replaceAggregatesByRefs } from './group-by.js'
+import type { CompareOptions } from '../builder/types.js'
+import type { WindowOptions } from './types.js'
+import type { CompiledSingleRowExpression } from './evaluators.js'
+import type { OrderBy, OrderByClause, QueryIR, Select } from '../ir.js'
 import type {
   CollectionLike,
   NamespacedAndKeyedStream,
   NamespacedRow,
-} from "../../types.js"
-import type { IStreamBuilder, KeyValue } from "@tanstack/db-ivm"
-import type { IndexInterface } from "../../indexes/base-index.js"
-import type { Collection } from "../../collection/index.js"
+} from '../../types.js'
+import type { IStreamBuilder, KeyValue } from '@tanstack/db-ivm'
+import type { IndexInterface } from '../../indexes/base-index.js'
+import type { Collection } from '../../collection/index.js'
 
 export type OrderByOptimizationInfo = {
   alias: string
@@ -25,10 +25,14 @@ export type OrderByOptimizationInfo = {
   limit: number
   comparator: (
     a: Record<string, unknown> | null | undefined,
-    b: Record<string, unknown> | null | undefined
+    b: Record<string, unknown> | null | undefined,
   ) => number
-  valueExtractorForRawRow: (row: Record<string, unknown>) => any
-  index: IndexInterface<string | number>
+  /** Extracts all orderBy column values from a raw row (array for multi-column) */
+  valueExtractorForRawRow: (row: Record<string, unknown>) => unknown
+  /** Extracts only the first column value - used for index-based cursor */
+  firstColumnValueExtractor: (row: Record<string, unknown>) => unknown
+  /** Index on the first orderBy column - used for lazy loading */
+  index?: IndexInterface<string | number>
   dataNeeded?: () => number
 }
 
@@ -46,14 +50,14 @@ export function processOrderBy(
   optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,
   setWindowFn: (windowFn: (options: WindowOptions) => void) => void,
   limit?: number,
-  offset?: number
+  offset?: number,
 ): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>> {
   // Pre-compile all order by expressions
   const compiledOrderBy = orderByClause.map((clause) => {
     const clauseWithoutAggregates = replaceAggregatesByRefs(
       clause.expression,
       selectClause,
-      `__select_results`
+      `__select_results`,
     )
 
     return {
@@ -74,7 +78,7 @@ export function processOrderBy(
     if (orderByClause.length > 1) {
       // For multiple orderBy columns, create a composite key
       return compiledOrderBy.map((compiled) =>
-        compiled.compiledExpression(orderByContext)
+        compiled.compiledExpression(orderByContext),
       )
     } else if (orderByClause.length === 1) {
       // For a single orderBy column, use the value directly
@@ -117,76 +121,165 @@ export function processOrderBy(
 
   let orderByOptimizationInfo: OrderByOptimizationInfo | undefined
 
-  // Optimize the orderBy operator to lazily load elements
-  // by using the range index of the collection.
-  // Only for orderBy clause on a single column for now (no composite ordering)
-  if (limit && orderByClause.length === 1) {
-    const clause = orderByClause[0]!
-    const orderByExpression = clause.expression
+  // When there's a limit, we create orderByOptimizationInfo to pass orderBy/limit
+  // to loadSubset so the sync layer can optimize the query.
+  // We try to use an index on the FIRST orderBy column for lazy loading,
+  // even for multi-column orderBy (using wider bounds on first column).
+  if (limit) {
+    let index: IndexInterface<string | number> | undefined
+    let followRefCollection: Collection | undefined
+    let firstColumnValueExtractor: CompiledSingleRowExpression | undefined
+    let orderByAlias: string = rawQuery.from.alias
 
-    if (orderByExpression.type === `ref`) {
+    // Try to create/find an index on the FIRST orderBy column for lazy loading
+    const firstClause = orderByClause[0]!
+    const firstOrderByExpression = firstClause.expression
+
+    if (firstOrderByExpression.type === `ref`) {
       const followRefResult = followRef(
         rawQuery,
-        orderByExpression,
-        collection
-      )!
-
-      const followRefCollection = followRefResult.collection
-      const fieldName = followRefResult.path[0]
-      const compareOpts = buildCompareOptions(clause, followRefCollection)
-      if (fieldName) {
-        ensureIndexForField(
-          fieldName,
-          followRefResult.path,
+        firstOrderByExpression,
+        collection,
+      )
+
+      if (followRefResult) {
+        followRefCollection = followRefResult.collection
+        const fieldName = followRefResult.path[0]
+        const compareOpts = buildCompareOptions(
+          firstClause,
           followRefCollection,
-          compareOpts,
-          compare
         )
-      }
 
-      const valueExtractorForRawRow = compileExpression(
-        new PropRef(followRefResult.path),
-        true
-      ) as CompiledSingleRowExpression
+        if (fieldName) {
+          ensureIndexForField(
+            fieldName,
+            followRefResult.path,
+            followRefCollection,
+            compareOpts,
+            compare,
+          )
+        }
 
-      const comparator = (
-        a: Record<string, unknown> | null | undefined,
-        b: Record<string, unknown> | null | undefined
-      ) => {
-        const extractedA = a ? valueExtractorForRawRow(a) : a
-        const extractedB = b ? valueExtractorForRawRow(b) : b
-        return compare(extractedA, extractedB)
-      }
+        // First column value extractor - used for index cursor
+        firstColumnValueExtractor = compileExpression(
+          new PropRef(followRefResult.path),
+          true,
+        ) as CompiledSingleRowExpression
 
-      const index: IndexInterface<string | number> | undefined =
-        findIndexForField(
+        index = findIndexForField(
           followRefCollection,
           followRefResult.path,
-          compareOpts
+          compareOpts,
         )
 
-      if (index && index.supports(`gt`)) {
-        // We found an index that we can use to lazily load ordered data
-        const orderByAlias =
-          orderByExpression.path.length > 1
-            ? String(orderByExpression.path[0])
+        // Only use the index if it supports range queries
+        if (!index?.supports(`gt`)) {
+          index = undefined
+        }
+
+        orderByAlias =
+          firstOrderByExpression.path.length > 1
+            ? String(firstOrderByExpression.path[0])
             : rawQuery.from.alias
+      }
+    }
 
-        orderByOptimizationInfo = {
-          alias: orderByAlias,
-          offset: offset ?? 0,
-          limit,
-          comparator,
-          valueExtractorForRawRow,
-          index,
-          orderBy: orderByClause,
+    // Only create comparator and value extractors if the first column is a ref expression
+    // For aggregate or computed expressions, we can't extract values from raw collection rows
+    if (!firstColumnValueExtractor) {
+      // Skip optimization for non-ref expressions (aggregates, computed values, etc.)
+      // The query will still work, but without lazy loading optimization
+    } else {
+      // Build value extractors for all columns (must all be ref expressions for multi-column)
+      // Check if all orderBy expressions are ref types (required for multi-column extraction)
+      const allColumnsAreRefs = orderByClause.every(
+        (clause) => clause.expression.type === `ref`,
+      )
+
+      // Create extractors for all columns if they're all refs
+      const allColumnExtractors:
+        | Array<CompiledSingleRowExpression>
+        | undefined = allColumnsAreRefs
+        ? orderByClause.map((clause) => {
+            // We know it's a ref since we checked allColumnsAreRefs
+            const refExpr = clause.expression as PropRef
+            const followResult = followRef(rawQuery, refExpr, collection)
+            if (followResult) {
+              return compileExpression(
+                new PropRef(followResult.path),
+                true,
+              ) as CompiledSingleRowExpression
+            }
+            // Fallback for refs that don't follow
+            return compileExpression(
+              clause.expression,
+              true,
+            ) as CompiledSingleRowExpression
+          })
+        : undefined
+
+      // Create a comparator for raw rows (used for tracking sent values)
+      // This compares ALL orderBy columns for proper ordering
+      const comparator = (
+        a: Record<string, unknown> | null | undefined,
+        b: Record<string, unknown> | null | undefined,
+      ) => {
+        if (orderByClause.length === 1) {
+          // Single column: extract and compare
+          const extractedA = a ? firstColumnValueExtractor(a) : a
+          const extractedB = b ? firstColumnValueExtractor(b) : b
+          return compare(extractedA, extractedB)
+        }
+        if (allColumnExtractors) {
+          // Multi-column with all refs: extract all values and compare
+          const extractAll = (
+            row: Record<string, unknown> | null | undefined,
+          ) => {
+            if (!row) return row
+            return allColumnExtractors.map((extractor) => extractor(row))
+          }
+          return compare(extractAll(a), extractAll(b))
         }
+        // Fallback: can't compare (shouldn't happen since we skip non-ref cases)
+        return 0
+      }
+
+      // Create a value extractor for raw rows that extracts ALL orderBy column values
+      // This is used for tracking sent values and building composite cursors
+      const rawRowValueExtractor = (row: Record<string, unknown>): unknown => {
+        if (orderByClause.length === 1) {
+          // Single column: return single value
+          return firstColumnValueExtractor(row)
+        }
+        if (allColumnExtractors) {
+          // Multi-column: return array of all values
+          return allColumnExtractors.map((extractor) => extractor(row))
+        }
+        // Fallback (shouldn't happen)
+        return undefined
+      }
+
+      orderByOptimizationInfo = {
+        alias: orderByAlias,
+        offset: offset ?? 0,
+        limit,
+        comparator,
+        valueExtractorForRawRow: rawRowValueExtractor,
+        firstColumnValueExtractor: firstColumnValueExtractor,
+        index,
+        orderBy: orderByClause,
+      }
 
-        optimizableOrderByCollections[followRefCollection.id] =
-          orderByOptimizationInfo
+      // Store the optimization info keyed by collection ID
+      // Use the followed collection if available, otherwise use the main collection
+      const targetCollectionId = followRefCollection?.id ?? collection.id
+      optimizableOrderByCollections[targetCollectionId] =
+        orderByOptimizationInfo
 
+      // Set up lazy loading callback if we have an index
+      if (index) {
         setSizeCallback = (getSize: () => number) => {
-          optimizableOrderByCollections[followRefCollection.id]![`dataNeeded`] =
+          optimizableOrderByCollections[targetCollectionId]![`dataNeeded`] =
             () => {
               const size = getSize()
               return Math.max(0, orderByOptimizationInfo!.limit - size)
@@ -204,7 +297,7 @@ export function processOrderBy(
       comparator: compare,
       setSizeCallback,
       setWindowFn: (
-        windowFn: (options: { offset?: number; limit?: number }) => void
+        windowFn: (options: { offset?: number; limit?: number }) => void,
       ) => {
         setWindowFn(
           // We wrap the move function such that we update the orderByOptimizationInfo
@@ -217,10 +310,10 @@ export function processOrderBy(
               orderByOptimizationInfo.limit =
                 options.limit ?? orderByOptimizationInfo.limit
             }
-          }
+          },
         )
       },
-    })
+    }),
     // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format
   )
 }
@@ -231,7 +324,7 @@ export function processOrderBy(
  */
 export function buildCompareOptions(
   clause: OrderByClause,
-  collection: CollectionLike<any, any>
+  collection: CollectionLike<any, any>,
 ): CompareOptions {
   if (clause.compareOptions.stringSort !== undefined) {
     return clause.compareOptions

package/src/query/compiler/select.ts

@@ -1,13 +1,13 @@
-import { map } from "@tanstack/db-ivm"
-import { PropRef, Value as ValClass, isExpressionLike } from "../ir.js"
-import { AggregateNotSupportedError } from "../../errors.js"
-import { compileExpression } from "./evaluators.js"
-import type { Aggregate, BasicExpression, Select } from "../ir.js"
+import { map } from '@tanstack/db-ivm'
+import { PropRef, Value as ValClass, isExpressionLike } from '../ir.js'
+import { AggregateNotSupportedError } from '../../errors.js'
+import { compileExpression } from './evaluators.js'
+import type { Aggregate, BasicExpression, Select } from '../ir.js'
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,
   NamespacedRow,
-} from "../../types.js"
+} from '../../types.js'
 
 /**
  * Type for operations array used in select processing
@@ -34,7 +34,7 @@ function unwrapVal(input: any): any {
 function processMerge(
   op: Extract<SelectOp, { kind: `merge` }>,
   namespacedRow: NamespacedRow,
-  selectResults: Record<string, any>
+  selectResults: Record<string, any>,
 ): void {
   const value = op.source(namespacedRow)
   if (value && typeof value === `object`) {
@@ -74,7 +74,7 @@ function processMerge(
 function processNonMergeOp(
   op: Extract<SelectOp, { kind: `field` }>,
   namespacedRow: NamespacedRow,
-  selectResults: Record<string, any>
+  selectResults: Record<string, any>,
 ): void {
   // Support nested alias paths like "meta.author.name"
   const path = op.alias.split(`.`)
@@ -99,7 +99,7 @@ function processNonMergeOp(
  */
 function processRow(
   [key, namespacedRow]: [unknown, NamespacedRow],
-  ops: Array<SelectOp>
+  ops: Array<SelectOp>,
 ): [unknown, typeof namespacedRow & { __select_results: any }] {
   const selectResults: Record<string, any> = {}
 
@@ -131,7 +131,7 @@ function processRow(
 export function processSelect(
   pipeline: NamespacedAndKeyedStream,
   select: Select,
-  _allInputs: Record<string, KeyedStream>
+  _allInputs: Record<string, KeyedStream>,
 ): NamespacedAndKeyedStream {
   // Build ordered operations to preserve authoring order (spreads and fields)
   const ops: Array<SelectOp> = []
@@ -145,7 +145,7 @@ export function processSelect(
  * Helper function to check if an expression is an aggregate
  */
 function isAggregateExpression(
-  expr: BasicExpression | Aggregate
+  expr: BasicExpression | Aggregate,
 ): expr is Aggregate {
   return expr.type === `agg`
 }
@@ -155,7 +155,7 @@ function isAggregateExpression(
  */
 export function processArgument(
   arg: BasicExpression | Aggregate,
-  namespacedRow: NamespacedRow
+  namespacedRow: NamespacedRow,
 ): any {
   if (isAggregateExpression(arg)) {
     throw new AggregateNotSupportedError()
@@ -189,7 +189,7 @@ function isNestedSelectObject(obj: any): boolean {
 function addFromObject(
   prefixPath: Array<string>,
   obj: any,
-  ops: Array<SelectOp>
+  ops: Array<SelectOp>,
 ) {
   for (const [key, value] of Object.entries(obj)) {
     if (key.startsWith(`__SPREAD_SENTINEL__`)) {

package/src/query/compiler/types.ts

@@ -1,5 +1,5 @@
-import type { QueryIR } from "../ir.js"
-import type { CompilationResult } from "./index.js"
+import type { QueryIR } from '../ir.js'
+import type { CompilationResult } from './index.js'
 
 /**
  * Cache for compiled subqueries to avoid duplicate compilation

package/src/query/expression-helpers.ts

@@ -27,7 +27,7 @@
  * ```
  */
 
-import type { IR, OperatorName } from "../index.js"
+import type { IR, OperatorName } from '../index.js'
 
 type BasicExpression<T = any> = IR.BasicExpression<T>
 type OrderBy = IR.OrderBy
@@ -149,7 +149,7 @@ export function extractValue(expr: BasicExpression): any {
  */
 export function walkExpression(
   expr: BasicExpression | undefined | null,
-  visitor: (node: BasicExpression) => void
+  visitor: (node: BasicExpression) => void,
 ): void {
   if (!expr) return
 
@@ -200,7 +200,7 @@ export function walkExpression(
  */
 export function parseWhereExpression<T = any>(
   expr: BasicExpression<boolean> | undefined | null,
-  options: ParseWhereOptions<T>
+  options: ParseWhereOptions<T>,
 ): T | null {
   if (!expr) return null
 
@@ -226,7 +226,7 @@ export function parseWhereExpression<T = any>(
       return onUnknownOperator(name, args)
     }
     throw new Error(
-      `No handler provided for operator: ${name}. Available handlers: ${Object.keys(handlers).join(`, `)}`
+      `No handler provided for operator: ${name}. Available handlers: ${Object.keys(handlers).join(`, `)}`,
     )
   }
 
@@ -263,7 +263,7 @@ export function parseWhereExpression<T = any>(
  * ```
  */
 export function parseOrderByExpression(
-  orderBy: OrderBy | undefined | null
+  orderBy: OrderBy | undefined | null,
 ): Array<ParsedOrderBy> {
   if (!orderBy || orderBy.length === 0) {
     return []
@@ -274,7 +274,7 @@ export function parseOrderByExpression(
 
     if (!field) {
       throw new Error(
-        `ORDER BY expression must be a field reference, got: ${clause.expression.type}`
+        `ORDER BY expression must be a field reference, got: ${clause.expression.type}`,
       )
     }
 
@@ -325,7 +325,7 @@ export function parseOrderByExpression(
  * ```
  */
 export function extractSimpleComparisons(
-  expr: BasicExpression<boolean> | undefined | null
+  expr: BasicExpression<boolean> | undefined | null,
 ): Array<SimpleComparison> {
   if (!expr) return []
 
@@ -344,7 +344,7 @@ export function extractSimpleComparisons(
         const [arg] = e.args
         if (!arg || arg.type !== `func`) {
           throw new Error(
-            `extractSimpleComparisons requires a comparison or null check inside 'not' operator.`
+            `extractSimpleComparisons requires a comparison or null check inside 'not' operator.`,
           )
         }
 
@@ -362,7 +362,7 @@ export function extractSimpleComparisons(
             })
           } else {
             throw new Error(
-              `extractSimpleComparisons requires a field reference for '${arg.name}' operator.`
+              `extractSimpleComparisons requires a field reference for '${arg.name}' operator.`,
             )
           }
           return
@@ -383,7 +383,7 @@ export function extractSimpleComparisons(
             })
           } else {
             throw new Error(
-              `extractSimpleComparisons requires simple field-value comparisons. Found complex expression for 'not(${arg.name})' operator.`
+              `extractSimpleComparisons requires simple field-value comparisons. Found complex expression for 'not(${arg.name})' operator.`,
             )
           }
           return
@@ -391,7 +391,7 @@ export function extractSimpleComparisons(
 
         // NOT can only wrap simple comparisons or null checks
         throw new Error(
-          `extractSimpleComparisons does not support 'not(${arg.name})'. NOT can only wrap comparison operators (eq, gt, gte, lt, lte, in) or null checks (isNull, isUndefined).`
+          `extractSimpleComparisons does not support 'not(${arg.name})'. NOT can only wrap comparison operators (eq, gt, gte, lt, lte, in) or null checks (isNull, isUndefined).`,
         )
       }
 
@@ -414,7 +414,7 @@ export function extractSimpleComparisons(
       ]
       if (unsupportedOps.includes(e.name)) {
         throw new Error(
-          `extractSimpleComparisons does not support '${e.name}' operator. Use parseWhereExpression with custom handlers for complex expressions.`
+          `extractSimpleComparisons does not support '${e.name}' operator. Use parseWhereExpression with custom handlers for complex expressions.`,
         )
       }
 
@@ -434,7 +434,7 @@ export function extractSimpleComparisons(
           })
         } else {
           throw new Error(
-            `extractSimpleComparisons requires a field reference for '${e.name}' operator.`
+            `extractSimpleComparisons requires a field reference for '${e.name}' operator.`,
           )
         }
         return
@@ -457,13 +457,13 @@ export function extractSimpleComparisons(
           })
         } else {
           throw new Error(
-            `extractSimpleComparisons requires simple field-value comparisons. Found complex expression for '${e.name}' operator.`
+            `extractSimpleComparisons requires simple field-value comparisons. Found complex expression for '${e.name}' operator.`,
           )
         }
       } else {
         // Unknown operator
         throw new Error(
-          `extractSimpleComparisons encountered unknown operator: '${e.name}'`
+          `extractSimpleComparisons encountered unknown operator: '${e.name}'`,
         )
       }
     }
@@ -504,7 +504,7 @@ export function parseLoadSubsetOptions(
         limit?: number
       }
     | undefined
-    | null
+    | null,
 ): {
   filters: Array<SimpleComparison>
   sorts: Array<ParsedOrderBy>

package/src/query/index.ts

@@ -10,7 +10,7 @@ export {
   type Source,
   type GetResult,
   type InferResultType,
-} from "./builder/index.js"
+} from './builder/index.js'
 
 // Expression functions exports
 export {
@@ -41,22 +41,22 @@ export {
   sum,
   min,
   max,
-} from "./builder/functions.js"
+} from './builder/functions.js'
 
 // Ref proxy utilities
-export type { Ref } from "./builder/types.js"
+export type { Ref } from './builder/types.js'
 
 // Compiler
-export { compileQuery } from "./compiler/index.js"
+export { compileQuery } from './compiler/index.js'
 
 // Live query collection utilities
 export {
   createLiveQueryCollection,
   liveQueryCollectionOptions,
-} from "./live-query-collection.js"
+} from './live-query-collection.js'
 
-export { type LiveQueryCollectionConfig } from "./live/types.js"
-export { type LiveQueryCollectionUtils } from "./live/collection-config-builder.js"
+export { type LiveQueryCollectionConfig } from './live/types.js'
+export { type LiveQueryCollectionUtils } from './live/collection-config-builder.js'
 
 // Predicate utilities for predicate push-down
 export {
@@ -65,7 +65,8 @@ export {
   minusWherePredicates,
   isOrderBySubset,
   isLimitSubset,
+  isOffsetLimitSubset,
   isPredicateSubset,
-} from "./predicate-utils.js"
+} from './predicate-utils.js'
 
-export { DeduplicatedLoadSubset } from "./subset-dedupe.js"
+export { DeduplicatedLoadSubset } from './subset-dedupe.js'