@tanstack/db 0.5.11 → 0.5.12

package/src/types.ts

@@ -1,9 +1,9 @@
-import type { IStreamBuilder } from "@tanstack/db-ivm"
-import type { Collection } from "./collection/index.js"
-import type { StandardSchemaV1 } from "@standard-schema/spec"
-import type { Transaction } from "./transactions"
-import type { BasicExpression, OrderBy } from "./query/ir.js"
-import type { EventEmitter } from "./event-emitter.js"
+import type { IStreamBuilder } from '@tanstack/db-ivm'
+import type { Collection } from './collection/index.js'
+import type { StandardSchemaV1 } from '@standard-schema/spec'
+import type { Transaction } from './transactions'
+import type { BasicExpression, OrderBy } from './query/ir.js'
+import type { EventEmitter } from './event-emitter.js'
 
 /**
  * Interface for a collection-like object that provides the necessary methods
@@ -124,7 +124,7 @@ export type MutationFnParams<T extends object = Record<string, unknown>> = {
 }
 
 export type MutationFn<T extends object = Record<string, unknown>> = (
-  params: MutationFnParams<T>
+  params: MutationFnParams<T>,
 ) => Promise<any>
 
 /**
@@ -238,9 +238,9 @@ export interface SubscriptionUnsubscribedEvent {
  * All subscription events
  */
 export type SubscriptionEvents = {
-  "status:change": SubscriptionStatusChangeEvent
-  "status:ready": SubscriptionStatusEvent<`ready`>
-  "status:loadingSubset": SubscriptionStatusEvent<`loadingSubset`>
+  'status:change': SubscriptionStatusChangeEvent
+  'status:ready': SubscriptionStatusEvent<`ready`>
+  'status:loadingSubset': SubscriptionStatusEvent<`loadingSubset`>
   unsubscribed: SubscriptionUnsubscribedEvent
 }
 
@@ -253,13 +253,52 @@ export interface Subscription extends EventEmitter<SubscriptionEvents> {
   readonly status: SubscriptionStatus
 }
 
+/**
+ * Cursor expressions for pagination, passed separately from the main `where` clause.
+ * The sync layer can choose to use cursor-based pagination (combining these with the where)
+ * or offset-based pagination (ignoring these and using the `offset` parameter).
+ *
+ * Neither expression includes the main `where` clause - they are cursor-specific only.
+ */
+export type CursorExpressions = {
+  /**
+   * Expression for rows greater than (after) the cursor value.
+   * For multi-column orderBy, this is a composite cursor using OR of conditions.
+   * Example for [col1 ASC, col2 DESC] with values [v1, v2]:
+   *   or(gt(col1, v1), and(eq(col1, v1), lt(col2, v2)))
+   */
+  whereFrom: BasicExpression<boolean>
+  /**
+   * Expression for rows equal to the current cursor value (first orderBy column only).
+   * Used to handle tie-breaking/duplicates at the boundary.
+   * Example: eq(col1, v1) or for Dates: and(gte(col1, v1), lt(col1, v1+1ms))
+   */
+  whereCurrent: BasicExpression<boolean>
+  /**
+   * The key of the last item that was loaded.
+   * Can be used by sync layers for tracking or deduplication.
+   */
+  lastKey?: string | number
+}
+
 export type LoadSubsetOptions = {
-  /** The where expression to filter the data */
+  /** The where expression to filter the data (does NOT include cursor expressions) */
   where?: BasicExpression<boolean>
   /** The order by clause to sort the data */
   orderBy?: OrderBy
   /** The limit of the data to load */
   limit?: number
+  /**
+   * Cursor expressions for cursor-based pagination.
+   * These are separate from `where` - the sync layer should combine them if using cursor-based pagination.
+   * Neither expression includes the main `where` clause.
+   */
+  cursor?: CursorExpressions
+  /**
+   * Row offset for offset-based pagination.
+   * The sync layer can use this instead of `cursor` if it prefers offset-based pagination.
+   */
+  offset?: number
   /**
    * The subscription that triggered the load.
    * Advanced sync implementations can use this for:
@@ -334,7 +373,7 @@ export interface OptimisticChangeMessage<
  * This follows the standard-schema specification: https://github.com/standard-schema/standard-schema
  */
 export type StandardSchema<T> = StandardSchemaV1 & {
-  "~standard": {
+  '~standard': {
     types?: {
       input: T
       output: T

package/src/utils/array-utils.ts

@@ -8,7 +8,7 @@
 export function findInsertPosition<T>(
   sortedArray: Array<[T, any]>,
   value: T,
-  compareFn: (a: T, b: T) => number
+  compareFn: (a: T, b: T) => number,
 ): number {
   let left = 0
   let right = sortedArray.length

package/src/utils/browser-polyfills.ts

@@ -7,7 +7,7 @@ export type IdleCallbackDeadline = {
 export type IdleCallbackFunction = (deadline: IdleCallbackDeadline) => void
 
 const requestIdleCallbackPolyfill = (
-  callback: IdleCallbackFunction
+  callback: IdleCallbackFunction,
 ): number => {
   // Use a very small timeout for the polyfill to simulate idle time
   const timeout = 0
@@ -26,7 +26,7 @@ const cancelIdleCallbackPolyfill = (id: number): void => {
 
 export const safeRequestIdleCallback: (
   callback: IdleCallbackFunction,
-  options?: { timeout?: number }
+  options?: { timeout?: number },
 ) => number =
   typeof window !== `undefined` && `requestIdleCallback` in window
     ? (callback, options) =>

package/src/utils/btree.ts

@@ -117,7 +117,7 @@ export class BTree<K = any, V = any> {
   public constructor(
     compare: (a: K, b: K) => number,
     entries?: Array<[K, V]>,
-    maxNodeSize?: number
+    maxNodeSize?: number,
   ) {
     this._maxNodeSize = maxNodeSize! >= 4 ? Math.min(maxNodeSize!, 256) : 32
     this._compare = compare
@@ -230,7 +230,7 @@ export class BTree<K = any, V = any> {
       0,
       (k, _v) => {
         results.push(k)
-      }
+      },
     )
     return results
   }
@@ -250,7 +250,7 @@ export class BTree<K = any, V = any> {
       key,
       this._compare,
       false,
-      reusedArray
+      reusedArray,
     )
   }
 
@@ -306,7 +306,7 @@ export class BTree<K = any, V = any> {
     high: K,
     includeHigh: boolean,
     onFound?: (k: K, v: V, counter: number) => void,
-    initialCounter?: number
+    initialCounter?: number,
   ): number
 
   /**
@@ -331,7 +331,7 @@ export class BTree<K = any, V = any> {
     high: K,
     includeHigh: boolean,
     onFound?: (k: K, v: V, counter: number) => { break?: R } | void,
-    initialCounter?: number
+    initialCounter?: number,
   ): R | number {
     const r = this._root.forRange(
       low,
@@ -340,7 +340,7 @@ export class BTree<K = any, V = any> {
       false,
       this,
       initialCounter || 0,
-      onFound
+      onFound,
     )
     return typeof r === `number` ? r : r.break!
   }
@@ -379,7 +379,7 @@ export class BTree<K = any, V = any> {
     high: K,
     includeHigh: boolean,
     onFound: (k: K, v: V, counter: number) => EditRangeResult<V, R> | void,
-    initialCounter?: number
+    initialCounter?: number,
   ): R | number {
     let root = this._root
     if (root.isShared) this._root = root = root.clone()
@@ -391,7 +391,7 @@ export class BTree<K = any, V = any> {
         true,
         this,
         initialCounter || 0,
-        onFound
+        onFound,
       )
       return typeof r === `number` ? r : r.break!
     } finally {
@@ -500,7 +500,7 @@ class BNode<K, V> {
     key: K,
     compare: (a: K, b: K) => number,
     inclusive: boolean,
-    reusedArray: [K, V]
+    reusedArray: [K, V],
   ): [K, V] | undefined {
     const i = this.indexOf(key, -1, compare)
     const indexOrLower = i < 0 ? ~i - 1 : inclusive ? i : i - 1
@@ -516,7 +516,7 @@ class BNode<K, V> {
     key: K,
     compare: (a: K, b: K) => number,
     inclusive: boolean,
-    reusedArray: [K, V]
+    reusedArray: [K, V],
   ): [K, V] | undefined {
     const i = this.indexOf(key, -1, compare)
     const indexOrLower = i < 0 ? ~i : inclusive ? i : i + 1
@@ -536,7 +536,7 @@ class BNode<K, V> {
     key: K,
     value: V,
     overwrite: boolean | undefined,
-    tree: BTree<K, V>
+    tree: BTree<K, V>,
   ): boolean | BNode<K, V> {
     let i = this.indexOf(key, -1, tree._compare)
     if (i < 0) {
@@ -636,7 +636,7 @@ class BNode<K, V> {
     editMode: boolean,
     tree: BTree<K, V>,
     count: number,
-    onFound?: (k: K, v: V, counter: number) => EditRangeResult<V, R> | void
+    onFound?: (k: K, v: V, counter: number) => EditRangeResult<V, R> | void,
   ): EditRangeResult<V, R> | number {
     const cmp = tree._compare
     let iLow, iHigh
@@ -732,7 +732,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
     key: K,
     compare: (a: K, b: K) => number,
     inclusive: boolean,
-    reusedArray: [K, V]
+    reusedArray: [K, V],
   ): [K, V] | undefined {
     const i = this.indexOf(key, 0, compare),
       children = this.children
@@ -741,7 +741,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
       key,
       compare,
       inclusive,
-      reusedArray
+      reusedArray,
     )
     if (result === undefined && i > 0) {
       return children[i - 1]!.maxPair(reusedArray)
@@ -753,7 +753,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
     key: K,
     compare: (a: K, b: K) => number,
     inclusive: boolean,
-    reusedArray: [K, V]
+    reusedArray: [K, V],
   ): [K, V] | undefined {
     const i = this.indexOf(key, 0, compare),
       children = this.children,
@@ -763,7 +763,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
       key,
       compare,
       inclusive,
-      reusedArray
+      reusedArray,
     )
     if (result === undefined && i < length - 1) {
       return children[i + 1]!.minPair(reusedArray)
@@ -778,7 +778,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
     key: K,
     value: V,
     overwrite: boolean | undefined,
-    tree: BTree<K, V>
+    tree: BTree<K, V>,
   ): boolean | BNodeInternal<K, V> {
     const c = this.children,
       max = tree._maxNodeSize,
@@ -854,7 +854,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
     const half = this.children.length >> 1
     return new BNodeInternal<K, V>(
       this.children.splice(half),
-      this.keys.splice(half)
+      this.keys.splice(half),
     )
   }
 
@@ -887,7 +887,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
     editMode: boolean,
     tree: BTree<K, V>,
     count: number,
-    onFound?: (k: K, v: V, counter: number) => EditRangeResult<V, R> | void
+    onFound?: (k: K, v: V, counter: number) => EditRangeResult<V, R> | void,
   ): EditRangeResult<V, R> | number {
     const cmp = tree._compare
     const keys = this.keys,
@@ -896,7 +896,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
       i = iLow
     const iHigh = Math.min(
       high === low ? iLow : this.indexOf(high, 0, cmp),
-      keys.length - 1
+      keys.length - 1,
     )
     if (!editMode) {
       // Simple case
@@ -908,7 +908,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
           editMode,
           tree,
           count,
-          onFound
+          onFound,
         )
         if (typeof result !== `number`) return result
         count = result
@@ -924,7 +924,7 @@ class BNodeInternal<K, V> extends BNode<K, V> {
             editMode,
             tree,
             count,
-            onFound
+            onFound,
           )
           // Note: if children[i] is empty then keys[i]=undefined.
           //       This is an invalid state, but it is fixed below.

package/src/utils/comparison.ts

@@ -1,4 +1,4 @@
-import type { CompareOptions } from "../query/builder/types"
+import type { CompareOptions } from '../query/builder/types'
 
 // WeakMap to store stable IDs for objects
 const objectIds = new WeakMap<object, number>()
@@ -84,7 +84,7 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => {
 export const descComparator = (
   a: unknown,
   b: unknown,
-  opts: CompareOptions
+  opts: CompareOptions,
 ): number => {
   return ascComparator(b, a, {
     ...opts,
@@ -93,7 +93,7 @@ export const descComparator = (
 }
 
 export function makeComparator(
-  opts: CompareOptions
+  opts: CompareOptions,
 ): (a: any, b: any) => number {
   return (a, b) => {
     if (opts.direction === `asc`) {

package/src/utils/cursor.ts

@@ -0,0 +1,78 @@
+import { and, eq, gt, lt, or } from '../query/builder/functions.js'
+import { Value } from '../query/ir.js'
+import type { BasicExpression, OrderBy } from '../query/ir.js'
+
+/**
+ * Builds a cursor expression for paginating through ordered results.
+ * For multi-column orderBy, creates a composite cursor that respects all columns.
+ *
+ * For [col1 ASC, col2 DESC] with values [v1, v2], produces:
+ *   or(
+ *     gt(col1, v1),                         // col1 > v1
+ *     and(eq(col1, v1), lt(col2, v2))       // col1 = v1 AND col2 < v2 (DESC)
+ *   )
+ *
+ * This creates a precise cursor that works with composite indexes on the backend.
+ *
+ * @param orderBy - The order-by clauses defining sort columns and directions
+ * @param values - The cursor values corresponding to each order-by column
+ * @returns A filter expression for rows after the cursor position, or undefined if empty
+ */
+export function buildCursor(
+  orderBy: OrderBy,
+  values: Array<unknown>,
+): BasicExpression<boolean> | undefined {
+  if (values.length === 0 || orderBy.length === 0) {
+    return undefined
+  }
+
+  // For single column, just use simple gt/lt
+  if (orderBy.length === 1) {
+    const { expression, compareOptions } = orderBy[0]!
+    const operator = compareOptions.direction === `asc` ? gt : lt
+    return operator(expression, new Value(values[0]))
+  }
+
+  // For multi-column, build the composite cursor:
+  // or(
+  //   gt(col1, v1),
+  //   and(eq(col1, v1), gt(col2, v2)),
+  //   and(eq(col1, v1), eq(col2, v2), gt(col3, v3)),
+  //   ...
+  // )
+  const clauses: Array<BasicExpression<boolean>> = []
+
+  for (let i = 0; i < orderBy.length && i < values.length; i++) {
+    const clause = orderBy[i]!
+    const value = values[i]
+
+    // Build equality conditions for all previous columns
+    const eqConditions: Array<BasicExpression<boolean>> = []
+    for (let j = 0; j < i; j++) {
+      const prevClause = orderBy[j]!
+      const prevValue = values[j]
+      eqConditions.push(eq(prevClause.expression, new Value(prevValue)))
+    }
+
+    // Add the comparison for the current column (respecting direction)
+    const operator = clause.compareOptions.direction === `asc` ? gt : lt
+    const comparison = operator(clause.expression, new Value(value))
+
+    if (eqConditions.length === 0) {
+      // First column: just the comparison
+      clauses.push(comparison)
+    } else {
+      // Subsequent columns: and(eq(prev...), comparison)
+      // We need to spread into and() which expects at least 2 args
+      const allConditions = [...eqConditions, comparison]
+      clauses.push(allConditions.reduce((acc, cond) => and(acc, cond)))
+    }
+  }
+
+  // Combine all clauses with OR
+  if (clauses.length === 1) {
+    return clauses[0]!
+  }
+  // Use reduce to combine with or() which expects exactly 2 args
+  return clauses.reduce((acc, clause) => or(acc, clause))
+}

package/src/utils/index-optimization.ts

@@ -15,12 +15,12 @@
  * - Optimizes IN array expressions
  */
 
-import { DEFAULT_COMPARE_OPTIONS } from "../utils.js"
-import { ReverseIndex } from "../indexes/reverse-index.js"
-import type { CompareOptions } from "../query/builder/types.js"
-import type { IndexInterface, IndexOperation } from "../indexes/base-index.js"
-import type { BasicExpression } from "../query/ir.js"
-import type { CollectionLike } from "../types.js"
+import { DEFAULT_COMPARE_OPTIONS } from '../utils.js'
+import { ReverseIndex } from '../indexes/reverse-index.js'
+import type { CompareOptions } from '../query/builder/types.js'
+import type { IndexInterface, IndexOperation } from '../indexes/base-index.js'
+import type { BasicExpression } from '../query/ir.js'
+import type { CollectionLike } from '../types.js'
 
 /**
  * Result of index-based query optimization
@@ -36,7 +36,7 @@ export interface OptimizationResult<TKey> {
 export function findIndexForField<TKey extends string | number>(
   collection: CollectionLike<any, TKey>,
   fieldPath: Array<string>,
-  compareOptions?: CompareOptions
+  compareOptions?: CompareOptions,
 ): IndexInterface<TKey> | undefined {
   const compareOpts = compareOptions ?? {
     ...DEFAULT_COMPARE_OPTIONS,
@@ -98,7 +98,7 @@ export function optimizeExpressionWithIndexes<
   TKey extends string | number,
 >(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   return optimizeQueryRecursive(expression, collection)
 }
@@ -108,7 +108,7 @@ export function optimizeExpressionWithIndexes<
  */
 function optimizeQueryRecursive<T extends object, TKey extends string | number>(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type === `func`) {
     switch (expression.name) {
@@ -172,7 +172,7 @@ function optimizeCompoundRangeQuery<
   TKey extends string | number,
 >(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type !== `func` || expression.args.length < 2) {
     return { canOptimize: false, matchingKeys: new Set() }
@@ -305,7 +305,7 @@ function optimizeSimpleComparison<
   TKey extends string | number,
 >(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type !== `func` || expression.args.length !== 2) {
     return { canOptimize: false, matchingKeys: new Set() }
@@ -405,7 +405,7 @@ function canOptimizeSimpleComparison<
  */
 function optimizeAndExpression<T extends object, TKey extends string | number>(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type !== `func` || expression.args.length < 2) {
     return { canOptimize: false, matchingKeys: new Set() }
@@ -457,7 +457,7 @@ function canOptimizeAndExpression<
  */
 function optimizeOrExpression<T extends object, TKey extends string | number>(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type !== `func` || expression.args.length < 2) {
     return { canOptimize: false, matchingKeys: new Set() }
@@ -506,7 +506,7 @@ function optimizeInArrayExpression<
   TKey extends string | number,
 >(
   expression: BasicExpression,
-  collection: CollectionLike<T, TKey>
+  collection: CollectionLike<T, TKey>,
 ): OptimizationResult<TKey> {
   if (expression.type !== `func` || expression.args.length !== 2) {
     return { canOptimize: false, matchingKeys: new Set() }

package/src/utils.ts

@@ -2,7 +2,7 @@
  * Generic utility functions
  */
 
-import type { CompareOptions } from "./query/builder/types"
+import type { CompareOptions } from './query/builder/types'
 
 interface TypedArray {
   length: number
@@ -36,7 +36,7 @@ export function deepEquals(a: any, b: any): boolean {
 function deepEqualsInternal(
   a: any,
   b: any,
-  visited: Map<object, object>
+  visited: Map<object, object>,
 ): boolean {
   // Handle strict equality (primitives, same reference)
   if (a === b) return true
@@ -154,7 +154,7 @@ function deepEqualsInternal(
     visited.set(a, b)
 
     const result = a.every((item, index) =>
-      deepEqualsInternal(item, b[index], visited)
+      deepEqualsInternal(item, b[index], visited),
     )
     visited.delete(a)
     return result
@@ -180,7 +180,7 @@ function deepEqualsInternal(
 
     // Check if all keys exist in both objects and their values are equal
     const result = keysA.every(
-      (key) => key in b && deepEqualsInternal(a[key], b[key], visited)
+      (key) => key in b && deepEqualsInternal(a[key], b[key], visited),
     )
 
     visited.delete(a)