@tanstack/db 0.5.24 → 0.5.26

package/src/indexes/btree-index.ts

@@ -1,6 +1,10 @@
 import { compareKeys } from '@tanstack/db-ivm'
 import { BTree } from '../utils/btree.js'
-import { defaultComparator, normalizeValue } from '../utils/comparison.js'
+import {
+  defaultComparator,
+  denormalizeUndefined,
+  normalizeForBTree,
+} from '../utils/comparison.js'
 import { BaseIndex } from './base-index.js'
 import type { CompareOptions } from '../query/builder/types.js'
 import type { BasicExpression } from '../query/ir.js'
@@ -29,7 +33,7 @@ export interface RangeQueryOptions {
  * This maintains items in sorted order and provides efficient range operations
  */
 export class BTreeIndex<
-  TKey extends string | number = string | number,
+  TKey extends string | number | undefined = string | number | undefined,
 > extends BaseIndex<TKey> {
   public readonly supportedOperations = new Set<IndexOperation>([
     `eq`,
@@ -55,7 +59,16 @@ export class BTreeIndex<
     options?: any,
   ) {
     super(id, expression, name, options)
-    this.compareFn = options?.compareFn ?? defaultComparator
+
+    // Get the base compare function
+    const baseCompareFn = options?.compareFn ?? defaultComparator
+
+    // Wrap it to denormalize sentinels before comparison
+    // This ensures UNDEFINED_SENTINEL is converted back to undefined
+    // before being passed to the baseCompareFn (which can be user-provided and is unaware of the UNDEFINED_SENTINEL)
+    this.compareFn = (a: any, b: any) =>
+      baseCompareFn(denormalizeUndefined(a), denormalizeUndefined(b))
+
     if (options?.compareOptions) {
       this.compareOptions = options!.compareOptions
     }
@@ -78,7 +91,7 @@ export class BTreeIndex<
     }
 
     // Normalize the value for Map key usage
-    const normalizedValue = normalizeValue(indexedValue)
+    const normalizedValue = normalizeForBTree(indexedValue)
 
     // Check if this value already exists
     if (this.valueMap.has(normalizedValue)) {
@@ -111,7 +124,7 @@ export class BTreeIndex<
     }
 
     // Normalize the value for Map key usage
-    const normalizedValue = normalizeValue(indexedValue)
+    const normalizedValue = normalizeForBTree(indexedValue)
 
     if (this.valueMap.has(normalizedValue)) {
       const keySet = this.valueMap.get(normalizedValue)!
@@ -207,7 +220,7 @@ export class BTreeIndex<
    * Performs an equality lookup
    */
   equalityLookup(value: any): Set<TKey> {
-    const normalizedValue = normalizeValue(value)
+    const normalizedValue = normalizeForBTree(value)
     return new Set(this.valueMap.get(normalizedValue) ?? [])
   }
 
@@ -219,10 +232,15 @@ export class BTreeIndex<
     const { from, to, fromInclusive = true, toInclusive = true } = options
     const result = new Set<TKey>()
 
-    const normalizedFrom = normalizeValue(from)
-    const normalizedTo = normalizeValue(to)
-    const fromKey = normalizedFrom ?? this.orderedEntries.minKey()
-    const toKey = normalizedTo ?? this.orderedEntries.maxKey()
+    // Check if from/to were explicitly provided (even if undefined)
+    // vs not provided at all (should use min/max key)
+    const hasFrom = `from` in options
+    const hasTo = `to` in options
+
+    const fromKey = hasFrom
+      ? normalizeForBTree(from)
+      : this.orderedEntries.minKey()
+    const toKey = hasTo ? normalizeForBTree(to) : this.orderedEntries.maxKey()
 
     this.orderedEntries.forRange(
       fromKey,
@@ -250,29 +268,43 @@ export class BTreeIndex<
    */
   rangeQueryReversed(options: RangeQueryOptions = {}): Set<TKey> {
     const { from, to, fromInclusive = true, toInclusive = true } = options
+    const hasFrom = `from` in options
+    const hasTo = `to` in options
+
+    // Swap from/to for reversed query, respecting explicit undefined values
     return this.rangeQuery({
-      from: to ?? this.orderedEntries.maxKey(),
-      to: from ?? this.orderedEntries.minKey(),
+      from: hasTo ? to : this.orderedEntries.maxKey(),
+      to: hasFrom ? from : this.orderedEntries.minKey(),
       fromInclusive: toInclusive,
       toInclusive: fromInclusive,
     })
   }
 
+  /**
+   * Internal method for taking items from the index.
+   * @param n - The number of items to return
+   * @param nextPair - Function to get the next pair from the BTree
+   * @param from - Already normalized! undefined means "start from beginning/end", sentinel means "start from the key undefined"
+   * @param filterFn - Optional filter function
+   * @param reversed - Whether to reverse the order of keys within each value
+   */
   private takeInternal(
     n: number,
     nextPair: (k?: any) => [any, any] | undefined,
-    from?: any,
+    from: any,
     filterFn?: (key: TKey) => boolean,
     reversed: boolean = false,
   ): Array<TKey> {
     const keysInResult: Set<TKey> = new Set()
     const result: Array<TKey> = []
     let pair: [any, any] | undefined
-    let key = normalizeValue(from)
+    let key = from // Use as-is - it's already normalized by the caller
 
     while ((pair = nextPair(key)) !== undefined && result.length < n) {
       key = pair[0]
-      const keys = this.valueMap.get(key)
+      const keys = this.valueMap.get(key) as
+        | Set<Exclude<TKey, undefined>>
+        | undefined
       if (keys && keys.size > 0) {
         // Sort keys for deterministic order, reverse if needed
         const sorted = Array.from(keys).sort(compareKeys)
@@ -291,29 +323,60 @@ export class BTreeIndex<
   }
 
   /**
-   * Returns the next n items after the provided item or the first n items if no from item is provided.
+   * Returns the next n items after the provided item.
    * @param n - The number of items to return
-   * @param from - The item to start from (exclusive). Starts from the smallest item (inclusive) if not provided.
-   * @returns The next n items after the provided key. Returns the first n items if no from item is provided.
+   * @param from - The item to start from (exclusive).
+   * @returns The next n items after the provided key.
    */
-  take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
+  take(n: number, from: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
     const nextPair = (k?: any) => this.orderedEntries.nextHigherPair(k)
-    return this.takeInternal(n, nextPair, from, filterFn)
+    // Normalize the from value
+    const normalizedFrom = normalizeForBTree(from)
+    return this.takeInternal(n, nextPair, normalizedFrom, filterFn)
   }
 
   /**
-   * Returns the next n items **before** the provided item (in descending order) or the last n items if no from item is provided.
+   * Returns the first n items from the beginning.
    * @param n - The number of items to return
-   * @param from - The item to start from (exclusive). Starts from the largest item (inclusive) if not provided.
-   * @returns The next n items **before** the provided key. Returns the last n items if no from item is provided.
+   * @param filterFn - Optional filter function
+   * @returns The first n items
+   */
+  takeFromStart(n: number, filterFn?: (key: TKey) => boolean): Array<TKey> {
+    const nextPair = (k?: any) => this.orderedEntries.nextHigherPair(k)
+    // Pass undefined to mean "start from beginning" (BTree's native behavior)
+    return this.takeInternal(n, nextPair, undefined, filterFn)
+  }
+
+  /**
+   * Returns the next n items **before** the provided item (in descending order).
+   * @param n - The number of items to return
+   * @param from - The item to start from (exclusive). Required.
+   * @returns The next n items **before** the provided key.
    */
   takeReversed(
     n: number,
-    from?: any,
+    from: any,
+    filterFn?: (key: TKey) => boolean,
+  ): Array<TKey> {
+    const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k)
+    // Normalize the from value
+    const normalizedFrom = normalizeForBTree(from)
+    return this.takeInternal(n, nextPair, normalizedFrom, filterFn, true)
+  }
+
+  /**
+   * Returns the last n items from the end.
+   * @param n - The number of items to return
+   * @param filterFn - Optional filter function
+   * @returns The last n items
+   */
+  takeReversedFromEnd(
+    n: number,
     filterFn?: (key: TKey) => boolean,
   ): Array<TKey> {
     const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k)
-    return this.takeInternal(n, nextPair, from, filterFn, true)
+    // Pass undefined to mean "start from end" (BTree's native behavior)
+    return this.takeInternal(n, nextPair, undefined, filterFn, true)
   }
 
   /**
@@ -323,7 +386,7 @@ export class BTreeIndex<
     const result = new Set<TKey>()
 
     for (const value of values) {
-      const normalizedValue = normalizeValue(value)
+      const normalizedValue = normalizeForBTree(value)
       const keys = this.valueMap.get(normalizedValue)
       if (keys) {
         keys.forEach((key) => result.add(key))
@@ -341,17 +404,25 @@ export class BTreeIndex<
   get orderedEntriesArray(): Array<[any, Set<TKey>]> {
     return this.orderedEntries
       .keysArray()
-      .map((key) => [key, this.valueMap.get(key) ?? new Set()])
+      .map((key) => [
+        denormalizeUndefined(key),
+        this.valueMap.get(key) ?? new Set(),
+      ])
   }
 
   get orderedEntriesArrayReversed(): Array<[any, Set<TKey>]> {
-    return this.takeReversed(this.orderedEntries.size).map((key) => [
-      key,
+    return this.takeReversedFromEnd(this.orderedEntries.size).map((key) => [
+      denormalizeUndefined(key),
       this.valueMap.get(key) ?? new Set(),
     ])
   }
 
   get valueMapData(): Map<any, Set<TKey>> {
-    return this.valueMap
+    // Return a new Map with denormalized keys
+    const result = new Map<any, Set<TKey>>()
+    for (const [key, value] of this.valueMap) {
+      result.set(denormalizeUndefined(key), value)
+    }
+    return result
   }
 }

package/src/indexes/reverse-index.ts

@@ -36,18 +36,29 @@ export class ReverseIndex<
     return this.originalIndex.rangeQuery(options)
   }
 
-  take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
+  take(n: number, from: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
     return this.originalIndex.takeReversed(n, from, filterFn)
   }
 
+  takeFromStart(n: number, filterFn?: (key: TKey) => boolean): Array<TKey> {
+    return this.originalIndex.takeReversedFromEnd(n, filterFn)
+  }
+
   takeReversed(
     n: number,
-    from?: any,
+    from: any,
     filterFn?: (key: TKey) => boolean,
   ): Array<TKey> {
     return this.originalIndex.take(n, from, filterFn)
   }
 
+  takeReversedFromEnd(
+    n: number,
+    filterFn?: (key: TKey) => boolean,
+  ): Array<TKey> {
+    return this.originalIndex.takeFromStart(n, filterFn)
+  }
+
   get orderedEntriesArray(): Array<[any, Set<TKey>]> {
     return this.originalIndex.orderedEntriesArrayReversed
   }

package/src/query/builder/index.ts

@@ -910,8 +910,26 @@ export type QueryResult<T> = GetResult<ExtractContext<T>>
 // Export the types from types.ts for convenience
 export type {
   Context,
+  ContextSchema,
   Source,
   GetResult,
   RefLeaf as Ref,
   InferResultType,
+  // Types used in public method signatures that must be exported
+  // for declaration emit to work (see https://github.com/TanStack/db/issues/1012)
+  SchemaFromSource,
+  InferCollectionType,
+  MergeContextWithJoinType,
+  MergeContextForJoinCallback,
+  ApplyJoinOptionalityToMergedSchema,
+  ResultTypeFromSelect,
+  WithResult,
+  JoinOnCallback,
+  RefsForContext,
+  WhereCallback,
+  OrderByCallback,
+  GroupByCallback,
+  SelectObject,
+  FunctionalHavingRow,
+  Prettify,
 } from './types.js'

package/src/query/compiler/evaluators.ts

@@ -336,7 +336,7 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any {
       const valueEvaluator = compiledArgs[0]!
       const arrayEvaluator = compiledArgs[1]!
       return (data) => {
-        const value = valueEvaluator(data)
+        const value = normalizeValue(valueEvaluator(data))
         const array = arrayEvaluator(data)
         // In 3-valued logic, if the value is null/undefined, return UNKNOWN
         if (isUnknown(value)) {
@@ -345,7 +345,7 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any {
         if (!Array.isArray(array)) {
           return false
         }
-        return array.includes(value)
+        return array.some((item) => normalizeValue(item) === value)
       }
     }
 

package/src/query/compiler/joins.ts

@@ -11,6 +11,7 @@ import {
   UnsupportedJoinSourceTypeError,
   UnsupportedJoinTypeError,
 } from '../../errors.js'
+import { normalizeValue } from '../../utils/comparison.js'
 import { ensureIndexForField } from '../../indexes/auto-index.js'
 import { PropRef, followRef } from '../ir.js'
 import { inArray } from '../builder/functions.js'
@@ -188,7 +189,7 @@ function processJoin(
   let mainPipeline = pipeline.pipe(
     map(([currentKey, namespacedRow]) => {
       // Extract the join key from the main source expression
-      const mainKey = compiledMainExpr(namespacedRow)
+      const mainKey = normalizeValue(compiledMainExpr(namespacedRow))
 
       // Return [joinKey, [originalKey, namespacedRow]]
       return [mainKey, [currentKey, namespacedRow]] as [
@@ -205,7 +206,7 @@ function processJoin(
       const namespacedRow: NamespacedRow = { [joinedSource]: row }
 
       // Extract the join key from the joined source expression
-      const joinedKey = compiledJoinedExpr(namespacedRow)
+      const joinedKey = normalizeValue(compiledJoinedExpr(namespacedRow))
 
       // Return [joinKey, [originalKey, namespacedRow]]
       return [joinedKey, [currentKey, namespacedRow]] as [

package/src/query/index.ts

@@ -7,11 +7,28 @@ export {
   type InitialQueryBuilder,
   type QueryBuilder,
   type Context,
+  type ContextSchema,
   type Source,
   type GetResult,
   type InferResultType,
   type ExtractContext,
   type QueryResult,
+  // Types needed for declaration emit (https://github.com/TanStack/db/issues/1012)
+  type SchemaFromSource,
+  type InferCollectionType,
+  type MergeContextWithJoinType,
+  type MergeContextForJoinCallback,
+  type ApplyJoinOptionalityToMergedSchema,
+  type ResultTypeFromSelect,
+  type WithResult,
+  type JoinOnCallback,
+  type RefsForContext,
+  type WhereCallback,
+  type OrderByCallback,
+  type GroupByCallback,
+  type SelectObject,
+  type FunctionalHavingRow,
+  type Prettify,
 } from './builder/index.js'
 
 // Expression functions exports

package/src/query/live/collection-config-builder.ts

@@ -336,6 +336,7 @@ export class CollectionConfigBuilder<
 
       // Always run the graph if subscribed (eager execution)
       if (syncState.subscribedToAllCollections) {
+        let callbackCalled = false
         while (syncState.graph.pendingWork()) {
           syncState.graph.run()
           // Flush accumulated changes after each graph step to commit them as one transaction.
@@ -343,6 +344,14 @@ export class CollectionConfigBuilder<
           // duplicate key errors when the full join result arrives in the same step.
           syncState.flushPendingChanges?.()
           callback?.()
+          callbackCalled = true
+        }
+
+        // Ensure the callback runs at least once even when the graph has no pending work.
+        // This handles lazy loading scenarios where setWindow() increases the limit or
+        // an async loadSubset completes and we need to re-check if more data is needed.
+        if (!callbackCalled) {
+          callback?.()
         }
 
         // On the initial run, we may need to do an empty commit to ensure that
@@ -830,17 +839,16 @@ export class CollectionConfigBuilder<
       return
     }
 
+    const subscribedToAll = this.currentSyncState?.subscribedToAllCollections
+    const allReady = this.allCollectionsReady()
+    const isLoading = this.liveQueryCollection?.isLoadingSubset
     // Mark ready when:
     // 1. All subscriptions are set up (subscribedToAllCollections)
     // 2. All source collections are ready
     // 3. The live query collection is not loading subset data
     // This prevents marking the live query ready before its data is processed
     // (fixes issue where useLiveQuery returns isReady=true with empty data)
-    if (
-      this.currentSyncState?.subscribedToAllCollections &&
-      this.allCollectionsReady() &&
-      !this.liveQueryCollection?.isLoadingSubset
-    ) {
+    if (subscribedToAll && allReady && !isLoading) {
       markReady()
     }
   }