@tanstack/db 0.4.2 → 0.4.4

package/src/collection/state.ts

@@ -217,7 +217,11 @@ export class CollectionStateManager<
     triggeredByUserAction: boolean = false
   ): void {
     // Skip redundant recalculations when we're in the middle of committing sync transactions
-    if (this.isCommittingSyncTransactions) {
+    // While the sync pipeline is replaying a large batch we still want to honour
+    // fresh optimistic mutations from the UI. Only skip recompute for the
+    // internal sync-driven redraws; user-triggered work (triggeredByUserAction)
+    // must run so live queries stay responsive during long commits.
+    if (this.isCommittingSyncTransactions && !triggeredByUserAction) {
       return
     }
 
@@ -708,10 +712,23 @@ export class CollectionStateManager<
 
         // Check if this sync operation is redundant with a completed optimistic operation
         const completedOp = completedOptimisticOps.get(key)
-        const isRedundantSync =
-          completedOp &&
-          newVisibleValue !== undefined &&
-          deepEquals(completedOp.value, newVisibleValue)
+        let isRedundantSync = false
+
+        if (completedOp) {
+          if (
+            completedOp.type === `delete` &&
+            previousVisibleValue !== undefined &&
+            newVisibleValue === undefined &&
+            deepEquals(completedOp.value, previousVisibleValue)
+          ) {
+            isRedundantSync = true
+          } else if (
+            newVisibleValue !== undefined &&
+            deepEquals(completedOp.value, newVisibleValue)
+          ) {
+            isRedundantSync = true
+          }
+        }
 
         if (!isRedundantSync) {
           if (

package/src/collection/subscription.ts

@@ -1,11 +1,11 @@
 import { ensureIndexForExpression } from "../indexes/auto-index.js"
-import { and } from "../query/index.js"
+import { and } from "../query/builder/functions.js"
 import {
   createFilterFunctionFromExpression,
   createFilteredCallback,
 } from "./change-events.js"
 import type { BasicExpression } from "../query/ir.js"
-import type { BaseIndex } from "../indexes/base-index.js"
+import type { IndexInterface } from "../indexes/base-index.js"
 import type { ChangeMessage } from "../types.js"
 import type { CollectionImpl } from "./index.js"
 
@@ -38,7 +38,7 @@ export class CollectionSubscription {
 
   private filteredCallback: (changes: Array<ChangeMessage<any, any>>) => void
 
-  private orderByIndex: BaseIndex<string | number> | undefined
+  private orderByIndex: IndexInterface<string | number> | undefined
 
   constructor(
     private collection: CollectionImpl<any, any, any, any, any>,
@@ -65,7 +65,7 @@ export class CollectionSubscription {
       : this.callback
   }
 
-  setOrderByIndex(index: BaseIndex<any>) {
+  setOrderByIndex(index: IndexInterface<any>) {
     this.orderByIndex = index
   }
 

package/src/collection/sync.ts

@@ -7,6 +7,7 @@ import {
   SyncTransactionAlreadyCommittedError,
   SyncTransactionAlreadyCommittedWriteError,
 } from "../errors"
+import { deepEquals } from "../utils"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
 import type { ChangeMessage, CollectionConfig } from "../types"
 import type { CollectionImpl } from "./index.js"
@@ -84,6 +85,8 @@ export class CollectionSyncManager<
           }
           const key = this.config.getKey(messageWithoutKey.value)
 
+          let messageType = messageWithoutKey.type
+
           // Check if an item with this key already exists when inserting
           if (messageWithoutKey.type === `insert`) {
             const insertingIntoExistingSynced = state.syncedData.has(key)
@@ -96,17 +99,29 @@ export class CollectionSyncManager<
               !hasPendingDeleteForKey &&
               !isTruncateTransaction
             ) {
-              throw new DuplicateKeySyncError(key, this.id)
+              const existingValue = state.syncedData.get(key)
+              if (
+                existingValue !== undefined &&
+                deepEquals(existingValue, messageWithoutKey.value)
+              ) {
+                // The "insert" is an echo of a value we already have locally.
+                // Treat it as an update so we preserve optimistic intent without
+                // throwing a duplicate-key error during reconciliation.
+                messageType = `update`
+              } else {
+                throw new DuplicateKeySyncError(key, this.id)
+              }
             }
           }
 
           const message: ChangeMessage<TOutput> = {
             ...messageWithoutKey,
+            type: messageType,
             key,
           }
           pendingTransaction.operations.push(message)
 
-          if (messageWithoutKey.type === `delete`) {
+          if (messageType === `delete`) {
             pendingTransaction.deletedKeys.add(key)
           }
         },

package/src/indexes/auto-index.ts

@@ -1,4 +1,6 @@
+import { DEFAULT_COMPARE_OPTIONS } from "../utils"
 import { BTreeIndex } from "./btree-index"
+import type { CompareOptions } from "../query/builder/types"
 import type { BasicExpression } from "../query/ir"
 import type { CollectionImpl } from "../collection/index.js"
 
@@ -30,6 +32,7 @@ export function ensureIndexForField<
   fieldName: string,
   fieldPath: Array<string>,
   collection: CollectionImpl<T, TKey, any, any, any>,
+  compareOptions: CompareOptions = DEFAULT_COMPARE_OPTIONS,
   compareFn?: (a: any, b: any) => number
 ) {
   if (!shouldAutoIndex(collection)) {
@@ -37,8 +40,10 @@ export function ensureIndexForField<
   }
 
   // Check if we already have an index for this field
-  const existingIndex = Array.from(collection.indexes.values()).find((index) =>
-    index.matchesField(fieldPath)
+  const existingIndex = Array.from(collection.indexes.values()).find(
+    (index) =>
+      index.matchesField(fieldPath) &&
+      index.matchesCompareOptions(compareOptions)
   )
 
   if (existingIndex) {
@@ -50,7 +55,7 @@ export function ensureIndexForField<
     collection.createIndex((row) => (row as any)[fieldName], {
       name: `auto_${fieldName}`,
       indexType: BTreeIndex,
-      options: compareFn ? { compareFn } : {},
+      options: compareFn ? { compareFn, compareOptions } : {},
     })
   } catch (error) {
     console.warn(`Failed to create auto-index for field "${fieldName}":`, error)

package/src/indexes/base-index.ts

@@ -1,6 +1,9 @@
 import { compileSingleRowExpression } from "../query/compiler/evaluators.js"
 import { comparisonFunctions } from "../query/builder/functions.js"
-import type { BasicExpression } from "../query/ir.js"
+import { DEFAULT_COMPARE_OPTIONS, deepEquals } from "../utils.js"
+import type { RangeQueryOptions } from "./btree-index.js"
+import type { CompareOptions } from "../query/builder/types.js"
+import type { BasicExpression, OrderByDirection } from "../query/ir.js"
 
 /**
  * Operations that indexes can support, imported from available comparison functions
@@ -22,12 +25,57 @@ export interface IndexStats {
   readonly lastUpdated: Date
 }
 
+export interface IndexInterface<
+  TKey extends string | number = string | number,
+> {
+  add: (key: TKey, item: any) => void
+  remove: (key: TKey, item: any) => void
+  update: (key: TKey, oldItem: any, newItem: any) => void
+
+  build: (entries: Iterable<[TKey, any]>) => void
+  clear: () => void
+
+  lookup: (operation: IndexOperation, value: any) => Set<TKey>
+
+  equalityLookup: (value: any) => Set<TKey>
+  inArrayLookup: (values: Array<any>) => Set<TKey>
+
+  rangeQuery: (options: RangeQueryOptions) => Set<TKey>
+  rangeQueryReversed: (options: RangeQueryOptions) => Set<TKey>
+
+  take: (
+    n: number,
+    from?: TKey,
+    filterFn?: (key: TKey) => boolean
+  ) => Array<TKey>
+  takeReversed: (
+    n: number,
+    from?: TKey,
+    filterFn?: (key: TKey) => boolean
+  ) => Array<TKey>
+
+  get keyCount(): number
+  get orderedEntriesArray(): Array<[any, Set<TKey>]>
+  get orderedEntriesArrayReversed(): Array<[any, Set<TKey>]>
+
+  get indexedKeysSet(): Set<TKey>
+  get valueMapData(): Map<any, Set<TKey>>
+
+  supports: (operation: IndexOperation) => boolean
+
+  matchesField: (fieldPath: Array<string>) => boolean
+  matchesCompareOptions: (compareOptions: CompareOptions) => boolean
+  matchesDirection: (direction: OrderByDirection) => boolean
+
+  getStats: () => IndexStats
+}
+
 /**
  * Base abstract class that all index types extend
  */
-export abstract class BaseIndex<
-  TKey extends string | number = string | number,
-> {
+export abstract class BaseIndex<TKey extends string | number = string | number>
+  implements IndexInterface<TKey>
+{
   public readonly id: number
   public readonly name?: string
   public readonly expression: BasicExpression
@@ -36,6 +84,7 @@ export abstract class BaseIndex<
   protected lookupCount = 0
   protected totalLookupTime = 0
   protected lastUpdated = new Date()
+  protected compareOptions: CompareOptions
 
   constructor(
     id: number,
@@ -45,6 +94,7 @@ export abstract class BaseIndex<
   ) {
     this.id = id
     this.expression = expression
+    this.compareOptions = DEFAULT_COMPARE_OPTIONS
     this.name = name
     this.initialize(options)
   }
@@ -61,7 +111,20 @@ export abstract class BaseIndex<
     from?: TKey,
     filterFn?: (key: TKey) => boolean
   ): Array<TKey>
+  abstract takeReversed(
+    n: number,
+    from?: TKey,
+    filterFn?: (key: TKey) => boolean
+  ): Array<TKey>
   abstract get keyCount(): number
+  abstract equalityLookup(value: any): Set<TKey>
+  abstract inArrayLookup(values: Array<any>): Set<TKey>
+  abstract rangeQuery(options: RangeQueryOptions): Set<TKey>
+  abstract rangeQueryReversed(options: RangeQueryOptions): Set<TKey>
+  abstract get orderedEntriesArray(): Array<[any, Set<TKey>]>
+  abstract get orderedEntriesArrayReversed(): Array<[any, Set<TKey>]>
+  abstract get indexedKeysSet(): Set<TKey>
+  abstract get valueMapData(): Map<any, Set<TKey>>
 
   // Common methods
   supports(operation: IndexOperation): boolean {
@@ -76,6 +139,33 @@ export abstract class BaseIndex<
     )
   }
 
+  /**
+   * Checks if the compare options match the index's compare options.
+   * The direction is ignored because the index can be reversed if the direction is different.
+   */
+  matchesCompareOptions(compareOptions: CompareOptions): boolean {
+    const thisCompareOptionsWithoutDirection = {
+      ...this.compareOptions,
+      direction: undefined,
+    }
+    const compareOptionsWithoutDirection = {
+      ...compareOptions,
+      direction: undefined,
+    }
+
+    return deepEquals(
+      thisCompareOptionsWithoutDirection,
+      compareOptionsWithoutDirection
+    )
+  }
+
+  /**
+   * Checks if the index matches the provided direction.
+   */
+  matchesDirection(direction: OrderByDirection): boolean {
+    return this.compareOptions.direction === direction
+  }
+
   getStats(): IndexStats {
     return {
       entryCount: this.keyCount,

package/src/indexes/btree-index.ts

@@ -1,6 +1,7 @@
 import { BTree } from "../utils/btree.js"
 import { defaultComparator, normalizeValue } from "../utils/comparison.js"
 import { BaseIndex } from "./base-index.js"
+import type { CompareOptions } from "../query/builder/types.js"
 import type { BasicExpression } from "../query/ir.js"
 import type { IndexOperation } from "./base-index.js"
 
@@ -9,6 +10,7 @@ import type { IndexOperation } from "./base-index.js"
  */
 export interface BTreeIndexOptions {
   compareFn?: (a: any, b: any) => number
+  compareOptions?: CompareOptions
 }
 
 /**
@@ -53,6 +55,9 @@ export class BTreeIndex<
   ) {
     super(id, expression, name, options)
     this.compareFn = options?.compareFn ?? defaultComparator
+    if (options?.compareOptions) {
+      this.compareOptions = options!.compareOptions
+    }
     this.orderedEntries = new BTree(this.compareFn)
   }
 
@@ -240,18 +245,31 @@ export class BTreeIndex<
   }
 
   /**
-   * Returns the next n items after the provided item or the first n items if no from item is provided.
-   * @param n - The number of items to return
-   * @param from - The item to start from (exclusive). Starts from the smallest item (inclusive) if not provided.
-   * @returns The next n items after the provided key. Returns the first n items if no from item is provided.
+   * Performs a reversed range query
    */
-  take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
+  rangeQueryReversed(options: RangeQueryOptions = {}): Set<TKey> {
+    const { from, to, fromInclusive = true, toInclusive = true } = options
+    return this.rangeQuery({
+      from: to ?? this.orderedEntries.maxKey(),
+      to: from ?? this.orderedEntries.minKey(),
+      fromInclusive: toInclusive,
+      toInclusive: fromInclusive,
+    })
+  }
+
+  private takeInternal(
+    n: number,
+    nextPair: (k?: any) => [any, any] | undefined,
+    from?: any,
+    filterFn?: (key: TKey) => boolean
+  ): Array<TKey> {
     const keysInResult: Set<TKey> = new Set()
     const result: Array<TKey> = []
-    const nextKey = (k?: any) => this.orderedEntries.nextHigherKey(k)
+    let pair: [any, any] | undefined
     let key = normalizeValue(from)
 
-    while ((key = nextKey(key)) && result.length < n) {
+    while ((pair = nextPair(key)) !== undefined && result.length < n) {
+      key = pair[0]
       const keys = this.valueMap.get(key)
       if (keys) {
         const it = keys.values()
@@ -268,6 +286,32 @@ export class BTreeIndex<
     return result
   }
 
+  /**
+   * Returns the next n items after the provided item or the first n items if no from item is provided.
+   * @param n - The number of items to return
+   * @param from - The item to start from (exclusive). Starts from the smallest item (inclusive) if not provided.
+   * @returns The next n items after the provided key. Returns the first n items if no from item is provided.
+   */
+  take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
+    const nextPair = (k?: any) => this.orderedEntries.nextHigherPair(k)
+    return this.takeInternal(n, nextPair, from, filterFn)
+  }
+
+  /**
+   * Returns the next n items **before** the provided item (in descending order) or the last n items if no from item is provided.
+   * @param n - The number of items to return
+   * @param from - The item to start from (exclusive). Starts from the largest item (inclusive) if not provided.
+   * @returns The next n items **before** the provided key. Returns the last n items if no from item is provided.
+   */
+  takeReversed(
+    n: number,
+    from?: any,
+    filterFn?: (key: TKey) => boolean
+  ): Array<TKey> {
+    const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k)
+    return this.takeInternal(n, nextPair, from, filterFn)
+  }
+
   /**
    * Performs an IN array lookup
    */
@@ -296,6 +340,13 @@ export class BTreeIndex<
       .map((key) => [key, this.valueMap.get(key) ?? new Set()])
   }
 
+  get orderedEntriesArrayReversed(): Array<[any, Set<TKey>]> {
+    return this.takeReversed(this.orderedEntries.size).map((key) => [
+      key,
+      this.valueMap.get(key) ?? new Set(),
+    ])
+  }
+
   get valueMapData(): Map<any, Set<TKey>> {
     return this.valueMap
   }

package/src/indexes/reverse-index.ts

@@ -0,0 +1,120 @@
+import type { CompareOptions } from "../query/builder/types"
+import type { OrderByDirection } from "../query/ir"
+import type { IndexInterface, IndexOperation, IndexStats } from "./base-index"
+import type { RangeQueryOptions } from "./btree-index"
+
+export class ReverseIndex<TKey extends string | number>
+  implements IndexInterface<TKey>
+{
+  private originalIndex: IndexInterface<TKey>
+
+  constructor(index: IndexInterface<TKey>) {
+    this.originalIndex = index
+  }
+
+  // Define the reversed operations
+
+  lookup(operation: IndexOperation, value: any): Set<TKey> {
+    const reverseOperation =
+      operation === `gt`
+        ? `lt`
+        : operation === `gte`
+          ? `lte`
+          : operation === `lt`
+            ? `gt`
+            : operation === `lte`
+              ? `gte`
+              : operation
+    return this.originalIndex.lookup(reverseOperation, value)
+  }
+
+  rangeQuery(options: RangeQueryOptions = {}): Set<TKey> {
+    return this.originalIndex.rangeQueryReversed(options)
+  }
+
+  rangeQueryReversed(options: RangeQueryOptions = {}): Set<TKey> {
+    return this.originalIndex.rangeQuery(options)
+  }
+
+  take(n: number, from?: any, filterFn?: (key: TKey) => boolean): Array<TKey> {
+    return this.originalIndex.takeReversed(n, from, filterFn)
+  }
+
+  takeReversed(
+    n: number,
+    from?: any,
+    filterFn?: (key: TKey) => boolean
+  ): Array<TKey> {
+    return this.originalIndex.take(n, from, filterFn)
+  }
+
+  get orderedEntriesArray(): Array<[any, Set<TKey>]> {
+    return this.originalIndex.orderedEntriesArrayReversed
+  }
+
+  get orderedEntriesArrayReversed(): Array<[any, Set<TKey>]> {
+    return this.originalIndex.orderedEntriesArray
+  }
+
+  // All operations below delegate to the original index
+
+  supports(operation: IndexOperation): boolean {
+    return this.originalIndex.supports(operation)
+  }
+
+  matchesField(fieldPath: Array<string>): boolean {
+    return this.originalIndex.matchesField(fieldPath)
+  }
+
+  matchesCompareOptions(compareOptions: CompareOptions): boolean {
+    return this.originalIndex.matchesCompareOptions(compareOptions)
+  }
+
+  matchesDirection(direction: OrderByDirection): boolean {
+    return this.originalIndex.matchesDirection(direction)
+  }
+
+  getStats(): IndexStats {
+    return this.originalIndex.getStats()
+  }
+
+  add(key: TKey, item: any): void {
+    this.originalIndex.add(key, item)
+  }
+
+  remove(key: TKey, item: any): void {
+    this.originalIndex.remove(key, item)
+  }
+
+  update(key: TKey, oldItem: any, newItem: any): void {
+    this.originalIndex.update(key, oldItem, newItem)
+  }
+
+  build(entries: Iterable<[TKey, any]>): void {
+    this.originalIndex.build(entries)
+  }
+
+  clear(): void {
+    this.originalIndex.clear()
+  }
+
+  get keyCount(): number {
+    return this.originalIndex.keyCount
+  }
+
+  equalityLookup(value: any): Set<TKey> {
+    return this.originalIndex.equalityLookup(value)
+  }
+
+  inArrayLookup(values: Array<any>): Set<TKey> {
+    return this.originalIndex.inArrayLookup(values)
+  }
+
+  get indexedKeysSet(): Set<TKey> {
+    return this.originalIndex.indexedKeysSet
+  }
+
+  get valueMapData(): Map<any, Set<TKey>> {
+    return this.originalIndex.valueMapData
+  }
+}

package/src/query/builder/index.ts

@@ -16,7 +16,7 @@ import {
   SubQueryMustHaveFromClauseError,
 } from "../../errors.js"
 import { createRefProxy, toExpression } from "./ref-proxy.js"
-import type { NamespacedRow } from "../../types.js"
+import type { NamespacedRow, SingleResult } from "../../types.js"
 import type {
   Aggregate,
   BasicExpression,
@@ -615,6 +615,28 @@ export class BaseQueryBuilder<TContext extends Context = Context> {
     }) as any
   }
 
+  /**
+   * Specify that the query should return a single result
+   * @returns A QueryBuilder that returns the first result
+   *
+   * @example
+   * ```ts
+   * // Get the user matching the query
+   * query
+   *   .from({ users: usersCollection })
+   *   .where(({users}) => eq(users.id, 1))
+   *   .findOne()
+   *```
+   */
+  findOne(): QueryBuilder<TContext & SingleResult> {
+    return new BaseQueryBuilder({
+      ...this.query,
+      // TODO: enforcing return only one result with also a default orderBy if none is specified
+      // limit: 1,
+      singleResult: true,
+    })
+  }
+
   // Helper methods
   private _getCurrentAliases(): Array<string> {
     const aliases: Array<string> = []
@@ -817,4 +839,10 @@ export type ExtractContext<T> =
       : never
 
 // Export the types from types.ts for convenience
-export type { Context, Source, GetResult, RefLeaf as Ref } from "./types.js"
+export type {
+  Context,
+  Source,
+  GetResult,
+  RefLeaf as Ref,
+  InferResultType,
+} from "./types.js"

package/src/query/builder/types.ts

@@ -1,4 +1,5 @@
 import type { CollectionImpl } from "../../collection/index.js"
+import type { SingleResult } from "../../types.js"
 import type {
   Aggregate,
   BasicExpression,
@@ -47,6 +48,8 @@ export interface Context {
   >
   // The result type after select (if select has been called)
   result?: any
+  // Single result only (if findOne has been called)
+  singleResult?: boolean
 }
 
 /**
@@ -571,6 +574,7 @@ export type MergeContextWithJoinType<
     [K in keyof TNewSchema & string]: TJoinType
   }
   result: TContext[`result`]
+  singleResult: TContext[`singleResult`] extends true ? true : false
 }
 
 /**
@@ -621,6 +625,14 @@ export type ApplyJoinOptionalityToMergedSchema<
       TNewSchema[K]
 }
 
+/**
+ * Utility type to infer the query result size (single row or an array)
+ */
+export type InferResultType<TContext extends Context> =
+  TContext extends SingleResult
+    ? GetResult<TContext> | undefined
+    : Array<GetResult<TContext>>
+
 /**
  * GetResult - Determines the final result type of a query
  *

package/src/query/compiler/group-by.ts

@@ -417,16 +417,7 @@ export function replaceAggregatesByRefs(
     }
 
     case `ref`: {
-      const refExpr = havingExpr
-      // Check if this is a direct reference to a SELECT alias
-      if (refExpr.path.length === 1) {
-        const alias = refExpr.path[0]!
-        if (selectClause[alias]) {
-          // This is a reference to a SELECT alias, convert to result.alias
-          return new PropRef([resultAlias, alias])
-        }
-      }
-      // Return as-is for other refs
+      // Non-aggregate refs are passed through unchanged (they reference table columns)
       return havingExpr as BasicExpression
     }
 

package/src/query/compiler/index.ts

@@ -127,7 +127,8 @@ export function compileQuery(
       callbacks,
       lazyCollections,
       optimizableOrderByCollections,
-      rawQuery
+      rawQuery,
+      compileQuery
     )
   }
 
@@ -512,3 +513,5 @@ export function followRef(
     }
   }
 }
+
+export type CompileQueryFn = typeof compileQuery