@tanstack/db 0.1.3 → 0.1.4

package/src/collection.ts

@@ -65,6 +65,7 @@ import type { BaseIndex, IndexResolver } from "./indexes/base-index.js"
 interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
   committed: boolean
   operations: Array<OptimisticChangeMessage<T>>
+  truncate?: boolean
 }
 
 /**
@@ -392,9 +393,8 @@ export class CollectionImpl<
         this.onFirstReadyCallbacks = []
         callbacks.forEach((callback) => callback())
 
-        // If the collection is empty when it becomes ready, emit an empty change event
         // to notify subscribers (like LiveQueryCollection) that the collection is ready
-        if (this.size === 0 && this.changeListeners.size > 0) {
+        if (this.changeListeners.size > 0) {
           this.emitEmptyReadyEvent()
         }
       }
@@ -559,11 +559,16 @@ export class CollectionImpl<
 
           // Check if an item with this key already exists when inserting
           if (messageWithoutKey.type === `insert`) {
+            const insertingIntoExistingSynced = this.syncedData.has(key)
+            const hasPendingDeleteForKey = pendingTransaction.operations.some(
+              (op) => op.key === key && op.type === `delete`
+            )
+            const isTruncateTransaction = pendingTransaction.truncate === true
+            // Allow insert after truncate in the same transaction even if it existed in syncedData
             if (
-              this.syncedData.has(key) &&
-              !pendingTransaction.operations.some(
-                (op) => op.key === key && op.type === `delete`
-              )
+              insertingIntoExistingSynced &&
+              !hasPendingDeleteForKey &&
+              !isTruncateTransaction
             ) {
               throw new DuplicateKeySyncError(key, this.id)
             }
@@ -600,6 +605,28 @@ export class CollectionImpl<
         markReady: () => {
           this.markReady()
         },
+        truncate: () => {
+          const pendingTransaction =
+            this.pendingSyncedTransactions[
+              this.pendingSyncedTransactions.length - 1
+            ]
+          if (!pendingTransaction) {
+            throw new NoPendingSyncTransactionWriteError()
+          }
+          if (pendingTransaction.committed) {
+            throw new SyncTransactionAlreadyCommittedWriteError()
+          }
+
+          // Clear all operations from the current transaction
+          pendingTransaction.operations = []
+
+          // Mark the transaction as a truncate operation. During commit, this triggers:
+          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
+          // - Clearing of syncedData/syncedMetadata
+          // - Subsequent synced ops applied on the fresh base
+          // - Finally, optimistic mutations re-applied on top (single batch)
+          pendingTransaction.truncate = true
+        },
       })
 
       // Store cleanup function if provided
@@ -957,6 +984,12 @@ export class CollectionImpl<
     for (const listener of this.changeListeners) {
       listener([])
     }
+    // Emit to key-specific listeners
+    for (const [_key, keyListeners] of this.changeKeyListeners) {
+      for (const listener of keyListeners) {
+        listener([])
+      }
+    }
   }
 
   /**
@@ -1149,7 +1182,11 @@ export class CollectionImpl<
       }
     }
 
-    if (!hasPersistingTransaction) {
+    const hasTruncateSync = this.pendingSyncedTransactions.some(
+      (t) => t.truncate === true
+    )
+
+    if (!hasPersistingTransaction || hasTruncateSync) {
       // Set flag to prevent redundant optimistic state recalculations
       this.isCommittingSyncTransactions = true
 
@@ -1179,6 +1216,28 @@ export class CollectionImpl<
       const rowUpdateMode = this.config.sync.rowUpdateMode || `partial`
 
       for (const transaction of this.pendingSyncedTransactions) {
+        // Handle truncate operations first
+        if (transaction.truncate) {
+          // TRUNCATE PHASE
+          // 1) Emit a delete for every currently-synced key so downstream listeners/indexes
+          //    observe a clear-before-rebuild. We intentionally skip keys already in
+          //    optimisticDeletes because their delete was previously emitted by the user.
+          for (const key of this.syncedData.keys()) {
+            if (this.optimisticDeletes.has(key)) continue
+            const previousValue =
+              this.optimisticUpserts.get(key) || this.syncedData.get(key)
+            if (previousValue !== undefined) {
+              events.push({ type: `delete`, key, value: previousValue })
+            }
+          }
+
+          // 2) Clear the authoritative synced base. Subsequent server ops in this
+          //    same commit will rebuild the base atomically.
+          this.syncedData.clear()
+          this.syncedMetadata.clear()
+          this.syncedKeys.clear()
+        }
+
         for (const operation of transaction.operations) {
           const key = operation.key as TKey
           this.syncedKeys.add(key)
@@ -1228,7 +1287,101 @@ export class CollectionImpl<
         }
       }
 
-      // Clear optimistic state since sync operations will now provide the authoritative data
+      // After applying synced operations, if this commit included a truncate,
+      // re-apply optimistic mutations on top of the fresh synced base. This ensures
+      // the UI preserves local intent while respecting server rebuild semantics.
+      // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts.
+      const hadTruncate = this.pendingSyncedTransactions.some(
+        (t) => t.truncate === true
+      )
+      if (hadTruncate) {
+        // Avoid duplicating keys that were inserted/updated by synced operations in this commit
+        const syncedInsertedOrUpdatedKeys = new Set<TKey>()
+        for (const t of this.pendingSyncedTransactions) {
+          for (const op of t.operations) {
+            if (op.type === `insert` || op.type === `update`) {
+              syncedInsertedOrUpdatedKeys.add(op.key as TKey)
+            }
+          }
+        }
+
+        // Build re-apply sets from ACTIVE optimistic transactions against the new synced base
+        // We do not copy maps; we compute intent directly from transactions to avoid drift.
+        const reapplyUpserts = new Map<TKey, T>()
+        const reapplyDeletes = new Set<TKey>()
+
+        for (const tx of this.transactions.values()) {
+          if ([`completed`, `failed`].includes(tx.state)) continue
+          for (const mutation of tx.mutations) {
+            if (mutation.collection !== this || !mutation.optimistic) continue
+            const key = mutation.key as TKey
+            switch (mutation.type) {
+              case `insert`:
+                reapplyUpserts.set(key, mutation.modified as T)
+                reapplyDeletes.delete(key)
+                break
+              case `update`: {
+                const base = this.syncedData.get(key)
+                const next = base
+                  ? (Object.assign({}, base, mutation.changes) as T)
+                  : (mutation.modified as T)
+                reapplyUpserts.set(key, next)
+                reapplyDeletes.delete(key)
+                break
+              }
+              case `delete`:
+                reapplyUpserts.delete(key)
+                reapplyDeletes.add(key)
+                break
+            }
+          }
+        }
+
+        // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete.
+        // If the server also inserted/updated the same key in this batch, override that value
+        // with the optimistic value to preserve local intent.
+        for (const [key, value] of reapplyUpserts) {
+          if (reapplyDeletes.has(key)) continue
+          if (syncedInsertedOrUpdatedKeys.has(key)) {
+            let foundInsert = false
+            for (let i = events.length - 1; i >= 0; i--) {
+              const evt = events[i]!
+              if (evt.key === key && evt.type === `insert`) {
+                evt.value = value
+                foundInsert = true
+                break
+              }
+            }
+            if (!foundInsert) {
+              events.push({ type: `insert`, key, value })
+            }
+          } else {
+            events.push({ type: `insert`, key, value })
+          }
+        }
+
+        // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete.
+        if (events.length > 0 && reapplyDeletes.size > 0) {
+          const filtered: Array<ChangeMessage<T, TKey>> = []
+          for (const evt of events) {
+            if (evt.type === `insert` && reapplyDeletes.has(evt.key)) {
+              continue
+            }
+            filtered.push(evt)
+          }
+          events.length = 0
+          events.push(...filtered)
+        }
+
+        // Ensure listeners are active before emitting this critical batch
+        if (!this.isReady()) {
+          this.setStatus(`ready`)
+        }
+      }
+
+      // Maintain optimistic state appropriately
+      // Clear optimistic state since sync operations will now provide the authoritative data.
+      // Any still-active user transactions will be re-applied below in recompute.
       this.optimisticUpserts.clear()
       this.optimisticDeletes.clear()
 
@@ -1397,8 +1550,8 @@ export class CollectionImpl<
 
   /**
    * Creates an index on a collection for faster queries.
-   * Indexes significantly improve query performance by allowing binary search
-   * and range queries instead of full scans.
+   * Indexes significantly improve query performance by allowing constant time lookups
+   * and logarithmic time range queries instead of full scans.
    *
    * @template TResolver - The type of the index resolver (constructor or async loader)
    * @param indexCallback - Function that extracts the indexed value from each item

package/src/errors.ts

@@ -377,6 +377,12 @@ export class UnknownFunctionError extends QueryCompilationError {
   }
 }
 
+export class JoinCollectionNotFoundError extends QueryCompilationError {
+  constructor(collectionId: string) {
+    super(`Collection "${collectionId}" not found during compilation of join`)
+  }
+}
+
 // JOIN Operation Errors
 export class JoinError extends TanStackDBError {
   constructor(message: string) {

package/src/indexes/auto-index.ts

@@ -6,6 +6,57 @@ export interface AutoIndexConfig {
   autoIndex?: `off` | `eager`
 }
 
+function shouldAutoIndex(collection: CollectionImpl<any, any, any, any, any>) {
+  // Only proceed if auto-indexing is enabled
+  if (collection.config.autoIndex !== `eager`) {
+    return false
+  }
+
+  // Don't auto-index during sync operations
+  if (
+    collection.status === `loading` ||
+    collection.status === `initialCommit`
+  ) {
+    return false
+  }
+
+  return true
+}
+
+export function ensureIndexForField<
+  T extends Record<string, any>,
+  TKey extends string | number,
+>(
+  fieldName: string,
+  fieldPath: Array<string>,
+  collection: CollectionImpl<T, TKey, any, any, any>,
+  compareFn?: (a: any, b: any) => number
+) {
+  if (!shouldAutoIndex(collection)) {
+    return
+  }
+
+  // Check if we already have an index for this field
+  const existingIndex = Array.from(collection.indexes.values()).find((index) =>
+    index.matchesField(fieldPath)
+  )
+
+  if (existingIndex) {
+    return // Index already exists
+  }
+
+  // Create a new index for this field using the collection's createIndex method
+  try {
+    collection.createIndex((row) => (row as any)[fieldName], {
+      name: `auto_${fieldName}`,
+      indexType: BTreeIndex,
+      options: compareFn ? { compareFn } : {},
+    })
+  } catch (error) {
+    console.warn(`Failed to create auto-index for field "${fieldName}":`, error)
+  }
+}
+
 /**
  * Analyzes a where expression and creates indexes for all simple operations on single fields
  */
@@ -16,16 +67,7 @@ export function ensureIndexForExpression<
   expression: BasicExpression,
   collection: CollectionImpl<T, TKey, any, any, any>
 ): void {
-  // Only proceed if auto-indexing is enabled
-  if (collection.config.autoIndex !== `eager`) {
-    return
-  }
-
-  // Don't auto-index during sync operations
-  if (
-    collection.status === `loading` ||
-    collection.status === `initialCommit`
-  ) {
+  if (!shouldAutoIndex(collection)) {
     return
   }
 
@@ -33,27 +75,7 @@ export function ensureIndexForExpression<
   const indexableExpressions = extractIndexableExpressions(expression)
 
   for (const { fieldName, fieldPath } of indexableExpressions) {
-    // Check if we already have an index for this field
-    const existingIndex = Array.from(collection.indexes.values()).find(
-      (index) => index.matchesField(fieldPath)
-    )
-
-    if (existingIndex) {
-      continue // Index already exists
-    }
-
-    // Create a new index for this field using the collection's createIndex method
-    try {
-      collection.createIndex((row) => (row as any)[fieldName], {
-        name: `auto_${fieldName}`,
-        indexType: BTreeIndex,
-      })
-    } catch (error) {
-      console.warn(
-        `Failed to create auto-index for field "${fieldName}":`,
-        error
-      )
-    }
+    ensureIndexForField(fieldName, fieldPath, collection)
   }
 }
 

package/src/indexes/base-index.ts

@@ -1,6 +1,6 @@
 import { compileSingleRowExpression } from "../query/compiler/evaluators.js"
 import { comparisonFunctions } from "../query/builder/functions.js"
-import type { BasicExpression } from "../query/ir.js"
+import type { BasicExpression, OrderByDirection } from "../query/ir.js"
 
 /**
  * Operations that indexes can support, imported from available comparison functions
@@ -56,6 +56,11 @@ export abstract class BaseIndex<
   abstract build(entries: Iterable<[TKey, any]>): void
   abstract clear(): void
   abstract lookup(operation: IndexOperation, value: any): Set<TKey>
+  abstract take(
+    n: number,
+    direction?: OrderByDirection,
+    from?: TKey
+  ): Array<TKey>
   abstract get keyCount(): number
 
   // Common methods

package/src/indexes/btree-index.ts

@@ -230,6 +230,35 @@ export class BTreeIndex<
     return result
   }
 
+  /**
+   * Returns the next n items after the provided item or the first n items if no from item is provided.
+   * @param n - The number of items to return
+   * @param from - The item to start from (exclusive). Starts from the smallest item (inclusive) if not provided.
+   * @returns The next n items after the provided key. Returns the first n items if no from item is provided.
+   */
+  take(n: number, from?: any): Array<TKey> {
+    const keysInResult: Set<TKey> = new Set()
+    const result: Array<TKey> = []
+    const nextKey = (k?: any) => this.orderedEntries.nextHigherKey(k)
+    let key = from
+
+    while ((key = nextKey(key)) && result.length < n) {
+      const keys = this.valueMap.get(key)
+      if (keys) {
+        const it = keys.values()
+        let ks: TKey | undefined
+        while (result.length < n && (ks = it.next().value)) {
+          if (!keysInResult.has(ks)) {
+            result.push(ks)
+            keysInResult.add(ks)
+          }
+        }
+      }
+    }
+
+    return result
+  }
+
   /**
    * Performs an IN array lookup
    */

package/src/indexes/index-options.ts

@@ -8,7 +8,7 @@ export interface IndexOptions<TResolver extends IndexResolver = IndexResolver> {
   indexType?: TResolver
   options?: TResolver extends IndexConstructor<any>
     ? TResolver extends new (
-        id: string,
+        id: number,
         expr: any,
         name?: string,
         options?: infer O
@@ -17,7 +17,7 @@ export interface IndexOptions<TResolver extends IndexResolver = IndexResolver> {
       : never
     : TResolver extends () => Promise<infer TCtor>
       ? TCtor extends new (
-          id: string,
+          id: number,
           expr: any,
           name?: string,
           options?: infer O

package/src/query/compiler/evaluators.ts

@@ -20,9 +20,12 @@ export type CompiledSingleRowExpression = (item: Record<string, unknown>) => any
  * Compiles an expression into an optimized evaluator function.
  * This eliminates branching during evaluation by pre-compiling the expression structure.
  */
-export function compileExpression(expr: BasicExpression): CompiledExpression {
-  const compiledFn = compileExpressionInternal(expr, false)
-  return compiledFn as CompiledExpression
+export function compileExpression(
+  expr: BasicExpression,
+  isSingleRow: boolean = false
+): CompiledExpression | CompiledSingleRowExpression {
+  const compiledFn = compileExpressionInternal(expr, isSingleRow)
+  return compiledFn
 }
 
 /**

package/src/query/compiler/group-by.ts

@@ -166,7 +166,9 @@ export function processGroupBy(
   const mapping = validateAndCreateMapping(groupByClause, selectClause)
 
   // Pre-compile groupBy expressions
-  const compiledGroupByExpressions = groupByClause.map(compileExpression)
+  const compiledGroupByExpressions = groupByClause.map((e) =>
+    compileExpression(e)
+  )
 
   // Create a key extractor function using simple __key_X format
   const keyExtractor = ([, row]: [

package/src/query/compiler/index.ts

@@ -7,17 +7,21 @@ import {
   LimitOffsetRequireOrderByError,
   UnsupportedFromTypeError,
 } from "../../errors.js"
+import { PropRef } from "../ir.js"
 import { compileExpression } from "./evaluators.js"
 import { processJoins } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
 import { processOrderBy } from "./order-by.js"
 import { processSelectToResults } from "./select.js"
+import type { OrderByOptimizationInfo } from "./order-by.js"
 import type {
   BasicExpression,
   CollectionRef,
   QueryIR,
   QueryRef,
 } from "../ir.js"
+import type { LazyCollectionCallbacks } from "./joins.js"
+import type { Collection } from "../../collection.js"
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,
@@ -29,6 +33,8 @@ import type { QueryCache, QueryMapping } from "./types.js"
  * Result of query compilation including both the pipeline and collection-specific WHERE clauses
  */
 export interface CompilationResult {
+  /** The ID of the main collection */
+  collectionId: string
   /** The compiled query pipeline */
   pipeline: ResultStream
   /** Map of collection aliases to their WHERE clauses for index optimization */
@@ -46,6 +52,10 @@ export interface CompilationResult {
 export function compileQuery(
   rawQuery: QueryIR,
   inputs: Record<string, KeyedStream>,
+  collections: Record<string, Collection<any, any, any, any, any>>,
+  callbacks: Record<string, LazyCollectionCallbacks>,
+  lazyCollections: Set<string>,
+  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,
   cache: QueryCache = new WeakMap(),
   queryMapping: QueryMapping = new WeakMap()
 ): CompilationResult {
@@ -70,9 +80,17 @@ export function compileQuery(
   const tables: Record<string, KeyedStream> = {}
 
   // Process the FROM clause to get the main table
-  const { alias: mainTableAlias, input: mainInput } = processFrom(
+  const {
+    alias: mainTableAlias,
+    input: mainInput,
+    collectionId: mainCollectionId,
+  } = processFrom(
     query.from,
     allInputs,
+    collections,
+    callbacks,
+    lazyCollections,
+    optimizableOrderByCollections,
     cache,
     queryMapping
   )
@@ -96,10 +114,16 @@ export function compileQuery(
       pipeline,
       query.join,
       tables,
+      mainCollectionId,
       mainTableAlias,
       allInputs,
       cache,
-      queryMapping
+      queryMapping,
+      collections,
+      callbacks,
+      lazyCollections,
+      optimizableOrderByCollections,
+      rawQuery
     )
   }
 
@@ -231,8 +255,11 @@ export function compileQuery(
   // Process orderBy parameter if it exists
   if (query.orderBy && query.orderBy.length > 0) {
     const orderedPipeline = processOrderBy(
+      rawQuery,
       pipeline,
       query.orderBy,
+      collections[mainCollectionId]!,
+      optimizableOrderByCollections,
       query.limit,
       query.offset
     )
@@ -249,6 +276,7 @@ export function compileQuery(
     const result = resultPipeline
     // Cache the result before returning (use original query as key)
     const compilationResult = {
+      collectionId: mainCollectionId,
       pipeline: result,
       collectionWhereClauses,
     }
@@ -275,6 +303,7 @@ export function compileQuery(
   const result = resultPipeline
   // Cache the result before returning (use original query as key)
   const compilationResult = {
+    collectionId: mainCollectionId,
     pipeline: result,
     collectionWhereClauses,
   }
@@ -289,16 +318,20 @@ export function compileQuery(
 function processFrom(
   from: CollectionRef | QueryRef,
   allInputs: Record<string, KeyedStream>,
+  collections: Record<string, Collection>,
+  callbacks: Record<string, LazyCollectionCallbacks>,
+  lazyCollections: Set<string>,
+  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,
   cache: QueryCache,
   queryMapping: QueryMapping
-): { alias: string; input: KeyedStream } {
+): { alias: string; input: KeyedStream; collectionId: string } {
   switch (from.type) {
     case `collectionRef`: {
       const input = allInputs[from.collection.id]
       if (!input) {
         throw new CollectionInputNotFoundError(from.collection.id)
       }
-      return { alias: from.alias, input }
+      return { alias: from.alias, input, collectionId: from.collection.id }
     }
     case `queryRef`: {
       // Find the original query for caching purposes
@@ -308,6 +341,10 @@ function processFrom(
       const subQueryResult = compileQuery(
         originalQuery,
         allInputs,
+        collections,
+        callbacks,
+        lazyCollections,
+        optimizableOrderByCollections,
         cache,
         queryMapping
       )
@@ -324,7 +361,11 @@ function processFrom(
         })
       )
 
-      return { alias: from.alias, input: extractedInput }
+      return {
+        alias: from.alias,
+        input: extractedInput,
+        collectionId: subQueryResult.collectionId,
+      }
     }
     default:
       throw new UnsupportedFromTypeError((from as any).type)
@@ -380,3 +421,69 @@ function mapNestedQueries(
     }
   }
 }
+
+function getRefFromAlias(
+  query: QueryIR,
+  alias: string
+): CollectionRef | QueryRef | void {
+  if (query.from.alias === alias) {
+    return query.from
+  }
+
+  for (const join of query.join || []) {
+    if (join.from.alias === alias) {
+      return join.from
+    }
+  }
+}
+
+/**
+ * Follows the given reference in a query
+ * until its finds the root field the reference points to.
+ * @returns The collection, its alias, and the path to the root field in this collection
+ */
+export function followRef(
+  query: QueryIR,
+  ref: PropRef<any>,
+  collection: Collection
+): { collection: Collection; path: Array<string> } | void {
+  if (ref.path.length === 0) {
+    return
+  }
+
+  if (ref.path.length === 1) {
+    // This field should be part of this collection
+    const field = ref.path[0]!
+    // is it part of the select clause?
+    if (query.select) {
+      const selectedField = query.select[field]
+      if (selectedField && selectedField.type === `ref`) {
+        return followRef(query, selectedField, collection)
+      }
+    }
+
+    // Either this field is not part of the select clause
+    // and thus it must be part of the collection itself
+    // or it is part of the select but is not a reference
+    // so we can stop here and don't have to follow it
+    return { collection, path: [field] }
+  }
+
+  if (ref.path.length > 1) {
+    // This is a nested field
+    const [alias, ...rest] = ref.path
+    const aliasRef = getRefFromAlias(query, alias!)
+    if (!aliasRef) {
+      return
+    }
+
+    if (aliasRef.type === `queryRef`) {
+      return followRef(aliasRef.query, new PropRef(rest), collection)
+    } else {
+      // This is a reference to a collection
+      // we can't follow it further
+      // so the field must be on the collection itself
+      return { collection: aliasRef.collection, path: rest }
+    }
+  }
+}