@tanstack/db 0.1.1 → 0.1.4

package/src/collection.ts

@@ -65,6 +65,7 @@ import type { BaseIndex, IndexResolver } from "./indexes/base-index.js"
 interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
   committed: boolean
   operations: Array<OptimisticChangeMessage<T>>
+  truncate?: boolean
 }
 
 /**
@@ -155,8 +156,81 @@ export interface Collection<
  *   sync: { sync: () => {} }
  * })
  *
- * // Note: You must provide either an explicit type or a schema, but not both.
+ * // Note: You can provide an explicit type, a schema, or both. When both are provided, the explicit type takes precedence.
  */
+
+// Overload for when schema is provided - infers schema type
+export function createCollection<
+  TSchema extends StandardSchemaV1,
+  TKey extends string | number = string | number,
+  TUtils extends UtilsRecord = {},
+  TFallback extends object = Record<string, unknown>,
+>(
+  options: CollectionConfig<
+    ResolveType<unknown, TSchema, TFallback>,
+    TKey,
+    TSchema,
+    ResolveInsertInput<unknown, TSchema, TFallback>
+  > & {
+    schema: TSchema
+    utils?: TUtils
+  }
+): Collection<
+  ResolveType<unknown, TSchema, TFallback>,
+  TKey,
+  TUtils,
+  TSchema,
+  ResolveInsertInput<unknown, TSchema, TFallback>
+>
+
+// Overload for when explicit type is provided with schema - explicit type takes precedence
+export function createCollection<
+  TExplicit extends object,
+  TKey extends string | number = string | number,
+  TUtils extends UtilsRecord = {},
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TFallback extends object = Record<string, unknown>,
+>(
+  options: CollectionConfig<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TSchema,
+    ResolveInsertInput<TExplicit, TSchema, TFallback>
+  > & {
+    schema: TSchema
+    utils?: TUtils
+  }
+): Collection<
+  ResolveType<TExplicit, TSchema, TFallback>,
+  TKey,
+  TUtils,
+  TSchema,
+  ResolveInsertInput<TExplicit, TSchema, TFallback>
+>
+
+// Overload for when explicit type is provided or no schema
+export function createCollection<
+  TExplicit = unknown,
+  TKey extends string | number = string | number,
+  TUtils extends UtilsRecord = {},
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TFallback extends object = Record<string, unknown>,
+>(
+  options: CollectionConfig<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TSchema,
+    ResolveInsertInput<TExplicit, TSchema, TFallback>
+  > & { utils?: TUtils }
+): Collection<
+  ResolveType<TExplicit, TSchema, TFallback>,
+  TKey,
+  TUtils,
+  TSchema,
+  ResolveInsertInput<TExplicit, TSchema, TFallback>
+>
+
+// Implementation
 export function createCollection<
   TExplicit = unknown,
   TKey extends string | number = string | number,
@@ -319,9 +393,8 @@ export class CollectionImpl<
         this.onFirstReadyCallbacks = []
         callbacks.forEach((callback) => callback())
 
-        // If the collection is empty when it becomes ready, emit an empty change event
         // to notify subscribers (like LiveQueryCollection) that the collection is ready
-        if (this.size === 0 && this.changeListeners.size > 0) {
+        if (this.changeListeners.size > 0) {
           this.emitEmptyReadyEvent()
         }
       }
@@ -486,11 +559,16 @@ export class CollectionImpl<
 
           // Check if an item with this key already exists when inserting
           if (messageWithoutKey.type === `insert`) {
+            const insertingIntoExistingSynced = this.syncedData.has(key)
+            const hasPendingDeleteForKey = pendingTransaction.operations.some(
+              (op) => op.key === key && op.type === `delete`
+            )
+            const isTruncateTransaction = pendingTransaction.truncate === true
+            // Allow insert after truncate in the same transaction even if it existed in syncedData
             if (
-              this.syncedData.has(key) &&
-              !pendingTransaction.operations.some(
-                (op) => op.key === key && op.type === `delete`
-              )
+              insertingIntoExistingSynced &&
+              !hasPendingDeleteForKey &&
+              !isTruncateTransaction
             ) {
               throw new DuplicateKeySyncError(key, this.id)
             }
@@ -527,6 +605,28 @@ export class CollectionImpl<
         markReady: () => {
           this.markReady()
         },
+        truncate: () => {
+          const pendingTransaction =
+            this.pendingSyncedTransactions[
+              this.pendingSyncedTransactions.length - 1
+            ]
+          if (!pendingTransaction) {
+            throw new NoPendingSyncTransactionWriteError()
+          }
+          if (pendingTransaction.committed) {
+            throw new SyncTransactionAlreadyCommittedWriteError()
+          }
+
+          // Clear all operations from the current transaction
+          pendingTransaction.operations = []
+
+          // Mark the transaction as a truncate operation. During commit, this triggers:
+          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
+          // - Clearing of syncedData/syncedMetadata
+          // - Subsequent synced ops applied on the fresh base
+          // - Finally, optimistic mutations re-applied on top (single batch)
+          pendingTransaction.truncate = true
+        },
       })
 
       // Store cleanup function if provided
@@ -884,6 +984,12 @@ export class CollectionImpl<
     for (const listener of this.changeListeners) {
       listener([])
     }
+    // Emit to key-specific listeners
+    for (const [_key, keyListeners] of this.changeKeyListeners) {
+      for (const listener of keyListeners) {
+        listener([])
+      }
+    }
   }
 
   /**
@@ -1076,7 +1182,11 @@ export class CollectionImpl<
       }
     }
 
-    if (!hasPersistingTransaction) {
+    const hasTruncateSync = this.pendingSyncedTransactions.some(
+      (t) => t.truncate === true
+    )
+
+    if (!hasPersistingTransaction || hasTruncateSync) {
       // Set flag to prevent redundant optimistic state recalculations
       this.isCommittingSyncTransactions = true
 
@@ -1106,6 +1216,28 @@ export class CollectionImpl<
       const rowUpdateMode = this.config.sync.rowUpdateMode || `partial`
 
       for (const transaction of this.pendingSyncedTransactions) {
+        // Handle truncate operations first
+        if (transaction.truncate) {
+          // TRUNCATE PHASE
+          // 1) Emit a delete for every currently-synced key so downstream listeners/indexes
+          //    observe a clear-before-rebuild. We intentionally skip keys already in
+          //    optimisticDeletes because their delete was previously emitted by the user.
+          for (const key of this.syncedData.keys()) {
+            if (this.optimisticDeletes.has(key)) continue
+            const previousValue =
+              this.optimisticUpserts.get(key) || this.syncedData.get(key)
+            if (previousValue !== undefined) {
+              events.push({ type: `delete`, key, value: previousValue })
+            }
+          }
+
+          // 2) Clear the authoritative synced base. Subsequent server ops in this
+          //    same commit will rebuild the base atomically.
+          this.syncedData.clear()
+          this.syncedMetadata.clear()
+          this.syncedKeys.clear()
+        }
+
         for (const operation of transaction.operations) {
           const key = operation.key as TKey
           this.syncedKeys.add(key)
@@ -1155,7 +1287,101 @@ export class CollectionImpl<
         }
       }
 
-      // Clear optimistic state since sync operations will now provide the authoritative data
+      // After applying synced operations, if this commit included a truncate,
+      // re-apply optimistic mutations on top of the fresh synced base. This ensures
+      // the UI preserves local intent while respecting server rebuild semantics.
+      // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts.
+      const hadTruncate = this.pendingSyncedTransactions.some(
+        (t) => t.truncate === true
+      )
+      if (hadTruncate) {
+        // Avoid duplicating keys that were inserted/updated by synced operations in this commit
+        const syncedInsertedOrUpdatedKeys = new Set<TKey>()
+        for (const t of this.pendingSyncedTransactions) {
+          for (const op of t.operations) {
+            if (op.type === `insert` || op.type === `update`) {
+              syncedInsertedOrUpdatedKeys.add(op.key as TKey)
+            }
+          }
+        }
+
+        // Build re-apply sets from ACTIVE optimistic transactions against the new synced base
+        // We do not copy maps; we compute intent directly from transactions to avoid drift.
+        const reapplyUpserts = new Map<TKey, T>()
+        const reapplyDeletes = new Set<TKey>()
+
+        for (const tx of this.transactions.values()) {
+          if ([`completed`, `failed`].includes(tx.state)) continue
+          for (const mutation of tx.mutations) {
+            if (mutation.collection !== this || !mutation.optimistic) continue
+            const key = mutation.key as TKey
+            switch (mutation.type) {
+              case `insert`:
+                reapplyUpserts.set(key, mutation.modified as T)
+                reapplyDeletes.delete(key)
+                break
+              case `update`: {
+                const base = this.syncedData.get(key)
+                const next = base
+                  ? (Object.assign({}, base, mutation.changes) as T)
+                  : (mutation.modified as T)
+                reapplyUpserts.set(key, next)
+                reapplyDeletes.delete(key)
+                break
+              }
+              case `delete`:
+                reapplyUpserts.delete(key)
+                reapplyDeletes.add(key)
+                break
+            }
+          }
+        }
+
+        // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete.
+        // If the server also inserted/updated the same key in this batch, override that value
+        // with the optimistic value to preserve local intent.
+        for (const [key, value] of reapplyUpserts) {
+          if (reapplyDeletes.has(key)) continue
+          if (syncedInsertedOrUpdatedKeys.has(key)) {
+            let foundInsert = false
+            for (let i = events.length - 1; i >= 0; i--) {
+              const evt = events[i]!
+              if (evt.key === key && evt.type === `insert`) {
+                evt.value = value
+                foundInsert = true
+                break
+              }
+            }
+            if (!foundInsert) {
+              events.push({ type: `insert`, key, value })
+            }
+          } else {
+            events.push({ type: `insert`, key, value })
+          }
+        }
+
+        // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete.
+        if (events.length > 0 && reapplyDeletes.size > 0) {
+          const filtered: Array<ChangeMessage<T, TKey>> = []
+          for (const evt of events) {
+            if (evt.type === `insert` && reapplyDeletes.has(evt.key)) {
+              continue
+            }
+            filtered.push(evt)
+          }
+          events.length = 0
+          events.push(...filtered)
+        }
+
+        // Ensure listeners are active before emitting this critical batch
+        if (!this.isReady()) {
+          this.setStatus(`ready`)
+        }
+      }
+
+      // Maintain optimistic state appropriately
+      // Clear optimistic state since sync operations will now provide the authoritative data.
+      // Any still-active user transactions will be re-applied below in recompute.
       this.optimisticUpserts.clear()
       this.optimisticDeletes.clear()
 
@@ -1324,8 +1550,8 @@ export class CollectionImpl<
 
   /**
    * Creates an index on a collection for faster queries.
-   * Indexes significantly improve query performance by allowing binary search
-   * and range queries instead of full scans.
+   * Indexes significantly improve query performance by allowing constant time lookups
+   * and logarithmic time range queries instead of full scans.
    *
    * @template TResolver - The type of the index resolver (constructor or async loader)
    * @param indexCallback - Function that extracts the indexed value from each item

package/src/errors.ts

@@ -377,6 +377,12 @@ export class UnknownFunctionError extends QueryCompilationError {
   }
 }
 
+export class JoinCollectionNotFoundError extends QueryCompilationError {
+  constructor(collectionId: string) {
+    super(`Collection "${collectionId}" not found during compilation of join`)
+  }
+}
+
 // JOIN Operation Errors
 export class JoinError extends TanStackDBError {
   constructor(message: string) {

package/src/indexes/auto-index.ts

@@ -6,6 +6,57 @@ export interface AutoIndexConfig {
   autoIndex?: `off` | `eager`
 }
 
+function shouldAutoIndex(collection: CollectionImpl<any, any, any, any, any>) {
+  // Only proceed if auto-indexing is enabled
+  if (collection.config.autoIndex !== `eager`) {
+    return false
+  }
+
+  // Don't auto-index during sync operations
+  if (
+    collection.status === `loading` ||
+    collection.status === `initialCommit`
+  ) {
+    return false
+  }
+
+  return true
+}
+
+export function ensureIndexForField<
+  T extends Record<string, any>,
+  TKey extends string | number,
+>(
+  fieldName: string,
+  fieldPath: Array<string>,
+  collection: CollectionImpl<T, TKey, any, any, any>,
+  compareFn?: (a: any, b: any) => number
+) {
+  if (!shouldAutoIndex(collection)) {
+    return
+  }
+
+  // Check if we already have an index for this field
+  const existingIndex = Array.from(collection.indexes.values()).find((index) =>
+    index.matchesField(fieldPath)
+  )
+
+  if (existingIndex) {
+    return // Index already exists
+  }
+
+  // Create a new index for this field using the collection's createIndex method
+  try {
+    collection.createIndex((row) => (row as any)[fieldName], {
+      name: `auto_${fieldName}`,
+      indexType: BTreeIndex,
+      options: compareFn ? { compareFn } : {},
+    })
+  } catch (error) {
+    console.warn(`Failed to create auto-index for field "${fieldName}":`, error)
+  }
+}
+
 /**
  * Analyzes a where expression and creates indexes for all simple operations on single fields
  */
@@ -16,16 +67,7 @@ export function ensureIndexForExpression<
   expression: BasicExpression,
   collection: CollectionImpl<T, TKey, any, any, any>
 ): void {
-  // Only proceed if auto-indexing is enabled
-  if (collection.config.autoIndex !== `eager`) {
-    return
-  }
-
-  // Don't auto-index during sync operations
-  if (
-    collection.status === `loading` ||
-    collection.status === `initialCommit`
-  ) {
+  if (!shouldAutoIndex(collection)) {
     return
   }
 
@@ -33,27 +75,7 @@ export function ensureIndexForExpression<
   const indexableExpressions = extractIndexableExpressions(expression)
 
   for (const { fieldName, fieldPath } of indexableExpressions) {
-    // Check if we already have an index for this field
-    const existingIndex = Array.from(collection.indexes.values()).find(
-      (index) => index.matchesField(fieldPath)
-    )
-
-    if (existingIndex) {
-      continue // Index already exists
-    }
-
-    // Create a new index for this field using the collection's createIndex method
-    try {
-      collection.createIndex((row) => (row as any)[fieldName], {
-        name: `auto_${fieldName}`,
-        indexType: BTreeIndex,
-      })
-    } catch (error) {
-      console.warn(
-        `Failed to create auto-index for field "${fieldName}":`,
-        error
-      )
-    }
+    ensureIndexForField(fieldName, fieldPath, collection)
   }
 }
 

package/src/indexes/base-index.ts

@@ -1,6 +1,6 @@
 import { compileSingleRowExpression } from "../query/compiler/evaluators.js"
 import { comparisonFunctions } from "../query/builder/functions.js"
-import type { BasicExpression } from "../query/ir.js"
+import type { BasicExpression, OrderByDirection } from "../query/ir.js"
 
 /**
  * Operations that indexes can support, imported from available comparison functions
@@ -56,6 +56,11 @@ export abstract class BaseIndex<
   abstract build(entries: Iterable<[TKey, any]>): void
   abstract clear(): void
   abstract lookup(operation: IndexOperation, value: any): Set<TKey>
+  abstract take(
+    n: number,
+    direction?: OrderByDirection,
+    from?: TKey
+  ): Array<TKey>
   abstract get keyCount(): number
 
   // Common methods

package/src/indexes/btree-index.ts

@@ -1,5 +1,5 @@
-import { ascComparator } from "../utils/comparison.js"
 import { BTree } from "../utils/btree.js"
+import { defaultComparator } from "../utils/comparison.js"
 import { BaseIndex } from "./base-index.js"
 import type { BasicExpression } from "../query/ir.js"
 import type { IndexOperation } from "./base-index.js"
@@ -43,7 +43,7 @@ export class BTreeIndex<
   private orderedEntries: BTree<any, undefined> // we don't associate values with the keys of the B+ tree (the keys are indexed values)
   private valueMap = new Map<any, Set<TKey>>() // instead we store a mapping of indexed values to a set of PKs
   private indexedKeys = new Set<TKey>()
-  private compareFn: (a: any, b: any) => number = ascComparator
+  private compareFn: (a: any, b: any) => number = defaultComparator
 
   constructor(
     id: number,
@@ -52,7 +52,7 @@ export class BTreeIndex<
     options?: any
   ) {
     super(id, expression, name, options)
-    this.compareFn = options?.compareFn ?? ascComparator
+    this.compareFn = options?.compareFn ?? defaultComparator
     this.orderedEntries = new BTree(this.compareFn)
   }
 
@@ -230,6 +230,35 @@ export class BTreeIndex<
     return result
   }
 
+  /**
+   * Returns the next n items after the provided item or the first n items if no from item is provided.
+   * @param n - The number of items to return
+   * @param from - The item to start from (exclusive). Starts from the smallest item (inclusive) if not provided.
+   * @returns The next n items after the provided key. Returns the first n items if no from item is provided.
+   */
+  take(n: number, from?: any): Array<TKey> {
+    const keysInResult: Set<TKey> = new Set()
+    const result: Array<TKey> = []
+    const nextKey = (k?: any) => this.orderedEntries.nextHigherKey(k)
+    let key = from
+
+    while ((key = nextKey(key)) && result.length < n) {
+      const keys = this.valueMap.get(key)
+      if (keys) {
+        const it = keys.values()
+        let ks: TKey | undefined
+        while (result.length < n && (ks = it.next().value)) {
+          if (!keysInResult.has(ks)) {
+            result.push(ks)
+            keysInResult.add(ks)
+          }
+        }
+      }
+    }
+
+    return result
+  }
+
   /**
    * Performs an IN array lookup
    */

package/src/indexes/index-options.ts

@@ -8,7 +8,7 @@ export interface IndexOptions<TResolver extends IndexResolver = IndexResolver> {
   indexType?: TResolver
   options?: TResolver extends IndexConstructor<any>
     ? TResolver extends new (
-        id: string,
+        id: number,
         expr: any,
         name?: string,
         options?: infer O
@@ -17,7 +17,7 @@ export interface IndexOptions<TResolver extends IndexResolver = IndexResolver> {
       : never
     : TResolver extends () => Promise<infer TCtor>
       ? TCtor extends new (
-          id: string,
+          id: number,
           expr: any,
           name?: string,
           options?: infer O

package/src/query/builder/index.ts

@@ -19,12 +19,14 @@ import type {
   QueryIR,
 } from "../ir.js"
 import type {
+  CompareOptions,
   Context,
   GroupByCallback,
   JoinOnCallback,
   MergeContext,
   MergeContextWithJoinType,
   OrderByCallback,
+  OrderByOptions,
   RefProxyForContext,
   ResultTypeFromSelect,
   SchemaFromSource,
@@ -478,16 +480,31 @@ export class BaseQueryBuilder<TContext extends Context = Context> {
    */
   orderBy(
     callback: OrderByCallback<TContext>,
-    direction: OrderByDirection = `asc`
+    options: OrderByDirection | OrderByOptions = `asc`
   ): QueryBuilder<TContext> {
     const aliases = this._getCurrentAliases()
     const refProxy = createRefProxy(aliases) as RefProxyForContext<TContext>
     const result = callback(refProxy)
 
+    const opts: CompareOptions =
+      typeof options === `string`
+        ? { direction: options, nulls: `first`, stringSort: `locale` }
+        : {
+            direction: options.direction ?? `asc`,
+            nulls: options.nulls ?? `first`,
+            stringSort: options.stringSort ?? `locale`,
+            locale:
+              options.stringSort === `locale` ? options.locale : undefined,
+            localeOptions:
+              options.stringSort === `locale`
+                ? options.localeOptions
+                : undefined,
+          }
+
     // Create the new OrderBy structure with expression and direction
     const orderByClause: OrderByClause = {
       expression: toExpression(result),
-      direction,
+      compareOptions: opts,
     }
 
     const existingOrderBy: OrderBy = this.query.orderBy || []