@tanstack/db 0.4.4 → 0.4.6

package/src/collection/sync.ts

@@ -9,7 +9,13 @@ import {
 } from "../errors"
 import { deepEquals } from "../utils"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
-import type { ChangeMessage, CollectionConfig } from "../types"
+import type {
+  ChangeMessage,
+  CleanupFn,
+  CollectionConfig,
+  OnLoadMoreOptions,
+  SyncConfigRes,
+} from "../types"
 import type { CollectionImpl } from "./index.js"
 import type { CollectionStateManager } from "./state"
 import type { CollectionLifecycleManager } from "./lifecycle"
@@ -28,6 +34,9 @@ export class CollectionSyncManager<
 
   public preloadPromise: Promise<void> | null = null
   public syncCleanupFn: (() => void) | null = null
+  public syncOnLoadMoreFn:
+    | ((options: OnLoadMoreOptions) => void | Promise<void>)
+    | null = null
 
   /**
    * Creates a new CollectionSyncManager instance
@@ -52,7 +61,6 @@ export class CollectionSyncManager<
    * This is called when the collection is first accessed or preloaded
    */
   public startSync(): void {
-    const state = this.state
     if (
       this.lifecycle.status !== `idle` &&
       this.lifecycle.status !== `cleaned-up`
@@ -63,120 +71,125 @@ export class CollectionSyncManager<
     this.lifecycle.setStatus(`loading`)
 
     try {
-      const cleanupFn = this.config.sync.sync({
-        collection: this.collection,
-        begin: () => {
-          state.pendingSyncedTransactions.push({
-            committed: false,
-            operations: [],
-            deletedKeys: new Set(),
-          })
-        },
-        write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-          const key = this.config.getKey(messageWithoutKey.value)
-
-          let messageType = messageWithoutKey.type
-
-          // Check if an item with this key already exists when inserting
-          if (messageWithoutKey.type === `insert`) {
-            const insertingIntoExistingSynced = state.syncedData.has(key)
-            const hasPendingDeleteForKey =
-              pendingTransaction.deletedKeys.has(key)
-            const isTruncateTransaction = pendingTransaction.truncate === true
-            // Allow insert after truncate in the same transaction even if it existed in syncedData
-            if (
-              insertingIntoExistingSynced &&
-              !hasPendingDeleteForKey &&
-              !isTruncateTransaction
-            ) {
-              const existingValue = state.syncedData.get(key)
+      const syncRes = normalizeSyncFnResult(
+        this.config.sync.sync({
+          collection: this.collection,
+          begin: () => {
+            this.state.pendingSyncedTransactions.push({
+              committed: false,
+              operations: [],
+              deletedKeys: new Set(),
+            })
+          },
+          write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+            const key = this.config.getKey(messageWithoutKey.value)
+
+            let messageType = messageWithoutKey.type
+
+            // Check if an item with this key already exists when inserting
+            if (messageWithoutKey.type === `insert`) {
+              const insertingIntoExistingSynced = this.state.syncedData.has(key)
+              const hasPendingDeleteForKey =
+                pendingTransaction.deletedKeys.has(key)
+              const isTruncateTransaction = pendingTransaction.truncate === true
+              // Allow insert after truncate in the same transaction even if it existed in syncedData
               if (
-                existingValue !== undefined &&
-                deepEquals(existingValue, messageWithoutKey.value)
+                insertingIntoExistingSynced &&
+                !hasPendingDeleteForKey &&
+                !isTruncateTransaction
               ) {
-                // The "insert" is an echo of a value we already have locally.
-                // Treat it as an update so we preserve optimistic intent without
-                // throwing a duplicate-key error during reconciliation.
-                messageType = `update`
-              } else {
-                throw new DuplicateKeySyncError(key, this.id)
+                const existingValue = this.state.syncedData.get(key)
+                if (
+                  existingValue !== undefined &&
+                  deepEquals(existingValue, messageWithoutKey.value)
+                ) {
+                  // The "insert" is an echo of a value we already have locally.
+                  // Treat it as an update so we preserve optimistic intent without
+                  // throwing a duplicate-key error during reconciliation.
+                  messageType = `update`
+                } else {
+                  throw new DuplicateKeySyncError(key, this.id)
+                }
               }
             }
-          }
-
-          const message: ChangeMessage<TOutput> = {
-            ...messageWithoutKey,
-            type: messageType,
-            key,
-          }
-          pendingTransaction.operations.push(message)
-
-          if (messageType === `delete`) {
-            pendingTransaction.deletedKeys.add(key)
-          }
-        },
-        commit: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionCommitError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedError()
-          }
-
-          pendingTransaction.committed = true
-
-          // Update status to initialCommit when transitioning from loading
-          // This indicates we're in the process of committing the first transaction
-          if (this.lifecycle.status === `loading`) {
-            this.lifecycle.setStatus(`initialCommit`)
-          }
-
-          state.commitPendingTransactions()
-        },
-        markReady: () => {
-          this.lifecycle.markReady()
-        },
-        truncate: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-
-          // Clear all operations from the current transaction
-          pendingTransaction.operations = []
-          pendingTransaction.deletedKeys.clear()
-
-          // Mark the transaction as a truncate operation. During commit, this triggers:
-          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
-          // - Clearing of syncedData/syncedMetadata
-          // - Subsequent synced ops applied on the fresh base
-          // - Finally, optimistic mutations re-applied on top (single batch)
-          pendingTransaction.truncate = true
-        },
-      })
+
+            const message: ChangeMessage<TOutput> = {
+              ...messageWithoutKey,
+              type: messageType,
+              key,
+            }
+            pendingTransaction.operations.push(message)
+
+            if (messageType === `delete`) {
+              pendingTransaction.deletedKeys.add(key)
+            }
+          },
+          commit: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionCommitError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedError()
+            }
+
+            pendingTransaction.committed = true
+
+            // Update status to initialCommit when transitioning from loading
+            // This indicates we're in the process of committing the first transaction
+            if (this.lifecycle.status === `loading`) {
+              this.lifecycle.setStatus(`initialCommit`)
+            }
+
+            this.state.commitPendingTransactions()
+          },
+          markReady: () => {
+            this.lifecycle.markReady()
+          },
+          truncate: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+
+            // Clear all operations from the current transaction
+            pendingTransaction.operations = []
+            pendingTransaction.deletedKeys.clear()
+
+            // Mark the transaction as a truncate operation. During commit, this triggers:
+            // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
+            // - Clearing of syncedData/syncedMetadata
+            // - Subsequent synced ops applied on the fresh base
+            // - Finally, optimistic mutations re-applied on top (single batch)
+            pendingTransaction.truncate = true
+          },
+        })
+      )
 
       // Store cleanup function if provided
-      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null
+      this.syncCleanupFn = syncRes?.cleanup ?? null
+
+      // Store onLoadMore function if provided
+      this.syncOnLoadMoreFn = syncRes?.onLoadMore ?? null
     } catch (error) {
       this.lifecycle.setStatus(`error`)
       throw error
@@ -225,6 +238,18 @@ export class CollectionSyncManager<
     return this.preloadPromise
   }
 
+  /**
+   * Requests the sync layer to load more data.
+   * @param options Options to control what data is being loaded
+   * @returns If data loading is asynchronous, this method returns a promise that resolves when the data is loaded.
+   *          If data loading is synchronous, the data is loaded when the method returns.
+   */
+  public syncMore(options: OnLoadMoreOptions): void | Promise<void> {
+    if (this.syncOnLoadMoreFn) {
+      return this.syncOnLoadMoreFn(options)
+    }
+  }
+
   public cleanup(): void {
     try {
       if (this.syncCleanupFn) {
@@ -248,3 +273,15 @@ export class CollectionSyncManager<
     this.preloadPromise = null
   }
 }
+
+function normalizeSyncFnResult(result: void | CleanupFn | SyncConfigRes) {
+  if (typeof result === `function`) {
+    return { cleanup: result }
+  }
+
+  if (typeof result === `object`) {
+    return result
+  }
+
+  return undefined
+}

package/src/index.ts

@@ -1,4 +1,8 @@
 // Re-export all public APIs
+// Re-export IR types under their own namespace
+// because custom collections need access to the IR types
+import * as IR from "./query/ir.js"
+
 export * from "./collection/index.js"
 export * from "./SortedMap"
 export * from "./transactions"
@@ -18,3 +22,4 @@ export { type IndexOptions } from "./indexes/index-options.js"
 
 // Re-export some stuff explicitly to ensure the type & value is exported
 export type { Collection } from "./collection/index.js"
+export { IR }

package/src/indexes/auto-index.ts

@@ -58,7 +58,10 @@ export function ensureIndexForField<
       options: compareFn ? { compareFn, compareOptions } : {},
     })
   } catch (error) {
-    console.warn(`Failed to create auto-index for field "${fieldName}":`, error)
+    console.warn(
+      `${collection.id ? `[${collection.id}] ` : ``}Failed to create auto-index for field "${fieldName}":`,
+      error
+    )
   }
 }
 

package/src/query/compiler/expressions.ts

@@ -1,5 +1,5 @@
 import { Func, PropRef, Value } from "../ir.js"
-import type { BasicExpression } from "../ir.js"
+import type { BasicExpression, OrderBy } from "../ir.js"
 
 /**
  * Functions supported by the collection index system.
@@ -90,3 +90,28 @@ export function convertToBasicExpression(
     return new Func(whereClause.name, args)
   }
 }
+
+export function convertOrderByToBasicExpression(
+  orderBy: OrderBy,
+  collectionAlias: string
+): OrderBy {
+  const normalizedOrderBy = orderBy.map((clause) => {
+    const basicExp = convertToBasicExpression(
+      clause.expression,
+      collectionAlias
+    )
+
+    if (!basicExp) {
+      throw new Error(
+        `Failed to convert orderBy expression to a basic expression: ${clause.expression}`
+      )
+    }
+
+    return {
+      ...clause,
+      expression: basicExp,
+    }
+  })
+
+  return normalizedOrderBy
+}

package/src/query/compiler/order-by.ts

@@ -6,13 +6,14 @@ import { findIndexForField } from "../../utils/index-optimization.js"
 import { compileExpression } from "./evaluators.js"
 import { replaceAggregatesByRefs } from "./group-by.js"
 import type { CompiledSingleRowExpression } from "./evaluators.js"
-import type { OrderByClause, QueryIR, Select } from "../ir.js"
+import type { OrderBy, OrderByClause, QueryIR, Select } from "../ir.js"
 import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js"
 import type { IStreamBuilder, KeyValue } from "@tanstack/db-ivm"
 import type { IndexInterface } from "../../indexes/base-index.js"
 import type { Collection } from "../../collection/index.js"
 
 export type OrderByOptimizationInfo = {
+  orderBy: OrderBy
   offset: number
   limit: number
   comparator: (
@@ -160,6 +161,7 @@ export function processOrderBy(
           comparator,
           valueExtractorForRawRow,
           index,
+          orderBy: orderByClause,
         }
 
         optimizableOrderByCollections[followRefCollection.id] =

package/src/query/live/collection-subscriber.ts

@@ -1,5 +1,8 @@
 import { MultiSet } from "@tanstack/db-ivm"
-import { convertToBasicExpression } from "../compiler/expressions.js"
+import {
+  convertOrderByToBasicExpression,
+  convertToBasicExpression,
+} from "../compiler/expressions.js"
 import type { FullSyncState } from "./types.js"
 import type { MultiSetArray, RootStreamBuilder } from "@tanstack/db-ivm"
 import type { Collection } from "../../collection/index.js"
@@ -16,26 +19,29 @@ export class CollectionSubscriber<
   // Keep track of the biggest value we've sent so far (needed for orderBy optimization)
   private biggest: any = undefined
 
+  private collectionAlias: string
+
   constructor(
     private collectionId: string,
     private collection: Collection,
     private config: Parameters<SyncConfig<TResult>[`sync`]>[0],
     private syncState: FullSyncState,
     private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>
-  ) {}
-
-  subscribe(): CollectionSubscription {
-    const collectionAlias = findCollectionAlias(
+  ) {
+    this.collectionAlias = findCollectionAlias(
       this.collectionId,
       this.collectionConfigBuilder.query
-    )
-    const whereClause = this.getWhereClauseFromAlias(collectionAlias)
+    )!
+  }
+
+  subscribe(): CollectionSubscription {
+    const whereClause = this.getWhereClauseFromAlias(this.collectionAlias)
 
     if (whereClause) {
       // Convert WHERE clause to BasicExpression format for collection subscription
       const whereExpression = convertToBasicExpression(
         whereClause,
-        collectionAlias!
+        this.collectionAlias
       )
 
       if (whereExpression) {
@@ -128,7 +134,7 @@ export class CollectionSubscriber<
   private subscribeToOrderedChanges(
     whereExpression: BasicExpression<boolean> | undefined
   ) {
-    const { offset, limit, comparator, dataNeeded, index } =
+    const { orderBy, offset, limit, comparator, dataNeeded, index } =
       this.collectionConfigBuilder.optimizableOrderByCollections[
         this.collectionId
       ]!
@@ -164,10 +170,17 @@ export class CollectionSubscriber<
 
     subscription.setOrderByIndex(index)
 
+    // Normalize the orderBy clauses such that the references are relative to the collection
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    )
+
     // Load the first `offset + limit` values from the index
     // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[
     subscription.requestLimitedSnapshot({
       limit: offset + limit,
+      orderBy: normalizedOrderBy,
     })
 
     return subscription
@@ -225,7 +238,7 @@ export class CollectionSubscriber<
   // Loads the next `n` items from the collection
   // starting from the biggest item it has sent
   private loadNextItems(n: number, subscription: CollectionSubscription) {
-    const { valueExtractorForRawRow } =
+    const { orderBy, valueExtractorForRawRow } =
       this.collectionConfigBuilder.optimizableOrderByCollections[
         this.collectionId
       ]!
@@ -233,8 +246,16 @@ export class CollectionSubscriber<
     const biggestSentValue = biggestSentRow
       ? valueExtractorForRawRow(biggestSentRow)
       : biggestSentRow
+
+    // Normalize the orderBy clauses such that the references are relative to the collection
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    )
+
     // Take the `n` items after the biggest sent value
     subscription.requestLimitedSnapshot({
+      orderBy: normalizedOrderBy,
       limit: n,
       minValue: biggestSentValue,
     })

package/src/types.ts

@@ -150,6 +150,18 @@ export type Row<TExtensions = never> = Record<string, Value<TExtensions>>
 
 export type OperationType = `insert` | `update` | `delete`
 
+export type OnLoadMoreOptions = {
+  where?: BasicExpression<boolean>
+  orderBy?: OrderBy
+  limit?: number
+}
+
+export type CleanupFn = () => void
+
+export type SyncConfigRes = {
+  cleanup?: CleanupFn
+  onLoadMore?: (options: OnLoadMoreOptions) => void | Promise<void>
+}
 export interface SyncConfig<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
@@ -161,7 +173,7 @@ export interface SyncConfig<
     commit: () => void
     markReady: () => void
     truncate: () => void
-  }) => void
+  }) => void | CleanupFn | SyncConfigRes
 
   /**
    * Get the sync metadata for insert operations