@tanstack/db 0.4.5 → 0.4.7

package/src/collection/sync.ts

@@ -9,7 +9,13 @@ import {
 } from "../errors"
 import { deepEquals } from "../utils"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
-import type { ChangeMessage, CollectionConfig } from "../types"
+import type {
+  ChangeMessage,
+  CleanupFn,
+  CollectionConfig,
+  OnLoadMoreOptions,
+  SyncConfigRes,
+} from "../types"
 import type { CollectionImpl } from "./index.js"
 import type { CollectionStateManager } from "./state"
 import type { CollectionLifecycleManager } from "./lifecycle"
@@ -28,6 +34,9 @@ export class CollectionSyncManager<
 
   public preloadPromise: Promise<void> | null = null
   public syncCleanupFn: (() => void) | null = null
+  public syncOnLoadMoreFn:
+    | ((options: OnLoadMoreOptions) => void | Promise<void>)
+    | null = null
 
   /**
    * Creates a new CollectionSyncManager instance
@@ -52,7 +61,6 @@ export class CollectionSyncManager<
    * This is called when the collection is first accessed or preloaded
    */
   public startSync(): void {
-    const state = this.state
     if (
       this.lifecycle.status !== `idle` &&
       this.lifecycle.status !== `cleaned-up`
@@ -63,120 +71,125 @@ export class CollectionSyncManager<
     this.lifecycle.setStatus(`loading`)
 
     try {
-      const cleanupFn = this.config.sync.sync({
-        collection: this.collection,
-        begin: () => {
-          state.pendingSyncedTransactions.push({
-            committed: false,
-            operations: [],
-            deletedKeys: new Set(),
-          })
-        },
-        write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-          const key = this.config.getKey(messageWithoutKey.value)
-
-          let messageType = messageWithoutKey.type
-
-          // Check if an item with this key already exists when inserting
-          if (messageWithoutKey.type === `insert`) {
-            const insertingIntoExistingSynced = state.syncedData.has(key)
-            const hasPendingDeleteForKey =
-              pendingTransaction.deletedKeys.has(key)
-            const isTruncateTransaction = pendingTransaction.truncate === true
-            // Allow insert after truncate in the same transaction even if it existed in syncedData
-            if (
-              insertingIntoExistingSynced &&
-              !hasPendingDeleteForKey &&
-              !isTruncateTransaction
-            ) {
-              const existingValue = state.syncedData.get(key)
+      const syncRes = normalizeSyncFnResult(
+        this.config.sync.sync({
+          collection: this.collection,
+          begin: () => {
+            this.state.pendingSyncedTransactions.push({
+              committed: false,
+              operations: [],
+              deletedKeys: new Set(),
+            })
+          },
+          write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+            const key = this.config.getKey(messageWithoutKey.value)
+
+            let messageType = messageWithoutKey.type
+
+            // Check if an item with this key already exists when inserting
+            if (messageWithoutKey.type === `insert`) {
+              const insertingIntoExistingSynced = this.state.syncedData.has(key)
+              const hasPendingDeleteForKey =
+                pendingTransaction.deletedKeys.has(key)
+              const isTruncateTransaction = pendingTransaction.truncate === true
+              // Allow insert after truncate in the same transaction even if it existed in syncedData
               if (
-                existingValue !== undefined &&
-                deepEquals(existingValue, messageWithoutKey.value)
+                insertingIntoExistingSynced &&
+                !hasPendingDeleteForKey &&
+                !isTruncateTransaction
               ) {
-                // The "insert" is an echo of a value we already have locally.
-                // Treat it as an update so we preserve optimistic intent without
-                // throwing a duplicate-key error during reconciliation.
-                messageType = `update`
-              } else {
-                throw new DuplicateKeySyncError(key, this.id)
+                const existingValue = this.state.syncedData.get(key)
+                if (
+                  existingValue !== undefined &&
+                  deepEquals(existingValue, messageWithoutKey.value)
+                ) {
+                  // The "insert" is an echo of a value we already have locally.
+                  // Treat it as an update so we preserve optimistic intent without
+                  // throwing a duplicate-key error during reconciliation.
+                  messageType = `update`
+                } else {
+                  throw new DuplicateKeySyncError(key, this.id)
+                }
               }
             }
-          }
-
-          const message: ChangeMessage<TOutput> = {
-            ...messageWithoutKey,
-            type: messageType,
-            key,
-          }
-          pendingTransaction.operations.push(message)
-
-          if (messageType === `delete`) {
-            pendingTransaction.deletedKeys.add(key)
-          }
-        },
-        commit: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionCommitError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedError()
-          }
-
-          pendingTransaction.committed = true
-
-          // Update status to initialCommit when transitioning from loading
-          // This indicates we're in the process of committing the first transaction
-          if (this.lifecycle.status === `loading`) {
-            this.lifecycle.setStatus(`initialCommit`)
-          }
-
-          state.commitPendingTransactions()
-        },
-        markReady: () => {
-          this.lifecycle.markReady()
-        },
-        truncate: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-
-          // Clear all operations from the current transaction
-          pendingTransaction.operations = []
-          pendingTransaction.deletedKeys.clear()
-
-          // Mark the transaction as a truncate operation. During commit, this triggers:
-          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
-          // - Clearing of syncedData/syncedMetadata
-          // - Subsequent synced ops applied on the fresh base
-          // - Finally, optimistic mutations re-applied on top (single batch)
-          pendingTransaction.truncate = true
-        },
-      })
+
+            const message: ChangeMessage<TOutput> = {
+              ...messageWithoutKey,
+              type: messageType,
+              key,
+            }
+            pendingTransaction.operations.push(message)
+
+            if (messageType === `delete`) {
+              pendingTransaction.deletedKeys.add(key)
+            }
+          },
+          commit: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionCommitError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedError()
+            }
+
+            pendingTransaction.committed = true
+
+            // Update status to initialCommit when transitioning from loading
+            // This indicates we're in the process of committing the first transaction
+            if (this.lifecycle.status === `loading`) {
+              this.lifecycle.setStatus(`initialCommit`)
+            }
+
+            this.state.commitPendingTransactions()
+          },
+          markReady: () => {
+            this.lifecycle.markReady()
+          },
+          truncate: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+
+            // Clear all operations from the current transaction
+            pendingTransaction.operations = []
+            pendingTransaction.deletedKeys.clear()
+
+            // Mark the transaction as a truncate operation. During commit, this triggers:
+            // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
+            // - Clearing of syncedData/syncedMetadata
+            // - Subsequent synced ops applied on the fresh base
+            // - Finally, optimistic mutations re-applied on top (single batch)
+            pendingTransaction.truncate = true
+          },
+        })
+      )
 
       // Store cleanup function if provided
-      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null
+      this.syncCleanupFn = syncRes?.cleanup ?? null
+
+      // Store onLoadMore function if provided
+      this.syncOnLoadMoreFn = syncRes?.onLoadMore ?? null
     } catch (error) {
       this.lifecycle.setStatus(`error`)
       throw error
@@ -225,6 +238,18 @@ export class CollectionSyncManager<
     return this.preloadPromise
   }
 
+  /**
+   * Requests the sync layer to load more data.
+   * @param options Options to control what data is being loaded
+   * @returns If data loading is asynchronous, this method returns a promise that resolves when the data is loaded.
+   *          If data loading is synchronous, the data is loaded when the method returns.
+   */
+  public syncMore(options: OnLoadMoreOptions): void | Promise<void> {
+    if (this.syncOnLoadMoreFn) {
+      return this.syncOnLoadMoreFn(options)
+    }
+  }
+
   public cleanup(): void {
     try {
       if (this.syncCleanupFn) {
@@ -248,3 +273,15 @@ export class CollectionSyncManager<
     this.preloadPromise = null
   }
 }
+
+function normalizeSyncFnResult(result: void | CleanupFn | SyncConfigRes) {
+  if (typeof result === `function`) {
+    return { cleanup: result }
+  }
+
+  if (typeof result === `object`) {
+    return result
+  }
+
+  return undefined
+}

package/src/index.ts

@@ -1,4 +1,8 @@
 // Re-export all public APIs
+// Re-export IR types under their own namespace
+// because custom collections need access to the IR types
+import * as IR from "./query/ir.js"
+
 export * from "./collection/index.js"
 export * from "./SortedMap"
 export * from "./transactions"
@@ -18,3 +22,4 @@ export { type IndexOptions } from "./indexes/index-options.js"
 
 // Re-export some stuff explicitly to ensure the type & value is exported
 export type { Collection } from "./collection/index.js"
+export { IR }

package/src/indexes/auto-index.ts

@@ -58,7 +58,10 @@ export function ensureIndexForField<
       options: compareFn ? { compareFn, compareOptions } : {},
     })
   } catch (error) {
-    console.warn(`Failed to create auto-index for field "${fieldName}":`, error)
+    console.warn(
+      `${collection.id ? `[${collection.id}] ` : ``}Failed to create auto-index for field "${fieldName}":`,
+      error
+    )
   }
 }
 

package/src/local-only.ts

@@ -1,14 +1,19 @@
 import type {
   BaseCollectionConfig,
   CollectionConfig,
+  DeleteMutationFn,
   DeleteMutationFnParams,
   InferSchemaOutput,
+  InsertMutationFn,
   InsertMutationFnParams,
   OperationType,
+  PendingMutation,
   SyncConfig,
+  UpdateMutationFn,
   UpdateMutationFnParams,
   UtilsRecord,
 } from "./types"
+import type { Collection } from "./collection/index"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
 
 /**
@@ -33,9 +38,44 @@ export interface LocalOnlyCollectionConfig<
 }
 
 /**
- * Local-only collection utilities type (currently empty but matches the pattern)
+ * Local-only collection utilities type
  */
-export interface LocalOnlyCollectionUtils extends UtilsRecord {}
+export interface LocalOnlyCollectionUtils extends UtilsRecord {
+  /**
+   * Accepts mutations from a transaction that belong to this collection and persists them.
+   * This should be called in your transaction's mutationFn to persist local-only data.
+   *
+   * @param transaction - The transaction containing mutations to accept
+   * @example
+   * const localData = createCollection(localOnlyCollectionOptions({...}))
+   *
+   * const tx = createTransaction({
+   *   mutationFn: async ({ transaction }) => {
+   *     // Make API call first
+   *     await api.save(...)
+   *     // Then persist local-only mutations after success
+   *     localData.utils.acceptMutations(transaction)
+   *   }
+   * })
+   */
+  acceptMutations: (transaction: {
+    mutations: Array<PendingMutation<Record<string, unknown>>>
+  }) => void
+}
+
+type LocalOnlyCollectionOptionsResult<
+  T extends object,
+  TKey extends string | number,
+  TSchema extends StandardSchemaV1 | never = never,
+> = Omit<
+  CollectionConfig<T, TKey, TSchema>,
+  `onInsert` | `onUpdate` | `onDelete`
+> & {
+  onInsert?: InsertMutationFn<T, TKey, LocalOnlyCollectionUtils>
+  onUpdate?: UpdateMutationFn<T, TKey, LocalOnlyCollectionUtils>
+  onDelete?: DeleteMutationFn<T, TKey, LocalOnlyCollectionUtils>
+  utils: LocalOnlyCollectionUtils
+}
 
 /**
  * Creates Local-only collection options for use with a standard Collection
@@ -44,10 +84,16 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {}
  * that immediately "syncs" all optimistic changes to the collection, making them permanent.
  * Perfect for local-only data that doesn't need persistence or external synchronization.
  *
+ * **Using with Manual Transactions:**
+ *
+ * For manual transactions, you must call `utils.acceptMutations()` in your transaction's `mutationFn`
+ * to persist changes made during `tx.mutate()`. This is necessary because local-only collections
+ * don't participate in the standard mutation handler flow for manual transactions.
+ *
  * @template T - The schema type if a schema is provided, otherwise the type of items in the collection
  * @template TKey - The type of the key returned by getKey
  * @param config - Configuration options for the Local-only collection
- * @returns Collection options with utilities (currently empty but follows the pattern)
+ * @returns Collection options with utilities including acceptMutations
  *
  * @example
  * // Basic local-only collection
@@ -80,6 +126,32 @@ export interface LocalOnlyCollectionUtils extends UtilsRecord {}
  *     },
  *   })
  * )
+ *
+ * @example
+ * // Using with manual transactions
+ * const localData = createCollection(
+ *   localOnlyCollectionOptions({
+ *     getKey: (item) => item.id,
+ *   })
+ * )
+ *
+ * const tx = createTransaction({
+ *   mutationFn: async ({ transaction }) => {
+ *     // Use local data in API call
+ *     const localMutations = transaction.mutations.filter(m => m.collection === localData)
+ *     await api.save({ metadata: localMutations[0]?.modified })
+ *
+ *     // Persist local-only mutations after API success
+ *     localData.utils.acceptMutations(transaction)
+ *   }
+ * })
+ *
+ * tx.mutate(() => {
+ *   localData.insert({ id: 1, data: 'metadata' })
+ *   apiCollection.insert({ id: 2, data: 'main data' })
+ * })
+ *
+ * await tx.commit()
  */
 
 // Overload for when schema is provided
@@ -90,8 +162,7 @@ export function localOnlyCollectionOptions<
   config: LocalOnlyCollectionConfig<InferSchemaOutput<T>, T, TKey> & {
     schema: T
   }
-): CollectionConfig<InferSchemaOutput<T>, TKey, T> & {
-  utils: LocalOnlyCollectionUtils
+): LocalOnlyCollectionOptionsResult<InferSchemaOutput<T>, TKey, T> & {
   schema: T
 }
 
@@ -104,15 +175,17 @@ export function localOnlyCollectionOptions<
   config: LocalOnlyCollectionConfig<T, never, TKey> & {
     schema?: never // prohibit schema
   }
-): CollectionConfig<T, TKey> & {
-  utils: LocalOnlyCollectionUtils
+): LocalOnlyCollectionOptionsResult<T, TKey> & {
   schema?: never // no schema in the result
 }
 
-export function localOnlyCollectionOptions(
-  config: LocalOnlyCollectionConfig<any, any, string | number>
-): CollectionConfig<any, string | number, any> & {
-  utils: LocalOnlyCollectionUtils
+export function localOnlyCollectionOptions<
+  T extends object = object,
+  TSchema extends StandardSchemaV1 = never,
+  TKey extends string | number = string | number,
+>(
+  config: LocalOnlyCollectionConfig<T, TSchema, TKey>
+): LocalOnlyCollectionOptionsResult<T, TKey, TSchema> & {
   schema?: StandardSchemaV1
 } {
   const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config
@@ -125,11 +198,7 @@ export function localOnlyCollectionOptions(
    * Wraps the user's onInsert handler to also confirm the transaction immediately
    */
   const wrappedOnInsert = async (
-    params: InsertMutationFnParams<
-      any,
-      string | number,
-      LocalOnlyCollectionUtils
-    >
+    params: InsertMutationFnParams<T, TKey, LocalOnlyCollectionUtils>
   ) => {
     // Call user handler first if provided
     let handlerResult
@@ -147,11 +216,7 @@ export function localOnlyCollectionOptions(
    * Wrapper for onUpdate handler that also confirms the transaction immediately
    */
   const wrappedOnUpdate = async (
-    params: UpdateMutationFnParams<
-      any,
-      string | number,
-      LocalOnlyCollectionUtils
-    >
+    params: UpdateMutationFnParams<T, TKey, LocalOnlyCollectionUtils>
   ) => {
     // Call user handler first if provided
     let handlerResult
@@ -169,11 +234,7 @@ export function localOnlyCollectionOptions(
    * Wrapper for onDelete handler that also confirms the transaction immediately
    */
   const wrappedOnDelete = async (
-    params: DeleteMutationFnParams<
-      any,
-      string | number,
-      LocalOnlyCollectionUtils
-    >
+    params: DeleteMutationFnParams<T, TKey, LocalOnlyCollectionUtils>
   ) => {
     // Call user handler first if provided
     let handlerResult
@@ -187,13 +248,38 @@ export function localOnlyCollectionOptions(
     return handlerResult
   }
 
+  /**
+   * Accepts mutations from a transaction that belong to this collection and persists them
+   */
+  const acceptMutations = (transaction: {
+    mutations: Array<PendingMutation<Record<string, unknown>>>
+  }) => {
+    // Filter mutations that belong to this collection
+    const collectionMutations = transaction.mutations.filter(
+      (m) =>
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        m.collection === syncResult.collection
+    )
+
+    if (collectionMutations.length === 0) {
+      return
+    }
+
+    // Persist the mutations through sync
+    syncResult.confirmOperationsSync(
+      collectionMutations as Array<PendingMutation<T>>
+    )
+  }
+
   return {
     ...restConfig,
     sync: syncResult.sync,
     onInsert: wrappedOnInsert,
     onUpdate: wrappedOnUpdate,
     onDelete: wrappedOnDelete,
-    utils: {} as LocalOnlyCollectionUtils,
+    utils: {
+      acceptMutations,
+    } as LocalOnlyCollectionUtils,
     startSync: true,
     gcTime: 0,
   }
@@ -212,11 +298,12 @@ export function localOnlyCollectionOptions(
 function createLocalOnlySync<T extends object, TKey extends string | number>(
   initialData?: Array<T>
 ) {
-  // Capture sync functions for transaction confirmation
+  // Capture sync functions and collection for transaction confirmation
   let syncBegin: (() => void) | null = null
   let syncWrite: ((message: { type: OperationType; value: T }) => void) | null =
     null
   let syncCommit: (() => void) | null = null
+  let collection: Collection<T, TKey, LocalOnlyCollectionUtils> | null = null
 
   const sync: SyncConfig<T, TKey> = {
     /**
@@ -227,10 +314,11 @@ function createLocalOnlySync<T extends object, TKey extends string | number>(
     sync: (params) => {
       const { begin, write, commit, markReady } = params
 
-      // Capture sync functions for later use by confirmOperationsSync
+      // Capture sync functions and collection for later use
       syncBegin = begin
       syncWrite = write
       syncCommit = commit
+      collection = params.collection
 
       // Apply initial data if provided
       if (initialData && initialData.length > 0) {
@@ -265,7 +353,7 @@ function createLocalOnlySync<T extends object, TKey extends string | number>(
    *
    * @param mutations - Array of mutation objects from the transaction
    */
-  const confirmOperationsSync = (mutations: Array<any>) => {
+  const confirmOperationsSync = (mutations: Array<PendingMutation<T>>) => {
     if (!syncBegin || !syncWrite || !syncCommit) {
       return // Sync not initialized yet, which is fine
     }
@@ -286,5 +374,6 @@ function createLocalOnlySync<T extends object, TKey extends string | number>(
   return {
     sync,
     confirmOperationsSync,
+    collection,
   }
 }