@tanstack/db 0.4.5 → 0.4.7

package/src/local-storage.ts

@@ -12,6 +12,7 @@ import type {
   DeleteMutationFnParams,
   InferSchemaOutput,
   InsertMutationFnParams,
+  PendingMutation,
   SyncConfig,
   UpdateMutationFnParams,
   UtilsRecord,
@@ -90,6 +91,26 @@ export type GetStorageSizeFn = () => number
 export interface LocalStorageCollectionUtils extends UtilsRecord {
   clearStorage: ClearStorageFn
   getStorageSize: GetStorageSizeFn
+  /**
+   * Accepts mutations from a transaction that belong to this collection and persists them to localStorage.
+   * This should be called in your transaction's mutationFn to persist local-storage data.
+   *
+   * @param transaction - The transaction containing mutations to accept
+   * @example
+   * const localSettings = createCollection(localStorageCollectionOptions({...}))
+   *
+   * const tx = createTransaction({
+   *   mutationFn: async ({ transaction }) => {
+   *     // Make API call first
+   *     await api.save(...)
+   *     // Then persist local-storage mutations after success
+   *     localSettings.utils.acceptMutations(transaction)
+   *   }
+   * })
+   */
+  acceptMutations: (transaction: {
+    mutations: Array<PendingMutation<Record<string, unknown>>>
+  }) => void
 }
 
 /**
@@ -123,11 +144,17 @@ function generateUuid(): string {
  * This function creates a collection that persists data to localStorage/sessionStorage
  * and synchronizes changes across browser tabs using storage events.
  *
+ * **Using with Manual Transactions:**
+ *
+ * For manual transactions, you must call `utils.acceptMutations()` in your transaction's `mutationFn`
+ * to persist changes made during `tx.mutate()`. This is necessary because local-storage collections
+ * don't participate in the standard mutation handler flow for manual transactions.
+ *
  * @template TExplicit - The explicit type of items in the collection (highest priority)
  * @template TSchema - The schema type for validation and type inference (second priority)
  * @template TFallback - The fallback type if no explicit or schema type is provided
  * @param config - Configuration options for the localStorage collection
- * @returns Collection options with utilities including clearStorage and getStorageSize
+ * @returns Collection options with utilities including clearStorage, getStorageSize, and acceptMutations
  *
  * @example
  * // Basic localStorage collection
@@ -159,6 +186,33 @@ function generateUuid(): string {
  *     },
  *   })
  * )
+ *
+ * @example
+ * // Using with manual transactions
+ * const localSettings = createCollection(
+ *   localStorageCollectionOptions({
+ *     storageKey: 'user-settings',
+ *     getKey: (item) => item.id,
+ *   })
+ * )
+ *
+ * const tx = createTransaction({
+ *   mutationFn: async ({ transaction }) => {
+ *     // Use settings data in API call
+ *     const settingsMutations = transaction.mutations.filter(m => m.collection === localSettings)
+ *     await api.updateUserProfile({ settings: settingsMutations[0]?.modified })
+ *
+ *     // Persist local-storage mutations after API success
+ *     localSettings.utils.acceptMutations(transaction)
+ *   }
+ * })
+ *
+ * tx.mutate(() => {
+ *   localSettings.insert({ id: 'theme', value: 'dark' })
+ *   apiCollection.insert({ id: 2, data: 'profile data' })
+ * })
+ *
+ * await tx.commit()
  */
 
 // Overload for when schema is provided
@@ -397,6 +451,76 @@ export function localStorageCollectionOptions(
   // Default id to a pattern based on storage key if not provided
   const collectionId = id ?? `local-collection:${config.storageKey}`
 
+  /**
+   * Accepts mutations from a transaction that belong to this collection and persists them to storage
+   */
+  const acceptMutations = (transaction: {
+    mutations: Array<PendingMutation<Record<string, unknown>>>
+  }) => {
+    // Filter mutations that belong to this collection
+    // Use collection ID for filtering if collection reference isn't available yet
+    const collectionMutations = transaction.mutations.filter((m) => {
+      // Try to match by collection reference first
+      if (sync.collection && m.collection === sync.collection) {
+        return true
+      }
+      // Fall back to matching by collection ID
+      return m.collection.id === collectionId
+    })
+
+    if (collectionMutations.length === 0) {
+      return
+    }
+
+    // Validate all mutations can be serialized before modifying storage
+    for (const mutation of collectionMutations) {
+      switch (mutation.type) {
+        case `insert`:
+        case `update`:
+          validateJsonSerializable(mutation.modified, mutation.type)
+          break
+        case `delete`:
+          validateJsonSerializable(mutation.original, mutation.type)
+          break
+      }
+    }
+
+    // Load current data from storage
+    const currentData = loadFromStorage<Record<string, unknown>>(
+      config.storageKey,
+      storage
+    )
+
+    // Apply each mutation
+    for (const mutation of collectionMutations) {
+      // Use the engine's pre-computed key to avoid key derivation issues
+      const key = mutation.key
+
+      switch (mutation.type) {
+        case `insert`:
+        case `update`: {
+          const storedItem: StoredItem<Record<string, unknown>> = {
+            versionKey: generateUuid(),
+            data: mutation.modified,
+          }
+          currentData.set(key, storedItem)
+          break
+        }
+        case `delete`: {
+          currentData.delete(key)
+          break
+        }
+      }
+    }
+
+    // Save to storage
+    saveToStorage(currentData)
+
+    // Confirm the mutations in the collection to move them from optimistic to synced state
+    // This writes them through the sync interface to make them "synced" instead of "optimistic"
+    sync.confirmOperationsSync(collectionMutations)
+  }
+
   return {
     ...restConfig,
     id: collectionId,
@@ -407,6 +531,7 @@ export function localStorageCollectionOptions(
     utils: {
       clearStorage,
       getStorageSize,
+      acceptMutations,
     },
   }
 }
@@ -480,8 +605,13 @@ function createLocalStorageSync<T extends object>(
   storageEventApi: StorageEventApi,
   _getKey: (item: T) => string | number,
   lastKnownData: Map<string | number, StoredItem<T>>
-): SyncConfig<T> & { manualTrigger?: () => void } {
+): SyncConfig<T> & {
+  manualTrigger?: () => void
+  collection: any
+  confirmOperationsSync: (mutations: Array<any>) => void
+} {
   let syncParams: Parameters<SyncConfig<T>[`sync`]>[0] | null = null
+  let collection: any = null
 
   /**
    * Compare two Maps to find differences using version keys
@@ -556,12 +686,16 @@ function createLocalStorageSync<T extends object>(
     }
   }
 
-  const syncConfig: SyncConfig<T> & { manualTrigger?: () => void } = {
+  const syncConfig: SyncConfig<T> & {
+    manualTrigger?: () => void
+    collection: any
+  } = {
     sync: (params: Parameters<SyncConfig<T>[`sync`]>[0]) => {
       const { begin, write, commit, markReady } = params
 
-      // Store sync params for later use
+      // Store sync params and collection for later use
       syncParams = params
+      collection = params.collection
 
       // Initial load
       const initialData = loadFromStorage<T>(storageKey, storage)
@@ -613,7 +747,38 @@ function createLocalStorageSync<T extends object>(
 
     // Manual trigger function for local updates
     manualTrigger: processStorageChanges,
+
+    // Collection instance reference
+    collection,
   }
 
-  return syncConfig
+  /**
+   * Confirms mutations by writing them through the sync interface
+   * This moves mutations from optimistic to synced state
+   * @param mutations - Array of mutation objects to confirm
+   */
+  const confirmOperationsSync = (mutations: Array<any>) => {
+    if (!syncParams) {
+      // Sync not initialized yet, mutations will be handled on next sync
+      return
+    }
+
+    const { begin, write, commit } = syncParams
+
+    // Write the mutations through sync to confirm them
+    begin()
+    mutations.forEach((mutation: any) => {
+      write({
+        type: mutation.type,
+        value:
+          mutation.type === `delete` ? mutation.original : mutation.modified,
+      })
+    })
+    commit()
+  }
+
+  return {
+    ...syncConfig,
+    confirmOperationsSync,
+  }
 }

package/src/query/compiler/expressions.ts

@@ -1,5 +1,5 @@
 import { Func, PropRef, Value } from "../ir.js"
-import type { BasicExpression } from "../ir.js"
+import type { BasicExpression, OrderBy } from "../ir.js"
 
 /**
  * Functions supported by the collection index system.
@@ -90,3 +90,28 @@ export function convertToBasicExpression(
     return new Func(whereClause.name, args)
   }
 }
+
+export function convertOrderByToBasicExpression(
+  orderBy: OrderBy,
+  collectionAlias: string
+): OrderBy {
+  const normalizedOrderBy = orderBy.map((clause) => {
+    const basicExp = convertToBasicExpression(
+      clause.expression,
+      collectionAlias
+    )
+
+    if (!basicExp) {
+      throw new Error(
+        `Failed to convert orderBy expression to a basic expression: ${clause.expression}`
+      )
+    }
+
+    return {
+      ...clause,
+      expression: basicExp,
+    }
+  })
+
+  return normalizedOrderBy
+}

package/src/query/compiler/order-by.ts

@@ -6,13 +6,14 @@ import { findIndexForField } from "../../utils/index-optimization.js"
 import { compileExpression } from "./evaluators.js"
 import { replaceAggregatesByRefs } from "./group-by.js"
 import type { CompiledSingleRowExpression } from "./evaluators.js"
-import type { OrderByClause, QueryIR, Select } from "../ir.js"
+import type { OrderBy, OrderByClause, QueryIR, Select } from "../ir.js"
 import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js"
 import type { IStreamBuilder, KeyValue } from "@tanstack/db-ivm"
 import type { IndexInterface } from "../../indexes/base-index.js"
 import type { Collection } from "../../collection/index.js"
 
 export type OrderByOptimizationInfo = {
+  orderBy: OrderBy
   offset: number
   limit: number
   comparator: (
@@ -160,6 +161,7 @@ export function processOrderBy(
           comparator,
           valueExtractorForRawRow,
           index,
+          orderBy: orderByClause,
         }
 
         optimizableOrderByCollections[followRefCollection.id] =

package/src/query/live/collection-subscriber.ts

@@ -1,5 +1,8 @@
 import { MultiSet } from "@tanstack/db-ivm"
-import { convertToBasicExpression } from "../compiler/expressions.js"
+import {
+  convertOrderByToBasicExpression,
+  convertToBasicExpression,
+} from "../compiler/expressions.js"
 import type { FullSyncState } from "./types.js"
 import type { MultiSetArray, RootStreamBuilder } from "@tanstack/db-ivm"
 import type { Collection } from "../../collection/index.js"
@@ -16,26 +19,29 @@ export class CollectionSubscriber<
   // Keep track of the biggest value we've sent so far (needed for orderBy optimization)
   private biggest: any = undefined
 
+  private collectionAlias: string
+
   constructor(
     private collectionId: string,
     private collection: Collection,
     private config: Parameters<SyncConfig<TResult>[`sync`]>[0],
     private syncState: FullSyncState,
     private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>
-  ) {}
-
-  subscribe(): CollectionSubscription {
-    const collectionAlias = findCollectionAlias(
+  ) {
+    this.collectionAlias = findCollectionAlias(
       this.collectionId,
       this.collectionConfigBuilder.query
-    )
-    const whereClause = this.getWhereClauseFromAlias(collectionAlias)
+    )!
+  }
+
+  subscribe(): CollectionSubscription {
+    const whereClause = this.getWhereClauseFromAlias(this.collectionAlias)
 
     if (whereClause) {
       // Convert WHERE clause to BasicExpression format for collection subscription
       const whereExpression = convertToBasicExpression(
         whereClause,
-        collectionAlias!
+        this.collectionAlias
       )
 
       if (whereExpression) {
@@ -128,7 +134,7 @@ export class CollectionSubscriber<
   private subscribeToOrderedChanges(
     whereExpression: BasicExpression<boolean> | undefined
   ) {
-    const { offset, limit, comparator, dataNeeded, index } =
+    const { orderBy, offset, limit, comparator, dataNeeded, index } =
       this.collectionConfigBuilder.optimizableOrderByCollections[
         this.collectionId
       ]!
@@ -164,10 +170,17 @@ export class CollectionSubscriber<
 
     subscription.setOrderByIndex(index)
 
+    // Normalize the orderBy clauses such that the references are relative to the collection
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    )
+
     // Load the first `offset + limit` values from the index
     // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[
     subscription.requestLimitedSnapshot({
       limit: offset + limit,
+      orderBy: normalizedOrderBy,
     })
 
     return subscription
@@ -225,7 +238,7 @@ export class CollectionSubscriber<
   // Loads the next `n` items from the collection
   // starting from the biggest item it has sent
   private loadNextItems(n: number, subscription: CollectionSubscription) {
-    const { valueExtractorForRawRow } =
+    const { orderBy, valueExtractorForRawRow } =
       this.collectionConfigBuilder.optimizableOrderByCollections[
         this.collectionId
       ]!
@@ -233,8 +246,16 @@ export class CollectionSubscriber<
     const biggestSentValue = biggestSentRow
       ? valueExtractorForRawRow(biggestSentRow)
       : biggestSentRow
+
+    // Normalize the orderBy clauses such that the references are relative to the collection
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    )
+
     // Take the `n` items after the biggest sent value
     subscription.requestLimitedSnapshot({
+      orderBy: normalizedOrderBy,
       limit: n,
       minValue: biggestSentValue,
     })

package/src/types.ts

@@ -150,6 +150,18 @@ export type Row<TExtensions = never> = Record<string, Value<TExtensions>>
 
 export type OperationType = `insert` | `update` | `delete`
 
+export type OnLoadMoreOptions = {
+  where?: BasicExpression<boolean>
+  orderBy?: OrderBy
+  limit?: number
+}
+
+export type CleanupFn = () => void
+
+export type SyncConfigRes = {
+  cleanup?: CleanupFn
+  onLoadMore?: (options: OnLoadMoreOptions) => void | Promise<void>
+}
 export interface SyncConfig<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
@@ -161,7 +173,7 @@ export interface SyncConfig<
     commit: () => void
     markReady: () => void
     truncate: () => void
-  }) => void
+  }) => void | CleanupFn | SyncConfigRes
 
   /**
    * Get the sync metadata for insert operations