@tanstack/db 0.4.19 → 0.5.0

package/src/query/subset-dedupe.ts

@@ -0,0 +1,243 @@
+import {
+  isPredicateSubset,
+  isWhereSubset,
+  minusWherePredicates,
+  unionWherePredicates,
+} from "./predicate-utils.js"
+import type { BasicExpression } from "./ir.js"
+import type { LoadSubsetOptions } from "../types.js"
+
+/**
+ * Deduplicated wrapper for a loadSubset function.
+ * Tracks what data has been loaded and avoids redundant calls by applying
+ * subset logic to predicates.
+ *
+ * @param opts - The options for the DeduplicatedLoadSubset
+ * @param opts.loadSubset - The underlying loadSubset function to wrap
+ * @param opts.onDeduplicate - An optional callback function that is invoked when a loadSubset call is deduplicated.
+ *                              If the call is deduplicated because the requested data is being loaded by an inflight request,
+ *                              then this callback is invoked when the inflight request completes successfully and the data is fully loaded.
+ *                              This callback is useful if you need to track rows per query, in which case you can't ignore deduplicated calls
+ *                              because you need to know which rows were loaded for each query.
+ * @example
+ * const dedupe = new DeduplicatedLoadSubset({ loadSubset: myLoadSubset, onDeduplicate: (opts) => console.log(`Call was deduplicated:`, opts) })
+ *
+ * // First call - fetches data
+ * await dedupe.loadSubset({ where: gt(ref('age'), val(10)) })
+ *
+ * // Second call - subset of first, returns true immediately
+ * await dedupe.loadSubset({ where: gt(ref('age'), val(20)) })
+ *
+ * // Clear state to start fresh
+ * dedupe.reset()
+ */
+export class DeduplicatedLoadSubset {
+  // The underlying loadSubset function to wrap
+  private readonly _loadSubset: (
+    options: LoadSubsetOptions
+  ) => true | Promise<void>
+
+  // An optional callback function that is invoked when a loadSubset call is deduplicated.
+  private readonly onDeduplicate:
+    | ((options: LoadSubsetOptions) => void)
+    | undefined
+
+  // Combined where predicate for all unlimited calls (no limit)
+  private unlimitedWhere: BasicExpression<boolean> | undefined = undefined
+
+  // Flag to track if we've loaded all data (unlimited call with no where clause)
+  private hasLoadedAllData = false
+
+  // List of all limited calls (with limit, possibly with orderBy)
+  // We clone options before storing to prevent mutation of stored predicates
+  private limitedCalls: Array<LoadSubsetOptions> = []
+
+  // Track in-flight calls to prevent concurrent duplicate requests
+  // We store both the options and the promise so we can apply subset logic
+  private inflightCalls: Array<{
+    options: LoadSubsetOptions
+    promise: Promise<void>
+  }> = []
+
+  // Generation counter to invalidate in-flight requests after reset()
+  // When reset() is called, this increments, and any in-flight completion handlers
+  // check if their captured generation matches before updating tracking state
+  private generation = 0
+
+  constructor(opts: {
+    loadSubset: (options: LoadSubsetOptions) => true | Promise<void>
+    onDeduplicate?: (options: LoadSubsetOptions) => void
+  }) {
+    this._loadSubset = opts.loadSubset
+    this.onDeduplicate = opts.onDeduplicate
+  }
+
+  /**
+   * Load a subset of data, with automatic deduplication based on previously
+   * loaded predicates and in-flight requests.
+   *
+   * This method is auto-bound, so it can be safely passed as a callback without
+   * losing its `this` context (e.g., `loadSubset: dedupe.loadSubset` in a sync config).
+   *
+   * @param options - The predicate options (where, orderBy, limit)
+   * @returns true if data is already loaded, or a Promise that resolves when data is loaded
+   */
+  loadSubset = (options: LoadSubsetOptions): true | Promise<void> => {
+    // If we've loaded all data, everything is covered
+    if (this.hasLoadedAllData) {
+      this.onDeduplicate?.(options)
+      return true
+    }
+
+    // Check against unlimited combined predicate
+    // If we've loaded all data matching a where clause, we don't need to refetch subsets
+    if (this.unlimitedWhere !== undefined && options.where !== undefined) {
+      if (isWhereSubset(options.where, this.unlimitedWhere)) {
+        this.onDeduplicate?.(options)
+        return true // Data already loaded via unlimited call
+      }
+    }
+
+    // Check against limited calls
+    if (options.limit !== undefined) {
+      const alreadyLoaded = this.limitedCalls.some((loaded) =>
+        isPredicateSubset(options, loaded)
+      )
+
+      if (alreadyLoaded) {
+        this.onDeduplicate?.(options)
+        return true // Already loaded
+      }
+    }
+
+    // Check against in-flight calls using the same subset logic as resolved calls
+    // This prevents duplicate requests when concurrent calls have subset relationships
+    const matchingInflight = this.inflightCalls.find((inflight) =>
+      isPredicateSubset(options, inflight.options)
+    )
+
+    if (matchingInflight !== undefined) {
+      // An in-flight call will load data that covers this request
+      // Return the same promise so this caller waits for the data to load
+      // The in-flight promise already handles tracking updates when it completes
+      const prom = matchingInflight.promise
+      // Call `onDeduplicate` when the inflight request has loaded the data
+      prom.then(() => this.onDeduplicate?.(options)).catch() // ignore errors
+      return prom
+    }
+
+    // Not fully covered by existing data
+    // Compute the subset of data that is not covered by the existing data
+    // such that we only have to load that subset of missing data
+    const clonedOptions = cloneOptions(options)
+    if (this.unlimitedWhere !== undefined && options.limit === undefined) {
+      // Compute difference to get only the missing data
+      // We can only do this for unlimited queries
+      // and we can only remove data that was loaded from unlimited queries
+      // because with limited queries we have no way to express that we already loaded part of the matching data
+      clonedOptions.where =
+        minusWherePredicates(clonedOptions.where, this.unlimitedWhere) ??
+        clonedOptions.where
+    }
+
+    // Call underlying loadSubset to load the missing data
+    const resultPromise = this._loadSubset(clonedOptions)
+
+    // Handle both sync (true) and async (Promise<void>) return values
+    if (resultPromise === true) {
+      // Sync return - update tracking synchronously
+      // Clone options before storing to protect against caller mutation
+      this.updateTracking(clonedOptions)
+      return true
+    } else {
+      // Async return - track the promise and update tracking after it resolves
+
+      // Capture the current generation - this lets us detect if reset() was called
+      // while this request was in-flight, so we can skip updating tracking state
+      const capturedGeneration = this.generation
+
+      // We need to create a reference to the in-flight entry so we can remove it later
+      const inflightEntry = {
+        options: clonedOptions, // Store cloned options for subset matching
+        promise: resultPromise
+          .then((result) => {
+            // Only update tracking if this request is still from the current generation
+            // If reset() was called, the generation will have incremented and we should
+            // not repopulate the state that was just cleared
+            if (capturedGeneration === this.generation) {
+              // Use the cloned options that we captured before any caller mutations
+              // This ensures we track exactly what was loaded, not what the caller changed
+              this.updateTracking(clonedOptions)
+            }
+            return result
+          })
+          .finally(() => {
+            // Always remove from in-flight array on completion OR rejection
+            // This ensures failed requests can be retried instead of being cached forever
+            const index = this.inflightCalls.indexOf(inflightEntry)
+            if (index !== -1) {
+              this.inflightCalls.splice(index, 1)
+            }
+          }),
+      }
+
+      // Store the in-flight entry so concurrent subset calls can wait for it
+      this.inflightCalls.push(inflightEntry)
+      return inflightEntry.promise
+    }
+  }
+
+  /**
+   * Reset all tracking state.
+   * Clears the history of loaded predicates and in-flight calls.
+   * Use this when you want to start fresh, for example after clearing the underlying data store.
+   *
+   * Note: Any in-flight requests will still complete, but they will not update the tracking
+   * state after the reset. This prevents old requests from repopulating cleared state.
+   */
+  reset(): void {
+    this.unlimitedWhere = undefined
+    this.hasLoadedAllData = false
+    this.limitedCalls = []
+    this.inflightCalls = []
+    // Increment generation to invalidate any in-flight completion handlers
+    // This ensures requests that were started before reset() don't repopulate the state
+    this.generation++
+  }
+
+  private updateTracking(options: LoadSubsetOptions): void {
+    // Update tracking based on whether this was a limited or unlimited call
+    if (options.limit === undefined) {
+      // Unlimited call - update combined where predicate
+      // We ignore orderBy for unlimited calls as mentioned in requirements
+      if (options.where === undefined) {
+        // No where clause = all data loaded
+        this.hasLoadedAllData = true
+        this.unlimitedWhere = undefined
+        this.limitedCalls = []
+        this.inflightCalls = []
+      } else if (this.unlimitedWhere === undefined) {
+        this.unlimitedWhere = options.where
+      } else {
+        this.unlimitedWhere = unionWherePredicates([
+          this.unlimitedWhere,
+          options.where,
+        ])
+      }
+    } else {
+      // Limited call - add to list for future subset checks
+      // Options are already cloned by caller to prevent mutation issues
+      this.limitedCalls.push(options)
+    }
+  }
+}
+
+/**
+ * Clones a LoadSubsetOptions object to prevent mutation of stored predicates.
+ * This is crucial because callers often reuse the same options object and mutate
+ * properties like limit or where between calls. Without cloning, our stored history
+ * would reflect the mutated values rather than what was actually loaded.
+ */
+export function cloneOptions(options: LoadSubsetOptions): LoadSubsetOptions {
+  return { ...options }
+}

package/src/types.ts

@@ -5,6 +5,37 @@ import type { Transaction } from "./transactions"
 import type { BasicExpression, OrderBy } from "./query/ir.js"
 import type { EventEmitter } from "./event-emitter.js"
 
+/**
+ * Interface for a collection-like object that provides the necessary methods
+ * for the change events system to work
+ */
+export interface CollectionLike<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> extends Pick<
+    Collection<T, TKey>,
+    `get` | `has` | `entries` | `indexes` | `id` | `compareOptions`
+  > {}
+
+/**
+ * StringSortOpts - Options for string sorting behavior
+ *
+ * This discriminated union allows for two types of string sorting:
+ * - **Lexical**: Simple character-by-character comparison (default)
+ * - **Locale**: Locale-aware sorting with optional customization
+ *
+ * The union ensures that locale options are only available when locale sorting is selected.
+ */
+export type StringCollationConfig =
+  | {
+      stringSort?: `lexical`
+    }
+  | {
+      stringSort?: `locale`
+      locale?: string
+      localeOptions?: object
+    }
+
 /**
  * Helper type to extract the output type from a standard schema
  *
@@ -35,9 +66,9 @@ export type TransactionState = `pending` | `persisting` | `completed` | `failed`
 export type Fn = (...args: Array<any>) => any
 
 /**
- * A record of utility functions that can be attached to a collection
+ * A record of utilities (functions or getters) that can be attached to a collection
  */
-export type UtilsRecord = Record<string, Fn>
+export type UtilsRecord = Record<string, any>
 
 /**
  *
@@ -582,6 +613,14 @@ export interface BaseCollectionConfig<
    */
   onDelete?: DeleteMutationFn<T, TKey, TUtils, TReturn>
 
+  /**
+   * Specifies how to compare data in the collection.
+   * This should be configured to match data ordering on the backend.
+   * E.g., when using the Electric DB collection these options
+   *       should match the database's collation settings.
+   */
+  defaultStringCollation?: StringCollationConfig
+
   utils?: TUtils
 }
 

package/src/utils/comparison.ts

@@ -112,11 +112,80 @@ export const defaultComparator = makeComparator({
 })
 
 /**
- * Normalize a value for comparison
+ * Compare two Uint8Arrays for content equality
+ */
+function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.byteLength !== b.byteLength) {
+    return false
+  }
+  for (let i = 0; i < a.byteLength; i++) {
+    if (a[i] !== b[i]) {
+      return false
+    }
+  }
+  return true
+}
+
+/**
+ * Threshold for normalizing Uint8Arrays to string representations.
+ * Arrays larger than this will use reference equality to avoid memory overhead.
+ * 128 bytes is enough for common ID formats (ULIDs are 16 bytes, UUIDs are 16 bytes)
+ * while avoiding excessive string allocation for large binary data.
+ */
+const UINT8ARRAY_NORMALIZE_THRESHOLD = 128
+
+/**
+ * Normalize a value for comparison and Map key usage
+ * Converts values that can't be directly compared or used as Map keys
+ * into comparable primitive representations
  */
 export function normalizeValue(value: any): any {
   if (value instanceof Date) {
     return value.getTime()
   }
+
+  // Normalize Uint8Arrays/Buffers to a string representation for Map key usage
+  // This enables content-based equality for binary data like ULIDs
+  const isUint8Array =
+    (typeof Buffer !== `undefined` && value instanceof Buffer) ||
+    value instanceof Uint8Array
+
+  if (isUint8Array) {
+    // Only normalize small arrays to avoid memory overhead for large binary data
+    if (value.byteLength <= UINT8ARRAY_NORMALIZE_THRESHOLD) {
+      // Convert to a string representation that can be used as a Map key
+      // Use a special prefix to avoid collisions with user strings
+      return `__u8__${Array.from(value).join(`,`)}`
+    }
+    // For large arrays, fall back to reference equality
+    // Users working with large binary data should use a derived key if needed
+  }
+
   return value
 }
+
+/**
+ * Compare two values for equality, with special handling for Uint8Arrays and Buffers
+ */
+export function areValuesEqual(a: any, b: any): boolean {
+  // Fast path for reference equality
+  if (a === b) {
+    return true
+  }
+
+  // Check for Uint8Array/Buffer comparison
+  const aIsUint8Array =
+    (typeof Buffer !== `undefined` && a instanceof Buffer) ||
+    a instanceof Uint8Array
+  const bIsUint8Array =
+    (typeof Buffer !== `undefined` && b instanceof Buffer) ||
+    b instanceof Uint8Array
+
+  // If both are Uint8Arrays, compare by content
+  if (aIsUint8Array && bIsUint8Array) {
+    return areUint8ArraysEqual(a, b)
+  }
+
+  // Different types or not Uint8Arrays
+  return false
+}