@tanstack/db 0.5.12 → 0.5.14

package/src/collection/subscription.ts

@@ -52,6 +52,11 @@ export class CollectionSubscription
 {
   private loadedInitialState = false
 
+  // Flag to skip filtering in filterAndFlipChanges.
+  // This is separate from loadedInitialState because we want to allow
+  // requestSnapshot to still work even when filtering is skipped.
+  private skipFiltering = false
+
   // Flag to indicate that we have sent at least 1 snapshot.
   // While `snapshotSent` is false we filter out all changes from subscription to the collection.
   private snapshotSent = false
@@ -79,6 +84,16 @@ export class CollectionSubscription
   private _status: SubscriptionStatus = `ready`
   private pendingLoadSubsetPromises: Set<Promise<void>> = new Set()
 
+  // Cleanup function for truncate event listener
+  private truncateCleanup: (() => void) | undefined
+
+  // Truncate buffering state
+  // When a truncate occurs, we buffer changes until all loadSubset refetches complete
+  // This prevents a flash of missing content between deletes and new inserts
+  private isBufferingForTruncate = false
+  private truncateBuffer: Array<Array<ChangeMessage<any, any>>> = []
+  private pendingTruncateRefetches: Set<Promise<void>> = new Set()
+
   public get status(): SubscriptionStatus {
     return this._status
   }
@@ -111,6 +126,123 @@ export class CollectionSubscription
     this.filteredCallback = options.whereExpression
       ? createFilteredCallback(this.callback, options)
       : this.callback
+
+    // Listen for truncate events to re-request data after must-refetch
+    // When a truncate happens (e.g., from a 409 must-refetch), all collection data is cleared.
+    // We need to re-request all previously loaded subsets to repopulate the data.
+    this.truncateCleanup = this.collection.on(`truncate`, () => {
+      this.handleTruncate()
+    })
+  }
+
+  /**
+   * Handle collection truncate event by resetting state and re-requesting subsets.
+   * This is called when the sync layer receives a must-refetch and clears all data.
+   *
+   * To prevent a flash of missing content, we buffer all changes (deletes from truncate
+   * and inserts from refetch) until all loadSubset promises resolve, then emit them together.
+   */
+  private handleTruncate() {
+    // Copy the loaded subsets before clearing (we'll re-request them)
+    const subsetsToReload = [...this.loadedSubsets]
+
+    // Only buffer if there's an actual loadSubset handler that can do async work.
+    // Without a loadSubset handler, there's nothing to re-request and no reason to buffer.
+    // This prevents unnecessary buffering in eager sync mode or when loadSubset isn't implemented.
+    const hasLoadSubsetHandler = this.collection._sync.syncLoadSubsetFn !== null
+
+    // If there are no subsets to reload OR no loadSubset handler, just reset state
+    if (subsetsToReload.length === 0 || !hasLoadSubsetHandler) {
+      this.snapshotSent = false
+      this.loadedInitialState = false
+      this.limitedSnapshotRowCount = 0
+      this.lastSentKey = undefined
+      this.loadedSubsets = []
+      return
+    }
+
+    // Start buffering BEFORE we receive the delete events from the truncate commit
+    // This ensures we capture both the deletes and subsequent inserts
+    this.isBufferingForTruncate = true
+    this.truncateBuffer = []
+    this.pendingTruncateRefetches.clear()
+
+    // Reset snapshot/pagination tracking state
+    // Note: We don't need to populate sentKeys here because filterAndFlipChanges
+    // will skip the delete filter when isBufferingForTruncate is true
+    this.snapshotSent = false
+    this.loadedInitialState = false
+    this.limitedSnapshotRowCount = 0
+    this.lastSentKey = undefined
+
+    // Clear the loadedSubsets array since we're re-requesting fresh
+    this.loadedSubsets = []
+
+    // Defer the loadSubset calls to a microtask so the truncate commit's delete events
+    // are buffered BEFORE the loadSubset calls potentially trigger nested commits.
+    // This ensures correct event ordering: deletes first, then inserts.
+    queueMicrotask(() => {
+      // Check if we were unsubscribed while waiting
+      if (!this.isBufferingForTruncate) {
+        return
+      }
+
+      // Re-request all previously loaded subsets and track their promises
+      for (const options of subsetsToReload) {
+        const syncResult = this.collection._sync.loadSubset(options)
+
+        // Track this loadSubset call so we can unload it later
+        this.loadedSubsets.push(options)
+        this.trackLoadSubsetPromise(syncResult)
+
+        // Track the promise for buffer flushing
+        if (syncResult instanceof Promise) {
+          this.pendingTruncateRefetches.add(syncResult)
+          syncResult
+            .catch(() => {
+              // Ignore errors - we still want to flush the buffer even if some requests fail
+            })
+            .finally(() => {
+              this.pendingTruncateRefetches.delete(syncResult)
+              this.checkTruncateRefetchComplete()
+            })
+        }
+      }
+
+      // If all loadSubset calls were synchronous (returned true), flush now
+      // At this point, delete events have already been buffered from the truncate commit
+      if (this.pendingTruncateRefetches.size === 0) {
+        this.flushTruncateBuffer()
+      }
+    })
+  }
+
+  /**
+   * Check if all truncate refetch promises have completed and flush buffer if so
+   */
+  private checkTruncateRefetchComplete() {
+    if (
+      this.pendingTruncateRefetches.size === 0 &&
+      this.isBufferingForTruncate
+    ) {
+      this.flushTruncateBuffer()
+    }
+  }
+
+  /**
+   * Flush the truncate buffer, emitting all buffered changes to the callback
+   */
+  private flushTruncateBuffer() {
+    this.isBufferingForTruncate = false
+
+    // Flatten all buffered changes into a single array for atomic emission
+    // This ensures consumers see all truncate changes (deletes + inserts) in one callback
+    const merged = this.truncateBuffer.flat()
+    if (merged.length > 0) {
+      this.filteredCallback(merged)
+    }
+
+    this.truncateBuffer = []
   }
 
   setOrderByIndex(index: IndexInterface<any>) {
@@ -174,7 +306,16 @@ export class CollectionSubscription
 
   emitEvents(changes: Array<ChangeMessage<any, any>>) {
     const newChanges = this.filterAndFlipChanges(changes)
-    this.filteredCallback(newChanges)
+
+    if (this.isBufferingForTruncate) {
+      // Buffer the changes instead of emitting immediately
+      // This prevents a flash of missing content during truncate/refetch
+      if (newChanges.length > 0) {
+        this.truncateBuffer.push(newChanges)
+      }
+    } else {
+      this.filteredCallback(newChanges)
+    }
   }
 
   /**
@@ -244,6 +385,13 @@ export class CollectionSubscription
       (change) => !this.sentKeys.has(change.key),
     )
 
+    // Add keys to sentKeys BEFORE calling callback to prevent race condition.
+    // If a change event arrives while the callback is executing, it will see
+    // the keys already in sentKeys and filter out duplicates correctly.
+    for (const change of filteredSnapshot) {
+      this.sentKeys.add(change.key)
+    }
+
     this.snapshotSent = true
     this.callback(filteredSnapshot)
     return true
@@ -367,6 +515,13 @@ export class CollectionSubscription
     // Use the current count as the offset for this load
     const currentOffset = this.limitedSnapshotRowCount
 
+    // Add keys to sentKeys BEFORE calling callback to prevent race condition.
+    // If a change event arrives while the callback is executing, it will see
+    // the keys already in sentKeys and filter out duplicates correctly.
+    for (const change of changes) {
+      this.sentKeys.add(change.key)
+    }
+
     this.callback(changes)
 
     // Update the row count and last key after sending (for next call's offset/cursor)
@@ -441,25 +596,52 @@ export class CollectionSubscription
    * Filters and flips changes for keys that have not been sent yet.
    * Deletes are filtered out for keys that have not been sent yet.
    * Updates are flipped into inserts for keys that have not been sent yet.
+   * Duplicate inserts are filtered out to prevent D2 multiplicity > 1.
    */
   private filterAndFlipChanges(changes: Array<ChangeMessage<any, any>>) {
-    if (this.loadedInitialState) {
-      // We loaded the entire initial state
+    if (this.loadedInitialState || this.skipFiltering) {
+      // We loaded the entire initial state or filtering is explicitly skipped
       // so no need to filter or flip changes
       return changes
     }
 
+    // When buffering for truncate, we need all changes (including deletes) to pass through.
+    // This is important because:
+    // 1. If loadedInitialState was previously true, sentKeys will be empty
+    //    (trackSentKeys early-returns when loadedInitialState is true)
+    // 2. The truncate deletes are for keys that WERE sent to the subscriber
+    // 3. We're collecting all changes atomically, so filtering doesn't make sense
+    const skipDeleteFilter = this.isBufferingForTruncate
+
     const newChanges = []
     for (const change of changes) {
       let newChange = change
-      if (!this.sentKeys.has(change.key)) {
+      const keyInSentKeys = this.sentKeys.has(change.key)
+
+      if (!keyInSentKeys) {
         if (change.type === `update`) {
           newChange = { ...change, type: `insert`, previousValue: undefined }
         } else if (change.type === `delete`) {
-          // filter out deletes for keys that have not been sent
-          continue
+          // Filter out deletes for keys that have not been sent,
+          // UNLESS we're buffering for truncate (where all deletes should pass through)
+          if (!skipDeleteFilter) {
+            continue
+          }
         }
         this.sentKeys.add(change.key)
+      } else {
+        // Key was already sent - handle based on change type
+        if (change.type === `insert`) {
+          // Filter out duplicate inserts - the key was already inserted.
+          // This prevents D2 multiplicity from going above 1, which would
+          // cause deletes to not properly remove items (multiplicity would
+          // go from 2 to 1 instead of 1 to 0).
+          continue
+        } else if (change.type === `delete`) {
+          // Remove from sentKeys so future inserts for this key are allowed
+          // (e.g., after truncate + reinsert)
+          this.sentKeys.delete(change.key)
+        }
       }
       newChanges.push(newChange)
     }
@@ -467,18 +649,42 @@ export class CollectionSubscription
   }
 
   private trackSentKeys(changes: Array<ChangeMessage<any, string | number>>) {
-    if (this.loadedInitialState) {
-      // No need to track sent keys if we loaded the entire state.
-      // Since we sent everything, all keys must have been observed.
+    if (this.loadedInitialState || this.skipFiltering) {
+      // No need to track sent keys if we loaded the entire state or filtering is skipped.
+      // Since filtering won't be applied, all keys are effectively "observed".
       return
     }
 
     for (const change of changes) {
-      this.sentKeys.add(change.key)
+      if (change.type === `delete`) {
+        // Remove deleted keys from sentKeys so future re-inserts are allowed
+        this.sentKeys.delete(change.key)
+      } else {
+        // For inserts and updates, track the key as sent
+        this.sentKeys.add(change.key)
+      }
     }
   }
 
+  /**
+   * Mark that the subscription should not filter any changes.
+   * This is used when includeInitialState is explicitly set to false,
+   * meaning the caller doesn't want initial state but does want ALL future changes.
+   */
+  markAllStateAsSeen() {
+    this.skipFiltering = true
+  }
+
   unsubscribe() {
+    // Clean up truncate event listener
+    this.truncateCleanup?.()
+    this.truncateCleanup = undefined
+
+    // Clean up truncate buffer state
+    this.isBufferingForTruncate = false
+    this.truncateBuffer = []
+    this.pendingTruncateRefetches.clear()
+
     // Unload all subsets that this subscription loaded
     // We pass the exact same LoadSubsetOptions we used for loadSubset
     for (const options of this.loadedSubsets) {

package/src/collection/sync.ts

@@ -12,10 +12,11 @@ import { deepEquals } from '../utils'
 import { LIVE_QUERY_INTERNAL } from '../query/live/internal.js'
 import type { StandardSchemaV1 } from '@standard-schema/spec'
 import type {
-  ChangeMessage,
+  ChangeMessageOrDeleteKeyMessage,
   CleanupFn,
   CollectionConfig,
   LoadSubsetOptions,
+  OptimisticChangeMessage,
   SyncConfigRes,
 } from '../types'
 import type { CollectionImpl } from './index.js'
@@ -94,7 +95,12 @@ export class CollectionSyncManager<
               deletedKeys: new Set(),
             })
           },
-          write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
+          write: (
+            messageWithOptionalKey: ChangeMessageOrDeleteKeyMessage<
+              TOutput,
+              TKey
+            >,
+          ) => {
             const pendingTransaction =
               this.state.pendingSyncedTransactions[
                 this.state.pendingSyncedTransactions.length - 1
@@ -105,12 +111,18 @@ export class CollectionSyncManager<
             if (pendingTransaction.committed) {
               throw new SyncTransactionAlreadyCommittedWriteError()
             }
-            const key = this.config.getKey(messageWithoutKey.value)
 
-            let messageType = messageWithoutKey.type
+            let key: TKey | undefined = undefined
+            if (`key` in messageWithOptionalKey) {
+              key = messageWithOptionalKey.key
+            } else {
+              key = this.config.getKey(messageWithOptionalKey.value)
+            }
+
+            let messageType = messageWithOptionalKey.type
 
             // Check if an item with this key already exists when inserting
-            if (messageWithoutKey.type === `insert`) {
+            if (messageWithOptionalKey.type === `insert`) {
               const insertingIntoExistingSynced = this.state.syncedData.has(key)
               const hasPendingDeleteForKey =
                 pendingTransaction.deletedKeys.has(key)
@@ -124,7 +136,7 @@ export class CollectionSyncManager<
                 const existingValue = this.state.syncedData.get(key)
                 if (
                   existingValue !== undefined &&
-                  deepEquals(existingValue, messageWithoutKey.value)
+                  deepEquals(existingValue, messageWithOptionalKey.value)
                 ) {
                   // The "insert" is an echo of a value we already have locally.
                   // Treat it as an update so we preserve optimistic intent without
@@ -142,11 +154,11 @@ export class CollectionSyncManager<
               }
             }
 
-            const message: ChangeMessage<TOutput> = {
-              ...messageWithoutKey,
+            const message = {
+              ...messageWithOptionalKey,
               type: messageType,
               key,
-            }
+            } as OptimisticChangeMessage<TOutput, TKey>
             pendingTransaction.operations.push(message)
 
             if (messageType === `delete`) {

package/src/types.ts

@@ -328,7 +328,7 @@ export interface SyncConfig<
   sync: (params: {
     collection: Collection<T, TKey, any, any, any>
     begin: () => void
-    write: (message: Omit<ChangeMessage<T>, `key`>) => void
+    write: (message: ChangeMessageOrDeleteKeyMessage<T, TKey>) => void
     commit: () => void
     markReady: () => void
     truncate: () => void
@@ -361,12 +361,28 @@ export interface ChangeMessage<
   metadata?: Record<string, unknown>
 }
 
-export interface OptimisticChangeMessage<
+export type DeleteKeyMessage<TKey extends string | number = string | number> =
+  Omit<ChangeMessage<any, TKey>, `value` | `previousValue` | `type`> & {
+    type: `delete`
+  }
+
+export type ChangeMessageOrDeleteKeyMessage<
   T extends object = Record<string, unknown>,
-> extends ChangeMessage<T> {
-  // Is this change message part of an active transaction. Only applies to optimistic changes.
-  isActive?: boolean
-}
+  TKey extends string | number = string | number,
+> = Omit<ChangeMessage<T>, `key`> | DeleteKeyMessage<TKey>
+
+export type OptimisticChangeMessage<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> =
+  | (ChangeMessage<T> & {
+      // Is this change message part of an active transaction. Only applies to optimistic changes.
+      isActive?: boolean
+    })
+  | (DeleteKeyMessage<TKey> & {
+      // Is this change message part of an active transaction. Only applies to optimistic changes.
+      isActive?: boolean
+    })
 
 /**
  * The Standard Schema interface.
@@ -894,3 +910,6 @@ export type WritableDeep<T> = T extends BuiltIns
           : T extends object
             ? WritableObjectDeep<T>
             : unknown
+
+export type MakeOptional<T, K extends keyof T> = Omit<T, K> &
+  Partial<Pick<T, K>>