@tanstack/db 0.4.7 → 0.4.8

package/src/collection/state.ts

@@ -12,11 +12,18 @@ import type { CollectionLifecycleManager } from "./lifecycle"
 import type { CollectionChangesManager } from "./changes"
 import type { CollectionIndexesManager } from "./indexes"
 
-interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
+interface PendingSyncedTransaction<
+  T extends object = Record<string, unknown>,
+  TKey extends string | number = string | number,
+> {
   committed: boolean
   operations: Array<OptimisticChangeMessage<T>>
   truncate?: boolean
   deletedKeys: Set<string | number>
+  optimisticSnapshot?: {
+    upserts: Map<TKey, T>
+    deletes: Set<TKey>
+  }
 }
 
 export class CollectionStateManager<
@@ -33,8 +40,9 @@ export class CollectionStateManager<
 
   // Core state - make public for testing
   public transactions: SortedMap<string, Transaction<any>>
-  public pendingSyncedTransactions: Array<PendingSyncedTransaction<TOutput>> =
-    []
+  public pendingSyncedTransactions: Array<
+    PendingSyncedTransaction<TOutput, TKey>
+  > = []
   public syncedData: Map<TKey, TOutput> | SortedMap<TKey, TOutput>
   public syncedMetadata = new Map<TKey, unknown>()
 
@@ -442,10 +450,10 @@ export class CollectionStateManager<
       },
       {
         committedSyncedTransactions: [] as Array<
-          PendingSyncedTransaction<TOutput>
+          PendingSyncedTransaction<TOutput, TKey>
         >,
         uncommittedSyncedTransactions: [] as Array<
-          PendingSyncedTransaction<TOutput>
+          PendingSyncedTransaction<TOutput, TKey>
         >,
         hasTruncateSync: false,
       }
@@ -455,6 +463,12 @@ export class CollectionStateManager<
       // Set flag to prevent redundant optimistic state recalculations
       this.isCommittingSyncTransactions = true
 
+      // Get the optimistic snapshot from the truncate transaction (captured when truncate() was called)
+      const truncateOptimisticSnapshot = hasTruncateSync
+        ? committedSyncedTransactions.find((t) => t.truncate)
+            ?.optimisticSnapshot
+        : null
+
       // First collect all keys that will be affected by sync operations
       const changedKeys = new Set<TKey>()
       for (const transaction of committedSyncedTransactions) {
@@ -484,13 +498,19 @@ export class CollectionStateManager<
         // Handle truncate operations first
         if (transaction.truncate) {
           // TRUNCATE PHASE
-          // 1) Emit a delete for every currently-synced key so downstream listeners/indexes
+          // 1) Emit a delete for every visible key (synced + optimistic) so downstream listeners/indexes
           //    observe a clear-before-rebuild. We intentionally skip keys already in
           //    optimisticDeletes because their delete was previously emitted by the user.
-          for (const key of this.syncedData.keys()) {
-            if (this.optimisticDeletes.has(key)) continue
+          // Use the snapshot to ensure we emit deletes for all items that existed at truncate start.
+          const visibleKeys = new Set([
+            ...this.syncedData.keys(),
+            ...(truncateOptimisticSnapshot?.upserts.keys() || []),
+          ])
+          for (const key of visibleKeys) {
+            if (truncateOptimisticSnapshot?.deletes.has(key)) continue
             const previousValue =
-              this.optimisticUpserts.get(key) || this.syncedData.get(key)
+              truncateOptimisticSnapshot?.upserts.get(key) ||
+              this.syncedData.get(key)
             if (previousValue !== undefined) {
               events.push({ type: `delete`, key, value: previousValue })
             }
@@ -574,41 +594,14 @@ export class CollectionStateManager<
           }
         }
 
-        // Build re-apply sets from ACTIVE optimistic transactions against the new synced base
-        // We do not copy maps; we compute intent directly from transactions to avoid drift.
-        const reapplyUpserts = new Map<TKey, TOutput>()
-        const reapplyDeletes = new Set<TKey>()
-
-        for (const tx of this.transactions.values()) {
-          if ([`completed`, `failed`].includes(tx.state)) continue
-          for (const mutation of tx.mutations) {
-            if (
-              !this.isThisCollection(mutation.collection) ||
-              !mutation.optimistic
-            )
-              continue
-            const key = mutation.key as TKey
-            switch (mutation.type) {
-              case `insert`:
-                reapplyUpserts.set(key, mutation.modified as TOutput)
-                reapplyDeletes.delete(key)
-                break
-              case `update`: {
-                const base = this.syncedData.get(key)
-                const next = base
-                  ? (Object.assign({}, base, mutation.changes) as TOutput)
-                  : (mutation.modified as TOutput)
-                reapplyUpserts.set(key, next)
-                reapplyDeletes.delete(key)
-                break
-              }
-              case `delete`:
-                reapplyUpserts.delete(key)
-                reapplyDeletes.add(key)
-                break
-            }
-          }
-        }
+        // Build re-apply sets from the snapshot taken at the start of this function.
+        // This prevents losing optimistic state if transactions complete during truncate processing.
+        const reapplyUpserts = new Map<TKey, TOutput>(
+          truncateOptimisticSnapshot!.upserts
+        )
+        const reapplyDeletes = new Set<TKey>(
+          truncateOptimisticSnapshot!.deletes
+        )
 
         // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete.
         // If the server also inserted/updated the same key in this batch, override that value
@@ -660,6 +653,20 @@ export class CollectionStateManager<
 
       // Reset flag and recompute optimistic state for any remaining active transactions
       this.isCommittingSyncTransactions = false
+
+      // If we had a truncate, restore the preserved optimistic state from the snapshot
+      // This includes items from transactions that may have completed during processing
+      if (hasTruncateSync && truncateOptimisticSnapshot) {
+        for (const [key, value] of truncateOptimisticSnapshot.upserts) {
+          this.optimisticUpserts.set(key, value)
+        }
+        for (const key of truncateOptimisticSnapshot.deletes) {
+          this.optimisticDeletes.add(key)
+        }
+      }
+
+      // Always overlay any still-active optimistic transactions so mutations that started
+      // after the truncate snapshot are preserved.
       for (const transaction of this.transactions.values()) {
         if (![`completed`, `failed`].includes(transaction.state)) {
           for (const mutation of transaction.mutations) {
@@ -785,12 +792,9 @@ export class CollectionStateManager<
         this.recentlySyncedKeys.clear()
       })
 
-      // Call any registered one-time commit listeners
+      // Mark that we've received the first commit (for tracking purposes)
       if (!this.hasReceivedFirstCommit) {
         this.hasReceivedFirstCommit = true
-        const callbacks = [...this.lifecycle.onFirstReadyCallbacks]
-        this.lifecycle.onFirstReadyCallbacks = []
-        callbacks.forEach((callback) => callback())
       }
     }
   }

package/src/collection/sync.ts

@@ -148,12 +148,6 @@ export class CollectionSyncManager<
 
             pendingTransaction.committed = true
 
-            // Update status to initialCommit when transitioning from loading
-            // This indicates we're in the process of committing the first transaction
-            if (this.lifecycle.status === `loading`) {
-              this.lifecycle.setStatus(`initialCommit`)
-            }
-
             this.state.commitPendingTransactions()
           },
           markReady: () => {
@@ -181,6 +175,13 @@ export class CollectionSyncManager<
             // - Subsequent synced ops applied on the fresh base
             // - Finally, optimistic mutations re-applied on top (single batch)
             pendingTransaction.truncate = true
+
+            // Capture optimistic state NOW to preserve it even if transactions complete
+            // before this truncate transaction is committed
+            pendingTransaction.optimisticSnapshot = {
+              upserts: new Map(this.state.optimisticUpserts),
+              deletes: new Set(this.state.optimisticDeletes),
+            }
           },
         })
       )

package/src/indexes/auto-index.ts

@@ -14,14 +14,6 @@ function shouldAutoIndex(collection: CollectionImpl<any, any, any, any, any>) {
     return false
   }
 
-  // Don't auto-index during sync operations
-  if (
-    collection.status === `loading` ||
-    collection.status === `initialCommit`
-  ) {
-    return false
-  }
-
   return true
 }
 

package/src/query/live/collection-config-builder.ts

@@ -21,10 +21,15 @@ import type {
   LiveQueryCollectionConfig,
   SyncState,
 } from "./types.js"
+import type { AllCollectionEvents } from "../../collection/events.js"
 
 // Global counter for auto-generated collection IDs
 let liveQueryCollectionCounter = 0
 
+type SyncMethods<TResult extends object> = Parameters<
+  SyncConfig<TResult>[`sync`]
+>[0]
+
 export class CollectionConfigBuilder<
   TContext extends Context,
   TResult extends object = GetResult<TContext>,
@@ -44,6 +49,12 @@ export class CollectionConfigBuilder<
 
   private isGraphRunning = false
 
+  // Error state tracking
+  private isInErrorState = false
+
+  // Reference to the live query collection for error state transitions
+  private liveQueryCollection?: Collection<TResult, any, any>
+
   private graphCache: D2 | undefined
   private inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined
   private pipelineCache: ResultStream | undefined
@@ -101,12 +112,12 @@ export class CollectionConfigBuilder<
   // This gives the callback a chance to load more data if needed,
   // that's used to optimize orderBy operators that set a limit,
   // in order to load some more data if we still don't have enough rows after the pipeline has run.
-  // That can happend because even though we load N rows, the pipeline might filter some of these rows out
+  // That can happen because even though we load N rows, the pipeline might filter some of these rows out
   // causing the orderBy operator to receive less than N rows or even no rows at all.
   // So this callback would notice that it doesn't have enough rows and load some more.
-  // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.
+  // The callback returns a boolean, when it's true it's done loading data.
   maybeRunGraph(
-    config: Parameters<SyncConfig<TResult>[`sync`]>[0],
+    config: SyncMethods<TResult>,
     syncState: FullSyncState,
     callback?: () => boolean
   ) {
@@ -120,13 +131,15 @@ export class CollectionConfigBuilder<
     this.isGraphRunning = true
 
     try {
-      const { begin, commit, markReady } = config
+      const { begin, commit } = config
+
+      // Don't run if the live query is in an error state
+      if (this.isInErrorState) {
+        return
+      }
 
-      // We only run the graph if all the collections are ready
-      if (
-        this.allCollectionsReadyOrInitialCommit() &&
-        syncState.subscribedToAllCollections
-      ) {
+      // Always run the graph if subscribed (eager execution)
+      if (syncState.subscribedToAllCollections) {
         while (syncState.graph.pendingWork()) {
           syncState.graph.run()
           callback?.()
@@ -137,10 +150,9 @@ export class CollectionConfigBuilder<
         if (syncState.messagesCount === 0) {
           begin()
           commit()
-        }
-        // Mark the collection as ready after the first successful run
-        if (this.allCollectionsReady()) {
-          markReady()
+          // After initial commit, check if we should mark ready
+          // (in case all sources were already ready before we subscribed)
+          this.updateLiveQueryStatus(config)
         }
       }
     } finally {
@@ -155,7 +167,10 @@ export class CollectionConfigBuilder<
     }
   }
 
-  private syncFn(config: Parameters<SyncConfig<TResult>[`sync`]>[0]) {
+  private syncFn(config: SyncMethods<TResult>) {
+    // Store reference to the live query collection for error state transitions
+    this.liveQueryCollection = config.collection
+
     const syncState: SyncState = {
       messagesCount: 0,
       subscribedToAllCollections: false,
@@ -233,7 +248,7 @@ export class CollectionConfigBuilder<
   }
 
   private extendPipelineWithChangeProcessing(
-    config: Parameters<SyncConfig<TResult>[`sync`]>[0],
+    config: SyncMethods<TResult>,
     syncState: SyncState
   ): FullSyncState {
     const { begin, commit } = config
@@ -266,7 +281,7 @@ export class CollectionConfigBuilder<
   }
 
   private applyChanges(
-    config: Parameters<SyncConfig<TResult>[`sync`]>[0],
+    config: SyncMethods<TResult>,
     changes: {
       deletes: number
       inserts: number
@@ -317,21 +332,76 @@ export class CollectionConfigBuilder<
     }
   }
 
+  /**
+   * Handle status changes from source collections
+   */
+  private handleSourceStatusChange(
+    config: SyncMethods<TResult>,
+    collectionId: string,
+    event: AllCollectionEvents[`status:change`]
+  ) {
+    const { status } = event
+
+    // Handle error state - any source collection in error puts live query in error
+    if (status === `error`) {
+      this.transitionToError(
+        `Source collection '${collectionId}' entered error state`
+      )
+      return
+    }
+
+    // Handle manual cleanup - this should not happen due to GC prevention,
+    // but could happen if user manually calls cleanup()
+    if (status === `cleaned-up`) {
+      this.transitionToError(
+        `Source collection '${collectionId}' was manually cleaned up while live query '${this.id}' depends on it. ` +
+          `Live queries prevent automatic GC, so this was likely a manual cleanup() call.`
+      )
+      return
+    }
+
+    // Update ready status based on all source collections
+    this.updateLiveQueryStatus(config)
+  }
+
+  /**
+   * Update the live query status based on source collection statuses
+   */
+  private updateLiveQueryStatus(config: SyncMethods<TResult>) {
+    const { markReady } = config
+
+    // Don't update status if already in error
+    if (this.isInErrorState) {
+      return
+    }
+
+    // Mark ready when all source collections are ready
+    if (this.allCollectionsReady()) {
+      markReady()
+    }
+  }
+
+  /**
+   * Transition the live query to error state
+   */
+  private transitionToError(message: string) {
+    this.isInErrorState = true
+
+    // Log error to console for debugging
+    console.error(`[Live Query Error] ${message}`)
+
+    // Transition live query collection to error state
+    this.liveQueryCollection?._lifecycle.setStatus(`error`)
+  }
+
   private allCollectionsReady() {
     return Object.values(this.collections).every((collection) =>
       collection.isReady()
     )
   }
 
-  private allCollectionsReadyOrInitialCommit() {
-    return Object.values(this.collections).every(
-      (collection) =>
-        collection.status === `ready` || collection.status === `initialCommit`
-    )
-  }
-
   private subscribeToAllCollections(
-    config: Parameters<SyncConfig<TResult>[`sync`]>[0],
+    config: SyncMethods<TResult>,
     syncState: FullSyncState
   ) {
     const loaders = Object.entries(this.collections).map(
@@ -347,6 +417,12 @@ export class CollectionConfigBuilder<
         const subscription = collectionSubscriber.subscribe()
         this.subscriptions[collectionId] = subscription
 
+        // Subscribe to status changes for status flow
+        const statusUnsubscribe = collection.on(`status:change`, (event) => {
+          this.handleSourceStatusChange(config, collectionId, event)
+        })
+        syncState.unsubscribeCallbacks.add(statusUnsubscribe)
+
         const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
           collectionSubscriber,
           subscription
@@ -364,6 +440,9 @@ export class CollectionConfigBuilder<
     // Mark the collections as subscribed in the sync state
     syncState.subscribedToAllCollections = true
 
+    // Initial status check after all subscriptions are set up
+    this.updateLiveQueryStatus(config)
+
     return loadMoreDataCallback
   }
 }

package/src/query/live/collection-subscriber.ts

@@ -103,9 +103,9 @@ export class CollectionSubscriber<
     // otherwise we end up in an infinite loop trying to load more data
     const dataLoader = sentChanges > 0 ? callback : undefined
 
-    // We need to call `maybeRunGraph` even if there's no data to load
-    // because we need to mark the collection as ready if it's not already
-    // and that's only done in `maybeRunGraph`
+    // Always call maybeRunGraph to process changes eagerly.
+    // The graph will run unless the live query is in an error state.
+    // Status management is handled separately via status:change event listeners.
     this.collectionConfigBuilder.maybeRunGraph(
       this.config,
       this.syncState,

package/src/types.ts

@@ -298,17 +298,15 @@ export type DeleteMutationFn<
  *
  * @example
  * // Status transitions
- * // idle → loading → initialCommit → ready
+ * // idle → loading → ready (when markReady() is called)
  * // Any status can transition to → error or cleaned-up
  */
 export type CollectionStatus =
   /** Collection is created but sync hasn't started yet (when startSync config is false) */
   | `idle`
-  /** Sync has started but hasn't received the first commit yet */
+  /** Sync has started and is loading data */
   | `loading`
-  /** Collection is in the process of committing its first transaction */
-  | `initialCommit`
-  /** Collection has received at least one commit and is ready for use */
+  /** Collection has been explicitly marked ready via markReady() */
   | `ready`
   /** An error occurred during sync initialization */
   | `error`