@tanstack/db 0.0.12 → 0.0.14

package/dist/cjs/collection.cjs

@@ -32,7 +32,7 @@ class CollectionImpl {
    * @throws Error if sync config is missing
    */
   constructor(config) {
-    this.syncedData = /* @__PURE__ */ new Map();
+    this.pendingSyncedTransactions = [];
     this.syncedMetadata = /* @__PURE__ */ new Map();
     this.derivedUpserts = /* @__PURE__ */ new Map();
     this.derivedDeletes = /* @__PURE__ */ new Set();
@@ -40,21 +40,48 @@ class CollectionImpl {
     this.changeListeners = /* @__PURE__ */ new Set();
     this.changeKeyListeners = /* @__PURE__ */ new Map();
     this.utils = {};
-    this.pendingSyncedTransactions = [];
     this.syncedKeys = /* @__PURE__ */ new Set();
+    this.preSyncVisibleState = /* @__PURE__ */ new Map();
+    this.recentlySyncedKeys = /* @__PURE__ */ new Set();
     this.hasReceivedFirstCommit = false;
+    this.isCommittingSyncTransactions = false;
     this.onFirstCommitCallbacks = [];
+    this._status = `idle`;
+    this.activeSubscribersCount = 0;
+    this.gcTimeoutId = null;
+    this.preloadPromise = null;
+    this.syncCleanupFn = null;
     this.id = ``;
     this.commitPendingTransactions = () => {
-      if (!Array.from(this.transactions.values()).some(
-        ({ state }) => state === `persisting`
-      )) {
+      let hasPersistingTransaction = false;
+      for (const transaction of this.transactions.values()) {
+        if (transaction.state === `persisting`) {
+          hasPersistingTransaction = true;
+          break;
+        }
+      }
+      if (!hasPersistingTransaction) {
+        this.isCommittingSyncTransactions = true;
         const changedKeys = /* @__PURE__ */ new Set();
+        for (const transaction of this.pendingSyncedTransactions) {
+          for (const operation of transaction.operations) {
+            changedKeys.add(operation.key);
+          }
+        }
+        let currentVisibleState = this.preSyncVisibleState;
+        if (currentVisibleState.size === 0) {
+          currentVisibleState = /* @__PURE__ */ new Map();
+          for (const key of changedKeys) {
+            const currentValue = this.get(key);
+            if (currentValue !== void 0) {
+              currentVisibleState.set(key, currentValue);
+            }
+          }
+        }
         const events = [];
         for (const transaction of this.pendingSyncedTransactions) {
           for (const operation of transaction.operations) {
             const key = operation.key;
-            changedKeys.add(key);
             this.syncedKeys.add(key);
             switch (operation.type) {
               case `insert`:
@@ -74,17 +101,9 @@ class CollectionImpl {
                 this.syncedMetadata.delete(key);
                 break;
             }
-            const previousValue = this.syncedData.get(key);
             switch (operation.type) {
               case `insert`:
                 this.syncedData.set(key, operation.value);
-                if (!this.derivedDeletes.has(key) && !this.derivedUpserts.has(key)) {
-                  events.push({
-                    type: `insert`,
-                    key,
-                    value: operation.value
-                  });
-                }
                 break;
               case `update`: {
                 const updatedValue = Object.assign(
@@ -93,34 +112,84 @@ class CollectionImpl {
                   operation.value
                 );
                 this.syncedData.set(key, updatedValue);
-                if (!this.derivedDeletes.has(key) && !this.derivedUpserts.has(key)) {
-                  events.push({
-                    type: `update`,
-                    key,
-                    value: updatedValue,
-                    previousValue
-                  });
-                }
                 break;
               }
               case `delete`:
                 this.syncedData.delete(key);
-                if (!this.derivedDeletes.has(key) && !this.derivedUpserts.has(key)) {
-                  if (previousValue) {
-                    events.push({
-                      type: `delete`,
-                      key,
-                      value: previousValue
-                    });
-                  }
-                }
                 break;
             }
           }
         }
+        this.derivedUpserts.clear();
+        this.derivedDeletes.clear();
+        this.isCommittingSyncTransactions = false;
+        for (const transaction of this.transactions.values()) {
+          if (![`completed`, `failed`].includes(transaction.state)) {
+            for (const mutation of transaction.mutations) {
+              if (mutation.collection === this) {
+                switch (mutation.type) {
+                  case `insert`:
+                  case `update`:
+                    this.derivedUpserts.set(mutation.key, mutation.modified);
+                    this.derivedDeletes.delete(mutation.key);
+                    break;
+                  case `delete`:
+                    this.derivedUpserts.delete(mutation.key);
+                    this.derivedDeletes.add(mutation.key);
+                    break;
+                }
+              }
+            }
+          }
+        }
+        const completedOptimisticOps = /* @__PURE__ */ new Map();
+        for (const transaction of this.transactions.values()) {
+          if (transaction.state === `completed`) {
+            for (const mutation of transaction.mutations) {
+              if (mutation.collection === this && changedKeys.has(mutation.key)) {
+                completedOptimisticOps.set(mutation.key, {
+                  type: mutation.type,
+                  value: mutation.modified
+                });
+              }
+            }
+          }
+        }
+        for (const key of changedKeys) {
+          const previousVisibleValue = currentVisibleState.get(key);
+          const newVisibleValue = this.get(key);
+          const completedOp = completedOptimisticOps.get(key);
+          const isRedundantSync = completedOp && newVisibleValue !== void 0 && this.deepEqual(completedOp.value, newVisibleValue);
+          if (!isRedundantSync) {
+            if (previousVisibleValue === void 0 && newVisibleValue !== void 0) {
+              events.push({
+                type: `insert`,
+                key,
+                value: newVisibleValue
+              });
+            } else if (previousVisibleValue !== void 0 && newVisibleValue === void 0) {
+              events.push({
+                type: `delete`,
+                key,
+                value: previousVisibleValue
+              });
+            } else if (previousVisibleValue !== void 0 && newVisibleValue !== void 0 && !this.deepEqual(previousVisibleValue, newVisibleValue)) {
+              events.push({
+                type: `update`,
+                key,
+                value: newVisibleValue,
+                previousValue: previousVisibleValue
+              });
+            }
+          }
+        }
         this._size = this.calculateSize();
         this.emitEvents(events);
         this.pendingSyncedTransactions = [];
+        this.preSyncVisibleState.clear();
+        Promise.resolve().then(() => {
+          this.recentlySyncedKeys.clear();
+        });
         if (!this.hasReceivedFirstCommit) {
           this.hasReceivedFirstCommit = true;
           const callbacks = [...this.onFirstCommitCallbacks];
@@ -130,6 +199,7 @@ class CollectionImpl {
       }
     };
     this.insert = (data, config2) => {
+      this.validateCollectionUsable(`insert`);
       const ambientTransaction = transactions.getActiveTransaction();
       if (!ambientTransaction && !this.config.onInsert) {
         throw new Error(
@@ -181,6 +251,7 @@ class CollectionImpl {
       }
     };
     this.delete = (keys, config2) => {
+      this.validateCollectionUsable(`delete`);
       const ambientTransaction = transactions.getActiveTransaction();
       if (!ambientTransaction && !this.config.onDelete) {
         throw new Error(
@@ -248,87 +319,305 @@ class CollectionImpl {
       (a, b) => a.createdAt.getTime() - b.createdAt.getTime()
     );
     this.config = config;
-    config.sync.sync({
-      collection: this,
-      begin: () => {
-        this.pendingSyncedTransactions.push({
-          committed: false,
-          operations: []
-        });
-      },
-      write: (messageWithoutKey) => {
-        const pendingTransaction = this.pendingSyncedTransactions[this.pendingSyncedTransactions.length - 1];
-        if (!pendingTransaction) {
-          throw new Error(`No pending sync transaction to write to`);
-        }
-        if (pendingTransaction.committed) {
-          throw new Error(
-            `The pending sync transaction is already committed, you can't still write to it.`
-          );
-        }
-        const key = this.getKeyFromItem(messageWithoutKey.value);
-        if (messageWithoutKey.type === `insert`) {
-          if (this.syncedData.has(key) && !pendingTransaction.operations.some(
-            (op) => op.key === key && op.type === `delete`
-          )) {
+    collectionsStore.set(this.id, this);
+    if (this.config.compare) {
+      this.syncedData = new SortedMap.SortedMap(this.config.compare);
+    } else {
+      this.syncedData = /* @__PURE__ */ new Map();
+    }
+    if (config.startSync === true) {
+      this.startSync();
+    }
+  }
+  /**
+   * Register a callback to be executed on the next commit
+   * Useful for preloading collections
+   * @param callback Function to call after the next commit
+   */
+  onFirstCommit(callback) {
+    this.onFirstCommitCallbacks.push(callback);
+  }
+  /**
+   * Gets the current status of the collection
+   */
+  get status() {
+    return this._status;
+  }
+  /**
+   * Validates that the collection is in a usable state for data operations
+   * @private
+   */
+  validateCollectionUsable(operation) {
+    switch (this._status) {
+      case `error`:
+        throw new Error(
+          `Cannot perform ${operation} on collection "${this.id}" - collection is in error state. Try calling cleanup() and restarting the collection.`
+        );
+      case `cleaned-up`:
+        throw new Error(
+          `Cannot perform ${operation} on collection "${this.id}" - collection has been cleaned up. The collection will automatically restart on next access.`
+        );
+    }
+  }
+  /**
+   * Validates state transitions to prevent invalid status changes
+   * @private
+   */
+  validateStatusTransition(from, to) {
+    if (from === to) {
+      return;
+    }
+    const validTransitions = {
+      idle: [`loading`, `error`, `cleaned-up`],
+      loading: [`ready`, `error`, `cleaned-up`],
+      ready: [`cleaned-up`, `error`],
+      error: [`cleaned-up`, `idle`],
+      "cleaned-up": [`loading`, `error`]
+    };
+    if (!validTransitions[from].includes(to)) {
+      throw new Error(
+        `Invalid collection status transition from "${from}" to "${to}" for collection "${this.id}"`
+      );
+    }
+  }
+  /**
+   * Safely update the collection status with validation
+   * @private
+   */
+  setStatus(newStatus) {
+    this.validateStatusTransition(this._status, newStatus);
+    this._status = newStatus;
+  }
+  /**
+   * Start sync immediately - internal method for compiled queries
+   * This bypasses lazy loading for special cases like live query results
+   */
+  startSyncImmediate() {
+    this.startSync();
+  }
+  /**
+   * Start the sync process for this collection
+   * This is called when the collection is first accessed or preloaded
+   */
+  startSync() {
+    if (this._status !== `idle` && this._status !== `cleaned-up`) {
+      return;
+    }
+    this.setStatus(`loading`);
+    try {
+      const cleanupFn = this.config.sync.sync({
+        collection: this,
+        begin: () => {
+          this.pendingSyncedTransactions.push({
+            committed: false,
+            operations: []
+          });
+        },
+        write: (messageWithoutKey) => {
+          const pendingTransaction = this.pendingSyncedTransactions[this.pendingSyncedTransactions.length - 1];
+          if (!pendingTransaction) {
+            throw new Error(`No pending sync transaction to write to`);
+          }
+          if (pendingTransaction.committed) {
+            throw new Error(
+              `The pending sync transaction is already committed, you can't still write to it.`
+            );
+          }
+          const key = this.getKeyFromItem(messageWithoutKey.value);
+          if (messageWithoutKey.type === `insert`) {
+            if (this.syncedData.has(key) && !pendingTransaction.operations.some(
+              (op) => op.key === key && op.type === `delete`
+            )) {
+              throw new Error(
+                `Cannot insert document with key "${key}" from sync because it already exists in the collection "${this.id}"`
+              );
+            }
+          }
+          const message = {
+            ...messageWithoutKey,
+            key
+          };
+          pendingTransaction.operations.push(message);
+        },
+        commit: () => {
+          const pendingTransaction = this.pendingSyncedTransactions[this.pendingSyncedTransactions.length - 1];
+          if (!pendingTransaction) {
+            throw new Error(`No pending sync transaction to commit`);
+          }
+          if (pendingTransaction.committed) {
             throw new Error(
-              `Cannot insert document with key "${key}" from sync because it already exists in the collection "${this.id}"`
+              `The pending sync transaction is already committed, you can't commit it again.`
             );
           }
+          pendingTransaction.committed = true;
+          this.commitPendingTransactions();
+          if (this._status === `loading`) {
+            this.setStatus(`ready`);
+          }
         }
-        const message = {
-          ...messageWithoutKey,
-          key
-        };
-        pendingTransaction.operations.push(message);
-      },
-      commit: () => {
-        const pendingTransaction = this.pendingSyncedTransactions[this.pendingSyncedTransactions.length - 1];
-        if (!pendingTransaction) {
-          throw new Error(`No pending sync transaction to commit`);
+      });
+      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null;
+    } catch (error) {
+      this.setStatus(`error`);
+      throw error;
+    }
+  }
+  /**
+   * Preload the collection data by starting sync if not already started
+   * Multiple concurrent calls will share the same promise
+   */
+  preload() {
+    if (this.preloadPromise) {
+      return this.preloadPromise;
+    }
+    this.preloadPromise = new Promise((resolve, reject) => {
+      if (this._status === `ready`) {
+        resolve();
+        return;
+      }
+      if (this._status === `error`) {
+        reject(new Error(`Collection is in error state`));
+        return;
+      }
+      this.onFirstCommit(() => {
+        resolve();
+      });
+      if (this._status === `idle` || this._status === `cleaned-up`) {
+        try {
+          this.startSync();
+        } catch (error) {
+          reject(error);
+          return;
         }
-        if (pendingTransaction.committed) {
+      }
+    });
+    return this.preloadPromise;
+  }
+  /**
+   * Clean up the collection by stopping sync and clearing data
+   * This can be called manually or automatically by garbage collection
+   */
+  async cleanup() {
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId);
+      this.gcTimeoutId = null;
+    }
+    try {
+      if (this.syncCleanupFn) {
+        this.syncCleanupFn();
+        this.syncCleanupFn = null;
+      }
+    } catch (error) {
+      queueMicrotask(() => {
+        if (error instanceof Error) {
+          const wrappedError = new Error(
+            `Collection "${this.id}" sync cleanup function threw an error: ${error.message}`
+          );
+          wrappedError.cause = error;
+          wrappedError.stack = error.stack;
+          throw wrappedError;
+        } else {
           throw new Error(
-            `The pending sync transaction is already committed, you can't commit it again.`
+            `Collection "${this.id}" sync cleanup function threw an error: ${String(error)}`
           );
         }
-        pendingTransaction.committed = true;
-        this.commitPendingTransactions();
+      });
+    }
+    this.syncedData.clear();
+    this.syncedMetadata.clear();
+    this.derivedUpserts.clear();
+    this.derivedDeletes.clear();
+    this._size = 0;
+    this.pendingSyncedTransactions = [];
+    this.syncedKeys.clear();
+    this.hasReceivedFirstCommit = false;
+    this.onFirstCommitCallbacks = [];
+    this.preloadPromise = null;
+    this.setStatus(`cleaned-up`);
+    return Promise.resolve();
+  }
+  /**
+   * Start the garbage collection timer
+   * Called when the collection becomes inactive (no subscribers)
+   */
+  startGCTimer() {
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId);
+    }
+    const gcTime = this.config.gcTime ?? 3e5;
+    this.gcTimeoutId = setTimeout(() => {
+      if (this.activeSubscribersCount === 0) {
+        this.cleanup();
       }
-    });
+    }, gcTime);
   }
   /**
-   * Register a callback to be executed on the next commit
-   * Useful for preloading collections
-   * @param callback Function to call after the next commit
+   * Cancel the garbage collection timer
+   * Called when the collection becomes active again
    */
-  onFirstCommit(callback) {
-    this.onFirstCommitCallbacks.push(callback);
+  cancelGCTimer() {
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId);
+      this.gcTimeoutId = null;
+    }
+  }
+  /**
+   * Increment the active subscribers count and start sync if needed
+   */
+  addSubscriber() {
+    this.activeSubscribersCount++;
+    this.cancelGCTimer();
+    if (this._status === `cleaned-up` || this._status === `idle`) {
+      this.startSync();
+    }
+  }
+  /**
+   * Decrement the active subscribers count and start GC timer if needed
+   */
+  removeSubscriber() {
+    this.activeSubscribersCount--;
+    if (this.activeSubscribersCount === 0) {
+      this.activeSubscribersCount = 0;
+      this.startGCTimer();
+    } else if (this.activeSubscribersCount < 0) {
+      throw new Error(
+        `Active subscribers count is negative - this should never happen`
+      );
+    }
   }
   /**
    * Recompute optimistic state from active transactions
    */
   recomputeOptimisticState() {
+    if (this.isCommittingSyncTransactions) {
+      return;
+    }
     const previousState = new Map(this.derivedUpserts);
     const previousDeletes = new Set(this.derivedDeletes);
     this.derivedUpserts.clear();
     this.derivedDeletes.clear();
-    const activeTransactions = Array.from(this.transactions.values());
+    const activeTransactions = [];
+    const completedTransactions = [];
+    for (const transaction of this.transactions.values()) {
+      if (transaction.state === `completed`) {
+        completedTransactions.push(transaction);
+      } else if (![`completed`, `failed`].includes(transaction.state)) {
+        activeTransactions.push(transaction);
+      }
+    }
     for (const transaction of activeTransactions) {
-      if (![`completed`, `failed`].includes(transaction.state)) {
-        for (const mutation of transaction.mutations) {
-          if (mutation.collection === this) {
-            switch (mutation.type) {
-              case `insert`:
-              case `update`:
-                this.derivedUpserts.set(mutation.key, mutation.modified);
-                this.derivedDeletes.delete(mutation.key);
-                break;
-              case `delete`:
-                this.derivedUpserts.delete(mutation.key);
-                this.derivedDeletes.add(mutation.key);
-                break;
-            }
+      for (const mutation of transaction.mutations) {
+        if (mutation.collection === this) {
+          switch (mutation.type) {
+            case `insert`:
+            case `update`:
+              this.derivedUpserts.set(mutation.key, mutation.modified);
+              this.derivedDeletes.delete(mutation.key);
+              break;
+            case `delete`:
+              this.derivedUpserts.delete(mutation.key);
+              this.derivedDeletes.add(mutation.key);
+              break;
           }
         }
       }
@@ -336,7 +625,41 @@ class CollectionImpl {
     this._size = this.calculateSize();
     const events = [];
     this.collectOptimisticChanges(previousState, previousDeletes, events);
-    this.emitEvents(events);
+    const filteredEventsBySyncStatus = events.filter(
+      (event) => !this.recentlySyncedKeys.has(event.key)
+    );
+    if (this.pendingSyncedTransactions.length > 0) {
+      const pendingSyncKeys = /* @__PURE__ */ new Set();
+      const completedTransactionMutations = /* @__PURE__ */ new Set();
+      for (const transaction of this.pendingSyncedTransactions) {
+        for (const operation of transaction.operations) {
+          pendingSyncKeys.add(operation.key);
+        }
+      }
+      for (const tx of completedTransactions) {
+        for (const mutation of tx.mutations) {
+          if (mutation.collection === this) {
+            completedTransactionMutations.add(mutation.mutationId);
+          }
+        }
+      }
+      const filteredEvents = filteredEventsBySyncStatus.filter((event) => {
+        if (event.type === `delete` && pendingSyncKeys.has(event.key)) {
+          const hasActiveOptimisticMutation = activeTransactions.some(
+            (tx) => tx.mutations.some(
+              (m) => m.collection === this && m.key === event.key
+            )
+          );
+          if (!hasActiveOptimisticMutation) {
+            return false;
+          }
+        }
+        return true;
+      });
+      this.emitEvents(filteredEvents);
+    } else {
+      this.emitEvents(filteredEventsBySyncStatus);
+    }
   }
   /**
    * Calculate the current size based on synced data and optimistic changes
@@ -473,7 +796,8 @@ class CollectionImpl {
     for (const key of this.keys()) {
       const value = this.get(key);
       if (value !== void 0) {
-        yield value;
+        const { _orderByIndex, ...copy } = value;
+        yield copy;
       }
     }
   }
@@ -484,7 +808,8 @@ class CollectionImpl {
     for (const key of this.keys()) {
       const value = this.get(key);
       if (value !== void 0) {
-        yield [key, value];
+        const { _orderByIndex, ...copy } = value;
+        yield [key, copy];
       }
     }
   }
@@ -507,6 +832,24 @@ class CollectionImpl {
     }
     return `KEY::${this.id}/${key}`;
   }
+  deepEqual(a, b) {
+    if (a === b) return true;
+    if (a == null || b == null) return false;
+    if (typeof a !== typeof b) return false;
+    if (typeof a === `object`) {
+      if (Array.isArray(a) !== Array.isArray(b)) return false;
+      const keysA = Object.keys(a);
+      const keysB = Object.keys(b);
+      if (keysA.length !== keysB.length) return false;
+      const keysBSet = new Set(keysB);
+      for (const key of keysA) {
+        if (!keysBSet.has(key)) return false;
+        if (!this.deepEqual(a[key], b[key])) return false;
+      }
+      return true;
+    }
+    return false;
+  }
   validateData(data, type, key) {
     if (!this.config.schema) return data;
     const standardSchema = this.ensureStandardSchema(this.config.schema);
@@ -551,6 +894,7 @@ class CollectionImpl {
     if (typeof keys === `undefined`) {
       throw new Error(`The first argument to update is missing`);
     }
+    this.validateCollectionUsable(`update`);
     const ambientTransaction = transactions.getActiveTransaction();
     if (!ambientTransaction && !this.config.onUpdate) {
       throw new Error(
@@ -725,18 +1069,21 @@ class CollectionImpl {
    * @returns A function that can be called to unsubscribe from the changes
    */
   subscribeChanges(callback, { includeInitialState = false } = {}) {
+    this.addSubscriber();
     if (includeInitialState) {
       callback(this.currentStateAsChanges());
     }
     this.changeListeners.add(callback);
     return () => {
       this.changeListeners.delete(callback);
+      this.removeSubscriber();
     };
   }
   /**
    * Subscribe to changes for a specific key
    */
   subscribeChangesKey(key, listener, { includeInitialState = false } = {}) {
+    this.addSubscriber();
     if (!this.changeKeyListeners.has(key)) {
       this.changeKeyListeners.set(key, /* @__PURE__ */ new Set());
     }
@@ -758,13 +1105,38 @@ class CollectionImpl {
           this.changeKeyListeners.delete(key);
         }
       }
+      this.removeSubscriber();
     };
   }
+  /**
+   * Capture visible state for keys that will be affected by pending sync operations
+   * This must be called BEFORE onTransactionStateChange clears optimistic state
+   */
+  capturePreSyncVisibleState() {
+    if (this.pendingSyncedTransactions.length === 0) return;
+    this.preSyncVisibleState.clear();
+    const syncedKeys = /* @__PURE__ */ new Set();
+    for (const transaction of this.pendingSyncedTransactions) {
+      for (const operation of transaction.operations) {
+        syncedKeys.add(operation.key);
+      }
+    }
+    for (const key of syncedKeys) {
+      this.recentlySyncedKeys.add(key);
+    }
+    for (const key of syncedKeys) {
+      const currentValue = this.get(key);
+      if (currentValue !== void 0) {
+        this.preSyncVisibleState.set(key, currentValue);
+      }
+    }
+  }
   /**
    * Trigger a recomputation when transactions change
    * This method should be called by the Transaction class when state changes
    */
   onTransactionStateChange() {
+    this.capturePreSyncVisibleState();
     this.recomputeOptimisticState();
   }
   /**
@@ -776,7 +1148,7 @@ class CollectionImpl {
   asStoreMap() {
     if (!this._storeMap) {
       this._storeMap = new store.Store(new Map(this.entries()));
-      this.subscribeChanges(() => {
+      this.changeListeners.add(() => {
         this._storeMap.setState(() => new Map(this.entries()));
       });
     }
@@ -791,7 +1163,7 @@ class CollectionImpl {
   asStoreArray() {
     if (!this._storeArray) {
       this._storeArray = new store.Store(this.toArray);
-      this.subscribeChanges(() => {
+      this.changeListeners.add(() => {
         this._storeArray.setState(() => this.toArray);
       });
     }