@tanstack/db 0.0.11 → 0.0.13

package/src/collection.ts

@@ -6,28 +6,22 @@ import type {
   ChangeListener,
   ChangeMessage,
   CollectionConfig,
+  CollectionStatus,
   Fn,
   InsertConfig,
   OperationConfig,
   OptimisticChangeMessage,
   PendingMutation,
+  ResolveType,
   StandardSchema,
   Transaction as TransactionType,
   UtilsRecord,
 } from "./types"
+import type { StandardSchemaV1 } from "@standard-schema/spec"
 
 // Store collections in memory
 export const collectionsStore = new Map<string, CollectionImpl<any, any>>()
 
-// Map to track loading collections
-const loadingCollectionResolvers = new Map<
-  string,
-  {
-    promise: Promise<CollectionImpl<any, any>>
-    resolve: (value: CollectionImpl<any, any>) => void
-  }
->()
-
 interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
   committed: boolean
   operations: Array<OptimisticChangeMessage<T>>
@@ -36,6 +30,7 @@ interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
 /**
  * Enhanced Collection interface that includes both data type T and utilities TUtils
  * @template T - The type of items in the collection
+ * @template TKey - The type of the key for the collection
  * @template TUtils - The utilities record type
  */
 export interface Collection<
@@ -49,20 +44,53 @@ export interface Collection<
 /**
  * Creates a new Collection instance with the given configuration
  *
- * @template T - The type of items in the collection
+ * @template TExplicit - The explicit type of items in the collection (highest priority)
  * @template TKey - The type of the key for the collection
  * @template TUtils - The utilities record type
+ * @template TSchema - The schema type for validation and type inference (second priority)
+ * @template TFallback - The fallback type if no explicit or schema type is provided
  * @param options - Collection options with optional utilities
  * @returns A new Collection with utilities exposed both at top level and under .utils
+ *
+ * @example
+ * // Using explicit type
+ * const todos = createCollection<Todo>({
+ *   getKey: (todo) => todo.id,
+ *   sync: { sync: () => {} }
+ * })
+ *
+ * // Using schema for type inference (preferred as it also gives you client side validation)
+ * const todoSchema = z.object({
+ *   id: z.string(),
+ *   title: z.string(),
+ *   completed: z.boolean()
+ * })
+ *
+ * const todos = createCollection({
+ *   schema: todoSchema,
+ *   getKey: (todo) => todo.id,
+ *   sync: { sync: () => {} }
+ * })
+ *
+ * // Note: You must provide either an explicit type or a schema, but not both
  */
 export function createCollection<
-  T extends object = Record<string, unknown>,
+  TExplicit = unknown,
   TKey extends string | number = string | number,
   TUtils extends UtilsRecord = {},
+  TSchema extends StandardSchemaV1 = StandardSchemaV1,
+  TFallback extends object = Record<string, unknown>,
 >(
-  options: CollectionConfig<T, TKey> & { utils?: TUtils }
-): Collection<T, TKey, TUtils> {
-  const collection = new CollectionImpl<T, TKey>(options)
+  options: CollectionConfig<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TSchema
+  > & { utils?: TUtils }
+): Collection<ResolveType<TExplicit, TSchema, TFallback>, TKey, TUtils> {
+  const collection = new CollectionImpl<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey
+  >(options)
 
   // Copy utils to both top level and .utils namespace
   if (options.utils) {
@@ -71,98 +99,11 @@ export function createCollection<
     collection.utils = {} as TUtils
   }
 
-  return collection as Collection<T, TKey, TUtils>
-}
-
-/**
- * Preloads a collection with the given configuration
- * Returns a promise that resolves once the sync tool has done its first commit (initial sync is finished)
- * If the collection has already loaded, it resolves immediately
- *
- * This function is useful in route loaders or similar pre-rendering scenarios where you want
- * to ensure data is available before a route transition completes. It uses the same shared collection
- * instance that will be used by useCollection, ensuring data consistency.
- *
- * @example
- * ```typescript
- * // In a route loader
- * async function loader({ params }) {
- *   await preloadCollection({
- *     id: `users-${params.userId}`,
- *     sync: { ... },
- *   });
- *
- *   return null;
- * }
- * ```
- *
- * @template T - The type of items in the collection
- * @param config - Configuration for the collection, including id and sync
- * @returns Promise that resolves when the initial sync is finished
- */
-export function preloadCollection<
-  T extends object = Record<string, unknown>,
-  TKey extends string | number = string | number,
->(config: CollectionConfig<T, TKey>): Promise<CollectionImpl<T, TKey>> {
-  if (!config.id) {
-    throw new Error(`The id property is required for preloadCollection`)
-  }
-
-  // If the collection is already fully loaded, return a resolved promise
-  if (
-    collectionsStore.has(config.id) &&
-    !loadingCollectionResolvers.has(config.id)
-  ) {
-    return Promise.resolve(
-      collectionsStore.get(config.id)! as CollectionImpl<T, TKey>
-    )
-  }
-
-  // If the collection is in the process of loading, return its promise
-  if (loadingCollectionResolvers.has(config.id)) {
-    return loadingCollectionResolvers.get(config.id)!.promise
-  }
-
-  // Create a new collection instance if it doesn't exist
-  if (!collectionsStore.has(config.id)) {
-    collectionsStore.set(
-      config.id,
-      createCollection<T, TKey>({
-        id: config.id,
-        getKey: config.getKey,
-        sync: config.sync,
-        schema: config.schema,
-      })
-    )
-  }
-
-  const collection = collectionsStore.get(config.id)! as CollectionImpl<T, TKey>
-
-  // Create a promise that will resolve after the first commit
-  let resolveFirstCommit: (value: CollectionImpl<T, TKey>) => void
-  const firstCommitPromise = new Promise<CollectionImpl<T, TKey>>((resolve) => {
-    resolveFirstCommit = resolve
-  })
-
-  // Store the loading promise first
-  loadingCollectionResolvers.set(config.id, {
-    promise: firstCommitPromise,
-    resolve: resolveFirstCommit!,
-  })
-
-  // Register a one-time listener for the first commit
-  collection.onFirstCommit(() => {
-    if (!config.id) {
-      throw new Error(`The id property is required for preloadCollection`)
-    }
-    if (loadingCollectionResolvers.has(config.id)) {
-      const resolver = loadingCollectionResolvers.get(config.id)!
-      loadingCollectionResolvers.delete(config.id)
-      resolver.resolve(collection)
-    }
-  })
-
-  return firstCommitPromise
+  return collection as Collection<
+    ResolveType<TExplicit, TSchema, TFallback>,
+    TKey,
+    TUtils
+  >
 }
 
 /**
@@ -184,8 +125,8 @@ export class SchemaValidationError extends Error {
     message?: string
   ) {
     const defaultMessage = `${type === `insert` ? `Insert` : `Update`} validation failed: ${issues
-      .map((issue) => issue.message)
-      .join(`, `)}`
+      .map((issue) => `\n- ${issue.message} - path: ${issue.path}`)
+      .join(``)}`
 
     super(message || defaultMessage)
     this.name = `SchemaValidationError`
@@ -198,10 +139,12 @@ export class CollectionImpl<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
 > {
-  public transactions: SortedMap<string, Transaction<any>>
+  public config: CollectionConfig<T, TKey, any>
 
   // Core state - make public for testing
-  public syncedData = new Map<TKey, T>()
+  public transactions: SortedMap<string, Transaction<any>>
+  public pendingSyncedTransactions: Array<PendingSyncedTransaction<T>> = []
+  public syncedData: Map<TKey, T> | SortedMap<TKey, T>
   public syncedMetadata = new Map<TKey, unknown>()
 
   // Optimistic state tracking - make public for testing
@@ -219,14 +162,23 @@ export class CollectionImpl<
   // This is populated by createCollection
   public utils: Record<string, Fn> = {}
 
-  private pendingSyncedTransactions: Array<PendingSyncedTransaction<T>> = []
+  // State used for computing the change events
   private syncedKeys = new Set<TKey>()
-  public config: CollectionConfig<T, TKey>
+  private preSyncVisibleState = new Map<TKey, T>()
+  private recentlySyncedKeys = new Set<TKey>()
   private hasReceivedFirstCommit = false
+  private isCommittingSyncTransactions = false
 
   // Array to store one-time commit listeners
   private onFirstCommitCallbacks: Array<() => void> = []
 
+  // Lifecycle management
+  private _status: CollectionStatus = `idle`
+  private activeSubscribersCount = 0
+  private gcTimeoutId: ReturnType<typeof setTimeout> | null = null
+  private preloadPromise: Promise<void> | null = null
+  private syncCleanupFn: (() => void) | null = null
+
   /**
    * Register a callback to be executed on the next commit
    * Useful for preloading collections
@@ -238,13 +190,78 @@ export class CollectionImpl<
 
   public id = ``
 
+  /**
+   * Gets the current status of the collection
+   */
+  public get status(): CollectionStatus {
+    return this._status
+  }
+
+  /**
+   * Validates that the collection is in a usable state for data operations
+   * @private
+   */
+  private validateCollectionUsable(operation: string): void {
+    switch (this._status) {
+      case `error`:
+        throw new Error(
+          `Cannot perform ${operation} on collection "${this.id}" - collection is in error state. ` +
+            `Try calling cleanup() and restarting the collection.`
+        )
+      case `cleaned-up`:
+        throw new Error(
+          `Cannot perform ${operation} on collection "${this.id}" - collection has been cleaned up. ` +
+            `The collection will automatically restart on next access.`
+        )
+    }
+  }
+
+  /**
+   * Validates state transitions to prevent invalid status changes
+   * @private
+   */
+  private validateStatusTransition(
+    from: CollectionStatus,
+    to: CollectionStatus
+  ): void {
+    if (from === to) {
+      // Allow same state transitions
+      return
+    }
+    const validTransitions: Record<
+      CollectionStatus,
+      Array<CollectionStatus>
+    > = {
+      idle: [`loading`, `error`, `cleaned-up`],
+      loading: [`ready`, `error`, `cleaned-up`],
+      ready: [`cleaned-up`, `error`],
+      error: [`cleaned-up`, `idle`],
+      "cleaned-up": [`loading`, `error`],
+    }
+
+    if (!validTransitions[from].includes(to)) {
+      throw new Error(
+        `Invalid collection status transition from "${from}" to "${to}" for collection "${this.id}"`
+      )
+    }
+  }
+
+  /**
+   * Safely update the collection status with validation
+   * @private
+   */
+  private setStatus(newStatus: CollectionStatus): void {
+    this.validateStatusTransition(this._status, newStatus)
+    this._status = newStatus
+  }
+
   /**
    * Creates a new Collection instance
    *
    * @param config - Configuration object for the collection
    * @throws Error if sync config is missing
    */
-  constructor(config: CollectionConfig<T, TKey>) {
+  constructor(config: CollectionConfig<T, TKey, any>) {
     // eslint-disable-next-line
     if (!config) {
       throw new Error(`Collection requires a config`)
@@ -266,74 +283,276 @@ export class CollectionImpl<
 
     this.config = config
 
-    // Start the sync process
-    config.sync.sync({
-      collection: this,
-      begin: () => {
-        this.pendingSyncedTransactions.push({
-          committed: false,
-          operations: [],
-        })
-      },
-      write: (messageWithoutKey: Omit<ChangeMessage<T>, `key`>) => {
-        const pendingTransaction =
-          this.pendingSyncedTransactions[
-            this.pendingSyncedTransactions.length - 1
-          ]
-        if (!pendingTransaction) {
-          throw new Error(`No pending sync transaction to write to`)
-        }
-        if (pendingTransaction.committed) {
-          throw new Error(
-            `The pending sync transaction is already committed, you can't still write to it.`
-          )
-        }
-        const key = this.getKeyFromItem(messageWithoutKey.value)
+    // Store in global collections store
+    collectionsStore.set(this.id, this)
 
-        // Check if an item with this key already exists when inserting
-        if (messageWithoutKey.type === `insert`) {
-          if (
-            this.syncedData.has(key) &&
-            !pendingTransaction.operations.some(
-              (op) => op.key === key && op.type === `delete`
+    // Set up data storage with optional comparison function
+    if (this.config.compare) {
+      this.syncedData = new SortedMap<TKey, T>(this.config.compare)
+    } else {
+      this.syncedData = new Map<TKey, T>()
+    }
+
+    // Only start sync immediately if explicitly enabled
+    if (config.startSync === true) {
+      this.startSync()
+    }
+  }
+
+  /**
+   * Start sync immediately - internal method for compiled queries
+   * This bypasses lazy loading for special cases like live query results
+   */
+  public startSyncImmediate(): void {
+    this.startSync()
+  }
+
+  /**
+   * Start the sync process for this collection
+   * This is called when the collection is first accessed or preloaded
+   */
+  private startSync(): void {
+    if (this._status !== `idle` && this._status !== `cleaned-up`) {
+      return // Already started or in progress
+    }
+
+    this.setStatus(`loading`)
+
+    try {
+      const cleanupFn = this.config.sync.sync({
+        collection: this,
+        begin: () => {
+          this.pendingSyncedTransactions.push({
+            committed: false,
+            operations: [],
+          })
+        },
+        write: (messageWithoutKey: Omit<ChangeMessage<T>, `key`>) => {
+          const pendingTransaction =
+            this.pendingSyncedTransactions[
+              this.pendingSyncedTransactions.length - 1
+            ]
+          if (!pendingTransaction) {
+            throw new Error(`No pending sync transaction to write to`)
+          }
+          if (pendingTransaction.committed) {
+            throw new Error(
+              `The pending sync transaction is already committed, you can't still write to it.`
             )
-          ) {
+          }
+          const key = this.getKeyFromItem(messageWithoutKey.value)
+
+          // Check if an item with this key already exists when inserting
+          if (messageWithoutKey.type === `insert`) {
+            if (
+              this.syncedData.has(key) &&
+              !pendingTransaction.operations.some(
+                (op) => op.key === key && op.type === `delete`
+              )
+            ) {
+              throw new Error(
+                `Cannot insert document with key "${key}" from sync because it already exists in the collection "${this.id}"`
+              )
+            }
+          }
+
+          const message: ChangeMessage<T> = {
+            ...messageWithoutKey,
+            key,
+          }
+          pendingTransaction.operations.push(message)
+        },
+        commit: () => {
+          const pendingTransaction =
+            this.pendingSyncedTransactions[
+              this.pendingSyncedTransactions.length - 1
+            ]
+          if (!pendingTransaction) {
+            throw new Error(`No pending sync transaction to commit`)
+          }
+          if (pendingTransaction.committed) {
             throw new Error(
-              `Cannot insert document with key "${key}" from sync because it already exists in the collection "${this.id}"`
+              `The pending sync transaction is already committed, you can't commit it again.`
             )
           }
-        }
 
-        const message: ChangeMessage<T> = {
-          ...messageWithoutKey,
-          key,
-        }
-        pendingTransaction.operations.push(message)
-      },
-      commit: () => {
-        const pendingTransaction =
-          this.pendingSyncedTransactions[
-            this.pendingSyncedTransactions.length - 1
-          ]
-        if (!pendingTransaction) {
-          throw new Error(`No pending sync transaction to commit`)
+          pendingTransaction.committed = true
+          this.commitPendingTransactions()
+
+          // Update status to ready after first commit
+          if (this._status === `loading`) {
+            this.setStatus(`ready`)
+          }
+        },
+      })
+
+      // Store cleanup function if provided
+      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null
+    } catch (error) {
+      this.setStatus(`error`)
+      throw error
+    }
+  }
+
+  /**
+   * Preload the collection data by starting sync if not already started
+   * Multiple concurrent calls will share the same promise
+   */
+  public preload(): Promise<void> {
+    if (this.preloadPromise) {
+      return this.preloadPromise
+    }
+
+    this.preloadPromise = new Promise<void>((resolve, reject) => {
+      if (this._status === `ready`) {
+        resolve()
+        return
+      }
+
+      if (this._status === `error`) {
+        reject(new Error(`Collection is in error state`))
+        return
+      }
+
+      // Register callback BEFORE starting sync to avoid race condition
+      this.onFirstCommit(() => {
+        resolve()
+      })
+
+      // Start sync if collection hasn't started yet or was cleaned up
+      if (this._status === `idle` || this._status === `cleaned-up`) {
+        try {
+          this.startSync()
+        } catch (error) {
+          reject(error)
+          return
         }
-        if (pendingTransaction.committed) {
+      }
+    })
+
+    return this.preloadPromise
+  }
+
+  /**
+   * Clean up the collection by stopping sync and clearing data
+   * This can be called manually or automatically by garbage collection
+   */
+  public async cleanup(): Promise<void> {
+    // Clear GC timeout
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId)
+      this.gcTimeoutId = null
+    }
+
+    // Stop sync - wrap in try/catch since it's user-provided code
+    try {
+      if (this.syncCleanupFn) {
+        this.syncCleanupFn()
+        this.syncCleanupFn = null
+      }
+    } catch (error) {
+      // Re-throw in a microtask to surface the error after cleanup completes
+      queueMicrotask(() => {
+        if (error instanceof Error) {
+          // Preserve the original error and stack trace
+          const wrappedError = new Error(
+            `Collection "${this.id}" sync cleanup function threw an error: ${error.message}`
+          )
+          wrappedError.cause = error
+          wrappedError.stack = error.stack
+          throw wrappedError
+        } else {
           throw new Error(
-            `The pending sync transaction is already committed, you can't commit it again.`
+            `Collection "${this.id}" sync cleanup function threw an error: ${String(error)}`
           )
         }
+      })
+    }
 
-        pendingTransaction.committed = true
-        this.commitPendingTransactions()
-      },
-    })
+    // Clear data
+    this.syncedData.clear()
+    this.syncedMetadata.clear()
+    this.derivedUpserts.clear()
+    this.derivedDeletes.clear()
+    this._size = 0
+    this.pendingSyncedTransactions = []
+    this.syncedKeys.clear()
+    this.hasReceivedFirstCommit = false
+    this.onFirstCommitCallbacks = []
+    this.preloadPromise = null
+
+    // Update status
+    this.setStatus(`cleaned-up`)
+
+    return Promise.resolve()
+  }
+
+  /**
+   * Start the garbage collection timer
+   * Called when the collection becomes inactive (no subscribers)
+   */
+  private startGCTimer(): void {
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId)
+    }
+
+    const gcTime = this.config.gcTime ?? 300000 // 5 minutes default
+    this.gcTimeoutId = setTimeout(() => {
+      if (this.activeSubscribersCount === 0) {
+        this.cleanup()
+      }
+    }, gcTime)
+  }
+
+  /**
+   * Cancel the garbage collection timer
+   * Called when the collection becomes active again
+   */
+  private cancelGCTimer(): void {
+    if (this.gcTimeoutId) {
+      clearTimeout(this.gcTimeoutId)
+      this.gcTimeoutId = null
+    }
+  }
+
+  /**
+   * Increment the active subscribers count and start sync if needed
+   */
+  private addSubscriber(): void {
+    this.activeSubscribersCount++
+    this.cancelGCTimer()
+
+    // Start sync if collection was cleaned up
+    if (this._status === `cleaned-up` || this._status === `idle`) {
+      this.startSync()
+    }
+  }
+
+  /**
+   * Decrement the active subscribers count and start GC timer if needed
+   */
+  private removeSubscriber(): void {
+    this.activeSubscribersCount--
+
+    if (this.activeSubscribersCount === 0) {
+      this.activeSubscribersCount = 0
+      this.startGCTimer()
+    } else if (this.activeSubscribersCount < 0) {
+      throw new Error(
+        `Active subscribers count is negative - this should never happen`
+      )
+    }
   }
 
   /**
    * Recompute optimistic state from active transactions
    */
   private recomputeOptimisticState(): void {
+    // Skip redundant recalculations when we're in the middle of committing sync transactions
+    if (this.isCommittingSyncTransactions) {
+      return
+    }
+
     const previousState = new Map(this.derivedUpserts)
     const previousDeletes = new Set(this.derivedDeletes)
 
@@ -341,23 +560,31 @@ export class CollectionImpl<
     this.derivedUpserts.clear()
     this.derivedDeletes.clear()
 
-    // Apply active transactions
-    const activeTransactions = Array.from(this.transactions.values())
+    const activeTransactions: Array<Transaction<any>> = []
+    const completedTransactions: Array<Transaction<any>> = []
+
+    for (const transaction of this.transactions.values()) {
+      if (transaction.state === `completed`) {
+        completedTransactions.push(transaction)
+      } else if (![`completed`, `failed`].includes(transaction.state)) {
+        activeTransactions.push(transaction)
+      }
+    }
+
+    // Apply active transactions only (completed transactions are handled by sync operations)
     for (const transaction of activeTransactions) {
-      if (![`completed`, `failed`].includes(transaction.state)) {
-        for (const mutation of transaction.mutations) {
-          if (mutation.collection === this) {
-            switch (mutation.type) {
-              case `insert`:
-              case `update`:
-                this.derivedUpserts.set(mutation.key, mutation.modified as T)
-                this.derivedDeletes.delete(mutation.key)
-                break
-              case `delete`:
-                this.derivedUpserts.delete(mutation.key)
-                this.derivedDeletes.add(mutation.key)
-                break
-            }
+      for (const mutation of transaction.mutations) {
+        if (mutation.collection === this) {
+          switch (mutation.type) {
+            case `insert`:
+            case `update`:
+              this.derivedUpserts.set(mutation.key, mutation.modified as T)
+              this.derivedDeletes.delete(mutation.key)
+              break
+            case `delete`:
+              this.derivedUpserts.delete(mutation.key)
+              this.derivedDeletes.add(mutation.key)
+              break
           }
         }
       }
@@ -370,8 +597,58 @@ export class CollectionImpl<
     const events: Array<ChangeMessage<T, TKey>> = []
     this.collectOptimisticChanges(previousState, previousDeletes, events)
 
-    // Emit all events at once
-    this.emitEvents(events)
+    // Filter out events for recently synced keys to prevent duplicates
+    const filteredEventsBySyncStatus = events.filter(
+      (event) => !this.recentlySyncedKeys.has(event.key)
+    )
+
+    // Filter out redundant delete events if there are pending sync transactions
+    // that will immediately restore the same data, but only for completed transactions
+    if (this.pendingSyncedTransactions.length > 0) {
+      const pendingSyncKeys = new Set<TKey>()
+      const completedTransactionMutations = new Set<string>()
+
+      // Collect keys from pending sync operations
+      for (const transaction of this.pendingSyncedTransactions) {
+        for (const operation of transaction.operations) {
+          pendingSyncKeys.add(operation.key as TKey)
+        }
+      }
+
+      // Collect mutation IDs from completed transactions
+      for (const tx of completedTransactions) {
+        for (const mutation of tx.mutations) {
+          if (mutation.collection === this) {
+            completedTransactionMutations.add(mutation.mutationId)
+          }
+        }
+      }
+
+      // Only filter out delete events for keys that:
+      // 1. Have pending sync operations AND
+      // 2. Are from completed transactions (being cleaned up)
+      const filteredEvents = filteredEventsBySyncStatus.filter((event) => {
+        if (event.type === `delete` && pendingSyncKeys.has(event.key)) {
+          // Check if this delete is from clearing optimistic state of completed transactions
+          // We can infer this by checking if we have no remaining optimistic mutations for this key
+          const hasActiveOptimisticMutation = activeTransactions.some((tx) =>
+            tx.mutations.some(
+              (m) => m.collection === this && m.key === event.key
+            )
+          )
+
+          if (!hasActiveOptimisticMutation) {
+            return false // Skip this delete event as sync will restore the data
+          }
+        }
+        return true
+      })
+
+      this.emitEvents(filteredEvents)
+    } else {
+      // Emit all events if no pending sync transactions
+      this.emitEvents(filteredEventsBySyncStatus)
+    }
   }
 
   /**
@@ -552,7 +829,10 @@ export class CollectionImpl<
     for (const key of this.keys()) {
       const value = this.get(key)
       if (value !== undefined) {
-        yield value
+        const { _orderByIndex, ...copy } = value as T & {
+          _orderByIndex?: number | string
+        }
+        yield copy as T
       }
     }
   }
@@ -564,7 +844,10 @@ export class CollectionImpl<
     for (const key of this.keys()) {
       const value = this.get(key)
       if (value !== undefined) {
-        yield [key, value]
+        const { _orderByIndex, ...copy } = value as T & {
+          _orderByIndex?: number | string
+        }
+        yield [key, copy as T]
       }
     }
   }
@@ -574,18 +857,46 @@ export class CollectionImpl<
    * This method processes operations from pending transactions and applies them to the synced data
    */
   commitPendingTransactions = () => {
-    if (
-      !Array.from(this.transactions.values()).some(
-        ({ state }) => state === `persisting`
-      )
-    ) {
+    // Check if there are any persisting transaction
+    let hasPersistingTransaction = false
+    for (const transaction of this.transactions.values()) {
+      if (transaction.state === `persisting`) {
+        hasPersistingTransaction = true
+        break
+      }
+    }
+
+    if (!hasPersistingTransaction) {
+      // Set flag to prevent redundant optimistic state recalculations
+      this.isCommittingSyncTransactions = true
+
+      // First collect all keys that will be affected by sync operations
       const changedKeys = new Set<TKey>()
+      for (const transaction of this.pendingSyncedTransactions) {
+        for (const operation of transaction.operations) {
+          changedKeys.add(operation.key as TKey)
+        }
+      }
+
+      // Use pre-captured state if available (from optimistic scenarios),
+      // otherwise capture current state (for pure sync scenarios)
+      let currentVisibleState = this.preSyncVisibleState
+      if (currentVisibleState.size === 0) {
+        // No pre-captured state, capture it now for pure sync operations
+        currentVisibleState = new Map<TKey, T>()
+        for (const key of changedKeys) {
+          const currentValue = this.get(key)
+          if (currentValue !== undefined) {
+            currentVisibleState.set(key, currentValue)
+          }
+        }
+      }
+
       const events: Array<ChangeMessage<T, TKey>> = []
 
       for (const transaction of this.pendingSyncedTransactions) {
         for (const operation of transaction.operations) {
           const key = operation.key as TKey
-          changedKeys.add(key)
           this.syncedKeys.add(key)
 
           // Update metadata
@@ -608,22 +919,10 @@ export class CollectionImpl<
               break
           }
 
-          // Update synced data and collect events
-          const previousValue = this.syncedData.get(key)
-
+          // Update synced data
           switch (operation.type) {
             case `insert`:
               this.syncedData.set(key, operation.value)
-              if (
-                !this.derivedDeletes.has(key) &&
-                !this.derivedUpserts.has(key)
-              ) {
-                events.push({
-                  type: `insert`,
-                  key,
-                  value: operation.value,
-                })
-              }
               break
             case `update`: {
               const updatedValue = Object.assign(
@@ -632,38 +931,103 @@ export class CollectionImpl<
                 operation.value
               )
               this.syncedData.set(key, updatedValue)
-              if (
-                !this.derivedDeletes.has(key) &&
-                !this.derivedUpserts.has(key)
-              ) {
-                events.push({
-                  type: `update`,
-                  key,
-                  value: updatedValue,
-                  previousValue,
-                })
-              }
               break
             }
             case `delete`:
               this.syncedData.delete(key)
-              if (
-                !this.derivedDeletes.has(key) &&
-                !this.derivedUpserts.has(key)
-              ) {
-                if (previousValue) {
-                  events.push({
-                    type: `delete`,
-                    key,
-                    value: previousValue,
-                  })
-                }
-              }
               break
           }
         }
       }
 
+      // Clear optimistic state since sync operations will now provide the authoritative data
+      this.derivedUpserts.clear()
+      this.derivedDeletes.clear()
+
+      // Reset flag and recompute optimistic state for any remaining active transactions
+      this.isCommittingSyncTransactions = false
+      for (const transaction of this.transactions.values()) {
+        if (![`completed`, `failed`].includes(transaction.state)) {
+          for (const mutation of transaction.mutations) {
+            if (mutation.collection === this) {
+              switch (mutation.type) {
+                case `insert`:
+                case `update`:
+                  this.derivedUpserts.set(mutation.key, mutation.modified as T)
+                  this.derivedDeletes.delete(mutation.key)
+                  break
+                case `delete`:
+                  this.derivedUpserts.delete(mutation.key)
+                  this.derivedDeletes.add(mutation.key)
+                  break
+              }
+            }
+          }
+        }
+      }
+
+      // Check for redundant sync operations that match completed optimistic operations
+      const completedOptimisticOps = new Map<TKey, any>()
+
+      for (const transaction of this.transactions.values()) {
+        if (transaction.state === `completed`) {
+          for (const mutation of transaction.mutations) {
+            if (mutation.collection === this && changedKeys.has(mutation.key)) {
+              completedOptimisticOps.set(mutation.key, {
+                type: mutation.type,
+                value: mutation.modified,
+              })
+            }
+          }
+        }
+      }
+
+      // Now check what actually changed in the final visible state
+      for (const key of changedKeys) {
+        const previousVisibleValue = currentVisibleState.get(key)
+        const newVisibleValue = this.get(key) // This returns the new derived state
+
+        // Check if this sync operation is redundant with a completed optimistic operation
+        const completedOp = completedOptimisticOps.get(key)
+        const isRedundantSync =
+          completedOp &&
+          newVisibleValue !== undefined &&
+          this.deepEqual(completedOp.value, newVisibleValue)
+
+        if (!isRedundantSync) {
+          if (
+            previousVisibleValue === undefined &&
+            newVisibleValue !== undefined
+          ) {
+            events.push({
+              type: `insert`,
+              key,
+              value: newVisibleValue,
+            })
+          } else if (
+            previousVisibleValue !== undefined &&
+            newVisibleValue === undefined
+          ) {
+            events.push({
+              type: `delete`,
+              key,
+              value: previousVisibleValue,
+            })
+          } else if (
+            previousVisibleValue !== undefined &&
+            newVisibleValue !== undefined &&
+            !this.deepEqual(previousVisibleValue, newVisibleValue)
+          ) {
+            events.push({
+              type: `update`,
+              key,
+              value: newVisibleValue,
+              previousValue: previousVisibleValue,
+            })
+          }
+        }
+      }
+
       // Update cached size after synced data changes
       this._size = this.calculateSize()
 
@@ -672,6 +1036,14 @@ export class CollectionImpl<
 
       this.pendingSyncedTransactions = []
 
+      // Clear the pre-sync state since sync operations are complete
+      this.preSyncVisibleState.clear()
+
+      // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them
+      Promise.resolve().then(() => {
+        this.recentlySyncedKeys.clear()
+      })
+
       // Call any registered one-time commit listeners
       if (!this.hasReceivedFirstCommit) {
         this.hasReceivedFirstCommit = true
@@ -707,6 +1079,29 @@ export class CollectionImpl<
     return `KEY::${this.id}/${key}`
   }
 
+  private deepEqual(a: any, b: any): boolean {
+    if (a === b) return true
+    if (a == null || b == null) return false
+    if (typeof a !== typeof b) return false
+
+    if (typeof a === `object`) {
+      if (Array.isArray(a) !== Array.isArray(b)) return false
+
+      const keysA = Object.keys(a)
+      const keysB = Object.keys(b)
+      if (keysA.length !== keysB.length) return false
+
+      const keysBSet = new Set(keysB)
+      for (const key of keysA) {
+        if (!keysBSet.has(key)) return false
+        if (!this.deepEqual(a[key], b[key])) return false
+      }
+      return true
+    }
+
+    return false
+  }
+
   private validateData(
     data: unknown,
     type: `insert` | `update`,
@@ -793,6 +1188,8 @@ export class CollectionImpl<
    * insert({ text: "Buy groceries" }, { key: "grocery-task" })
    */
   insert = (data: T | Array<T>, config?: InsertConfig) => {
+    this.validateCollectionUsable(`insert`)
+
     const ambientTransaction = getActiveTransaction()
 
     // If no ambient transaction exists, check for an onInsert handler early
@@ -803,7 +1200,7 @@ export class CollectionImpl<
     }
 
     const items = Array.isArray(data) ? data : [data]
-    const mutations: Array<PendingMutation<T>> = []
+    const mutations: Array<PendingMutation<T, `insert`>> = []
 
     // Create mutations for each item
     items.forEach((item) => {
@@ -817,7 +1214,7 @@ export class CollectionImpl<
       }
       const globalKey = this.generateGlobalKey(key, item)
 
-      const mutation: PendingMutation<T> = {
+      const mutation: PendingMutation<T, `insert`> = {
         mutationId: crypto.randomUUID(),
         original: {},
         modified: validatedData,
@@ -938,6 +1335,8 @@ export class CollectionImpl<
       throw new Error(`The first argument to update is missing`)
     }
 
+    this.validateCollectionUsable(`update`)
+
     const ambientTransaction = getActiveTransaction()
 
     // If no ambient transaction exists, check for an onUpdate handler early
@@ -987,7 +1386,7 @@ export class CollectionImpl<
     }
 
     // Create mutations for each object that has changes
-    const mutations: Array<PendingMutation<T>> = keysArray
+    const mutations: Array<PendingMutation<T, `update`>> = keysArray
       .map((key, index) => {
         const itemChanges = changesArray[index] // User-provided changes for this specific item
 
@@ -1025,9 +1424,9 @@ export class CollectionImpl<
 
         return {
           mutationId: crypto.randomUUID(),
-          original: originalItem as Record<string, unknown>,
-          modified: modifiedItem as Record<string, unknown>,
-          changes: validatedUpdatePayload as Record<string, unknown>,
+          original: originalItem,
+          modified: modifiedItem,
+          changes: validatedUpdatePayload as Partial<T>,
           globalKey,
           key,
           metadata: config.metadata as unknown,
@@ -1041,7 +1440,7 @@ export class CollectionImpl<
           collection: this,
         }
       })
-      .filter(Boolean) as Array<PendingMutation<T>>
+      .filter(Boolean) as Array<PendingMutation<T, `update`>>
 
     // If no changes were made, return an empty transaction early
     if (mutations.length === 0) {
@@ -1103,6 +1502,8 @@ export class CollectionImpl<
     keys: Array<TKey> | TKey,
     config?: OperationConfig
   ): TransactionType<any> => {
+    this.validateCollectionUsable(`delete`)
+
     const ambientTransaction = getActiveTransaction()
 
     // If no ambient transaction exists, check for an onDelete handler early
@@ -1117,15 +1518,20 @@ export class CollectionImpl<
     }
 
     const keysArray = Array.isArray(keys) ? keys : [keys]
-    const mutations: Array<PendingMutation<T>> = []
+    const mutations: Array<PendingMutation<T, `delete`>> = []
 
     for (const key of keysArray) {
+      if (!this.has(key)) {
+        throw new Error(
+          `Collection.delete was called with key '${key}' but there is no item in the collection with this key`
+        )
+      }
       const globalKey = this.generateGlobalKey(key, this.get(key)!)
-      const mutation: PendingMutation<T> = {
+      const mutation: PendingMutation<T, `delete`> = {
         mutationId: crypto.randomUUID(),
-        original: this.get(key) || {},
+        original: this.get(key)!,
         modified: this.get(key)!,
-        changes: this.get(key) || {},
+        changes: this.get(key)!,
         globalKey,
         key,
         metadata: config?.metadata as unknown,
@@ -1266,6 +1672,9 @@ export class CollectionImpl<
     callback: (changes: Array<ChangeMessage<T>>) => void,
     { includeInitialState = false }: { includeInitialState?: boolean } = {}
   ): () => void {
+    // Start sync and track subscriber
+    this.addSubscriber()
+
     if (includeInitialState) {
       // First send the current state as changes
       callback(this.currentStateAsChanges())
@@ -1276,6 +1685,7 @@ export class CollectionImpl<
 
     return () => {
       this.changeListeners.delete(callback)
+      this.removeSubscriber()
     }
   }
 
@@ -1287,6 +1697,9 @@ export class CollectionImpl<
     listener: ChangeListener<T, TKey>,
     { includeInitialState = false }: { includeInitialState?: boolean } = {}
   ): () => void {
+    // Start sync and track subscriber
+    this.addSubscriber()
+
     if (!this.changeKeyListeners.has(key)) {
       this.changeKeyListeners.set(key, new Set())
     }
@@ -1312,6 +1725,40 @@ export class CollectionImpl<
           this.changeKeyListeners.delete(key)
         }
       }
+      this.removeSubscriber()
+    }
+  }
+
+  /**
+   * Capture visible state for keys that will be affected by pending sync operations
+   * This must be called BEFORE onTransactionStateChange clears optimistic state
+   */
+  private capturePreSyncVisibleState(): void {
+    if (this.pendingSyncedTransactions.length === 0) return
+
+    // Clear any previous capture
+    this.preSyncVisibleState.clear()
+
+    // Get all keys that will be affected by sync operations
+    const syncedKeys = new Set<TKey>()
+    for (const transaction of this.pendingSyncedTransactions) {
+      for (const operation of transaction.operations) {
+        syncedKeys.add(operation.key as TKey)
+      }
+    }
+
+    // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState
+    for (const key of syncedKeys) {
+      this.recentlySyncedKeys.add(key)
+    }
+
+    // Only capture current visible state for keys that will be affected by sync operations
+    // This is much more efficient than capturing the entire collection state
+    for (const key of syncedKeys) {
+      const currentValue = this.get(key)
+      if (currentValue !== undefined) {
+        this.preSyncVisibleState.set(key, currentValue)
+      }
     }
   }
 
@@ -1320,6 +1767,9 @@ export class CollectionImpl<
    * This method should be called by the Transaction class when state changes
    */
   public onTransactionStateChange(): void {
+    // CRITICAL: Capture visible state BEFORE clearing optimistic state
+    this.capturePreSyncVisibleState()
+
     this.recomputeOptimisticState()
   }
 
@@ -1334,7 +1784,7 @@ export class CollectionImpl<
   public asStoreMap(): Store<Map<TKey, T>> {
     if (!this._storeMap) {
       this._storeMap = new Store(new Map(this.entries()))
-      this.subscribeChanges(() => {
+      this.changeListeners.add(() => {
         this._storeMap!.setState(() => new Map(this.entries()))
       })
     }
@@ -1352,7 +1802,7 @@ export class CollectionImpl<
   public asStoreArray(): Store<Array<T>> {
     if (!this._storeArray) {
       this._storeArray = new Store(this.toArray)
-      this.subscribeChanges(() => {
+      this.changeListeners.add(() => {
         this._storeArray!.setState(() => this.toArray)
       })
     }