@tanstack/db 0.4.17 → 0.4.18

package/dist/esm/query/live/collection-subscriber.js

@@ -41,20 +41,26 @@ class CollectionSubscriber {
         includeInitialState
       );
     }
+    const trackLoadPromise = () => {
+      if (!this.subscriptionLoadingPromises.has(subscription)) {
+        let resolve;
+        const promise = new Promise((res) => {
+          resolve = res;
+        });
+        this.subscriptionLoadingPromises.set(subscription, {
+          resolve
+        });
+        this.collectionConfigBuilder.liveQueryCollection._sync.trackLoadPromise(
+          promise
+        );
+      }
+    };
+    if (subscription.status === `loadingSubset`) {
+      trackLoadPromise();
+    }
     const statusUnsubscribe = subscription.on(`status:change`, (event) => {
       if (event.status === `loadingSubset`) {
-        if (!this.subscriptionLoadingPromises.has(subscription)) {
-          let resolve;
-          const promise = new Promise((res) => {
-            resolve = res;
-          });
-          this.subscriptionLoadingPromises.set(subscription, {
-            resolve
-          });
-          this.collectionConfigBuilder.liveQueryCollection._sync.trackLoadPromise(
-            promise
-          );
-        }
+        trackLoadPromise();
       } else {
         const deferred = this.subscriptionLoadingPromises.get(subscription);
         if (deferred) {
@@ -100,18 +106,10 @@ class CollectionSubscriber {
     return subscription;
   }
   subscribeToOrderedChanges(whereExpression, orderByInfo) {
-    const { orderBy, offset, limit, comparator, dataNeeded, index } = orderByInfo;
+    const { orderBy, offset, limit, index } = orderByInfo;
     const sendChangesInRange = (changes) => {
       const splittedChanges = splitUpdates(changes);
-      let filteredChanges = splittedChanges;
-      if (dataNeeded && dataNeeded() === 0) {
-        filteredChanges = filterChangesSmallerOrEqualToMax(
-          splittedChanges,
-          comparator,
-          this.biggest
-        );
-      }
-      this.sendChangesToPipelineWithTracking(filteredChanges, subscription);
+      this.sendChangesToPipelineWithTracking(splittedChanges, subscription);
     };
     const subscription = this.collection.subscribeChanges(sendChangesInRange, {
       whereExpression
@@ -234,18 +232,6 @@ function* splitUpdates(changes) {
     }
   }
 }
-function* filterChanges(changes, f) {
-  for (const change of changes) {
-    if (f(change)) {
-      yield change;
-    }
-  }
-}
-function* filterChangesSmallerOrEqualToMax(changes, comparator, maxValue) {
-  yield* filterChanges(changes, (change) => {
-    return !maxValue || comparator(change.value, maxValue) <= 0;
-  });
-}
 export {
   CollectionSubscriber
 };

package/dist/esm/query/live/collection-subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport {\n  convertOrderByToBasicExpression,\n  convertToBasicExpression,\n} from \"../compiler/expressions.js\"\nimport { WhereClauseConversionError } from \"../../errors.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection/index.js\"\nimport type { ChangeMessage } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { OrderByOptimizationInfo } from \"../compiler/order-by.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\nimport type { CollectionSubscription } from \"../../collection/subscription.js\"\n\nconst loadMoreCallbackSymbol = Symbol.for(\n  `@tanstack/db.collection-config-builder`\n)\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  // Track deferred promises for subscription loading states\n  private subscriptionLoadingPromises = new Map<\n    CollectionSubscription,\n    { resolve: () => void }\n  >()\n\n  constructor(\n    private alias: string,\n    private collectionId: string,\n    private collection: Collection,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe(): CollectionSubscription {\n    const whereClause = this.getWhereClauseForAlias()\n\n    if (whereClause) {\n      const whereExpression = convertToBasicExpression(whereClause, this.alias)\n\n      if (whereExpression) {\n        return this.subscribeToChanges(whereExpression)\n      }\n\n      throw new WhereClauseConversionError(this.collectionId, this.alias)\n    }\n\n    return this.subscribeToChanges()\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let subscription: CollectionSubscription\n    const orderByInfo = this.getOrderByInfo()\n    if (orderByInfo) {\n      subscription = this.subscribeToOrderedChanges(\n        whereExpression,\n        orderByInfo\n      )\n    } else {\n      // If the source alias is lazy then we should not include the initial state\n      const includeInitialState = !this.collectionConfigBuilder.isLazyAlias(\n        this.alias\n      )\n\n      subscription = this.subscribeToMatchingChanges(\n        whereExpression,\n        includeInitialState\n      )\n    }\n\n    // Subscribe to subscription status changes to propagate loading state\n    const statusUnsubscribe = subscription.on(`status:change`, (event) => {\n      // TODO: For now we are setting this loading state whenever the subscription\n      // status changes to 'loadingSubset'. But we have discussed it only happening\n      // when the the live query has it's offset/limit changed, and that triggers the\n      // subscription to request a snapshot. This will require more work to implement,\n      // and builds on https://github.com/TanStack/db/pull/663 which this PR\n      // does not yet depend on.\n      if (event.status === `loadingSubset`) {\n        // Guard against duplicate transitions\n        if (!this.subscriptionLoadingPromises.has(subscription)) {\n          let resolve: () => void\n          const promise = new Promise<void>((res) => {\n            resolve = res\n          })\n\n          this.subscriptionLoadingPromises.set(subscription, {\n            resolve: resolve!,\n          })\n          this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(\n            promise\n          )\n        }\n      } else {\n        // status is 'ready'\n        const deferred = this.subscriptionLoadingPromises.get(subscription)\n        if (deferred) {\n          // Clear the map entry FIRST (before resolving)\n          this.subscriptionLoadingPromises.delete(subscription)\n          deferred.resolve()\n        }\n      }\n    })\n\n    const unsubscribe = () => {\n      // If subscription has a pending promise, resolve it before unsubscribing\n      const deferred = this.subscriptionLoadingPromises.get(subscription)\n      if (deferred) {\n        // Clear the map entry FIRST (before resolving)\n        this.subscriptionLoadingPromises.delete(subscription)\n        deferred.resolve()\n      }\n\n      statusUnsubscribe()\n      subscription.unsubscribe()\n    }\n    // currentSyncState is always defined when subscribe() is called\n    // (called during sync session setup)\n    this.collectionConfigBuilder.currentSyncState!.unsubscribeCallbacks.add(\n      unsubscribe\n    )\n    return subscription\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    // currentSyncState and input are always defined when this method is called\n    // (only called from active subscriptions during a sync session)\n    const input =\n      this.collectionConfigBuilder.currentSyncState!.inputs[this.alias]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to schedule a graph run even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `scheduleGraphRun`\n    this.collectionConfigBuilder.scheduleGraphRun(dataLoader, {\n      alias: this.alias,\n    })\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    includeInitialState: boolean = false\n  ) {\n    const sendChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      this.sendChangesToPipeline(changes)\n    }\n\n    const subscription = this.collection.subscribeChanges(sendChanges, {\n      includeInitialState,\n      whereExpression,\n    })\n\n    return subscription\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    orderByInfo: OrderByOptimizationInfo\n  ) {\n    const { orderBy, offset, limit, comparator, dataNeeded, index } =\n      orderByInfo\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK (and if later we need more data, we will dynamically load more data)\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded && dataNeeded() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n\n      this.sendChangesToPipelineWithTracking(filteredChanges, subscription)\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const subscription = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    subscription.setOrderByIndex(index)\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.alias\n    )\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    subscription.requestLimitedSnapshot({\n      limit: offset + limit,\n      orderBy: normalizedOrderBy,\n    })\n\n    return subscription\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded(subscription: CollectionSubscription) {\n    const orderByInfo = this.getOrderByInfo()\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    if (n > 0) {\n      this.loadNextItems(n, subscription)\n    }\n    return true\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    subscription: CollectionSubscription\n  ) {\n    const orderByInfo = this.getOrderByInfo()\n    if (!orderByInfo) {\n      this.sendChangesToPipeline(changes)\n      return\n    }\n\n    const trackedChanges = this.trackSentValues(changes, orderByInfo.comparator)\n\n    // Cache the loadMoreIfNeeded callback on the subscription using a symbol property.\n    // This ensures we pass the same function instance to the scheduler each time,\n    // allowing it to deduplicate callbacks when multiple changes arrive during a transaction.\n    type SubscriptionWithLoader = CollectionSubscription & {\n      [loadMoreCallbackSymbol]?: () => boolean\n    }\n\n    const subscriptionWithLoader = subscription as SubscriptionWithLoader\n\n    subscriptionWithLoader[loadMoreCallbackSymbol] ??=\n      this.loadMoreIfNeeded.bind(this, subscription)\n\n    this.sendChangesToPipeline(\n      trackedChanges,\n      subscriptionWithLoader[loadMoreCallbackSymbol]\n    )\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number, subscription: CollectionSubscription) {\n    const orderByInfo = this.getOrderByInfo()\n    if (!orderByInfo) {\n      return\n    }\n    const { orderBy, valueExtractorForRawRow } = orderByInfo\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.alias\n    )\n\n    // Take the `n` items after the biggest sent value\n    subscription.requestLimitedSnapshot({\n      orderBy: normalizedOrderBy,\n      limit: n,\n      minValue: biggestSentValue,\n    })\n  }\n\n  private getWhereClauseForAlias(): BasicExpression<boolean> | undefined {\n    const sourceWhereClausesCache =\n      this.collectionConfigBuilder.sourceWhereClausesCache\n    if (!sourceWhereClausesCache) {\n      return undefined\n    }\n    return sourceWhereClausesCache.get(this.alias)\n  }\n\n  private getOrderByInfo(): OrderByOptimizationInfo | undefined {\n    const info =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n    if (info && info.alias === this.alias) {\n      return info\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n\n  if (multiSetArray.length !== 0) {\n    input.sendData(new MultiSet(multiSetArray))\n  }\n\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller or equal to the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;;AAeA,MAAM,yBAAyB,OAAO;AAAA,EACpC;AACF;AAEO,MAAM,qBAGX;AAAA,EAUA,YACU,OACA,cACA,YACA,yBACR;AAJQ,SAAA,QAAA;AACA,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,0BAAA;AAZV,SAAQ,UAAe;AAGvB,SAAQ,kDAAkC,IAAA;AAAA,EAUvC;AAAA,EAEH,YAAoC;AAClC,UAAM,cAAc,KAAK,uBAAA;AAEzB,QAAI,aAAa;AACf,YAAM,kBAAkB,yBAAyB,aAAa,KAAK,KAAK;AAExE,UAAI,iBAAiB;AACnB,eAAO,KAAK,mBAAmB,eAAe;AAAA,MAChD;AAEA,YAAM,IAAI,2BAA2B,KAAK,cAAc,KAAK,KAAK;AAAA,IACpE;AAEA,WAAO,KAAK,mBAAA;AAAA,EACd;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,aAAa;AACf,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ,OAAO;AAEL,YAAM,sBAAsB,CAAC,KAAK,wBAAwB;AAAA,QACxD,KAAK;AAAA,MAAA;AAGP,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AAGA,UAAM,oBAAoB,aAAa,GAAG,iBAAiB,CAAC,UAAU;AAOpE,UAAI,MAAM,WAAW,iBAAiB;AAEpC,YAAI,CAAC,KAAK,4BAA4B,IAAI,YAAY,GAAG;AACvD,cAAI;AACJ,gBAAM,UAAU,IAAI,QAAc,CAAC,QAAQ;AACzC,sBAAU;AAAA,UACZ,CAAC;AAED,eAAK,4BAA4B,IAAI,cAAc;AAAA,YACjD;AAAA,UAAA,CACD;AACD,eAAK,wBAAwB,oBAAqB,MAAM;AAAA,YACtD;AAAA,UAAA;AAAA,QAEJ;AAAA,MACF,OAAO;AAEL,cAAM,WAAW,KAAK,4BAA4B,IAAI,YAAY;AAClE,YAAI,UAAU;AAEZ,eAAK,4BAA4B,OAAO,YAAY;AACpD,mBAAS,QAAA;AAAA,QACX;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,cAAc,MAAM;AAExB,YAAM,WAAW,KAAK,4BAA4B,IAAI,YAAY;AAClE,UAAI,UAAU;AAEZ,aAAK,4BAA4B,OAAO,YAAY;AACpD,iBAAS,QAAA;AAAA,MACX;AAEA,wBAAA;AACA,mBAAa,YAAA;AAAA,IACf;AAGA,SAAK,wBAAwB,iBAAkB,qBAAqB;AAAA,MAClE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,UACA;AAGA,UAAM,QACJ,KAAK,wBAAwB,iBAAkB,OAAO,KAAK,KAAK;AAClE,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB,iBAAiB,YAAY;AAAA,MACxD,OAAO,KAAK;AAAA,IAAA,CACb;AAAA,EACH;AAAA,EAEQ,2BACN,iBACA,sBAA+B,OAC/B;AACA,UAAM,cAAc,CAClB,YACG;AACH,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAEA,UAAM,eAAe,KAAK,WAAW,iBAAiB,aAAa;AAAA,MACjE;AAAA,MACA;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA,aACA;AACA,UAAM,EAAE,SAAS,QAAQ,OAAO,YAAY,YAAY,UACtD;AAEF,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,cAAc,WAAA,MAAiB,GAAG;AAKpC,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AAEA,WAAK,kCAAkC,iBAAiB,YAAY;AAAA,IACtE;AAIA,UAAM,eAAe,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACxE;AAAA,IAAA,CACD;AAED,iBAAa,gBAAgB,KAAK;AAGlC,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAKP,iBAAa,uBAAuB;AAAA,MAClC,OAAO,SAAS;AAAA,MAChB,SAAS;AAAA,IAAA,CACV;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAiB,cAAsC;AACrD,UAAM,cAAc,KAAK,eAAA;AAEzB,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,IAAI,GAAG;AACT,WAAK,cAAc,GAAG,YAAY;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,kCACN,SACA,cACA;AACA,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,CAAC,aAAa;AAChB,WAAK,sBAAsB,OAAO;AAClC;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,YAAY,UAAU;AAS3E,UAAM,yBAAyB;AAE/B,2BAAuB,sBAAsB,MAC3C,KAAK,iBAAiB,KAAK,MAAM,YAAY;AAE/C,SAAK;AAAA,MACH;AAAA,MACA,uBAAuB,sBAAsB;AAAA,IAAA;AAAA,EAEjD;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW,cAAsC;AACrE,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,CAAC,aAAa;AAChB;AAAA,IACF;AACA,UAAM,EAAE,SAAS,wBAAA,IAA4B;AAC7C,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAGJ,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAIP,iBAAa,uBAAuB;AAAA,MAClC,SAAS;AAAA,MACT,OAAO;AAAA,MACP,UAAU;AAAA,IAAA,CACX;AAAA,EACH;AAAA,EAEQ,yBAA+D;AACrE,UAAM,0BACJ,KAAK,wBAAwB;AAC/B,QAAI,CAAC,yBAAyB;AAC5B,aAAO;AAAA,IACT;AACA,WAAO,wBAAwB,IAAI,KAAK,KAAK;AAAA,EAC/C;AAAA,EAEQ,iBAAsD;AAC5D,UAAM,OACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,QAAI,QAAQ,KAAK,UAAU,KAAK,OAAO;AACrC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAAA,EAC5C;AAEA,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}
+{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport {\n  convertOrderByToBasicExpression,\n  convertToBasicExpression,\n} from \"../compiler/expressions.js\"\nimport { WhereClauseConversionError } from \"../../errors.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection/index.js\"\nimport type { ChangeMessage } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { OrderByOptimizationInfo } from \"../compiler/order-by.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\nimport type { CollectionSubscription } from \"../../collection/subscription.js\"\n\nconst loadMoreCallbackSymbol = Symbol.for(\n  `@tanstack/db.collection-config-builder`\n)\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  // Track deferred promises for subscription loading states\n  private subscriptionLoadingPromises = new Map<\n    CollectionSubscription,\n    { resolve: () => void }\n  >()\n\n  constructor(\n    private alias: string,\n    private collectionId: string,\n    private collection: Collection,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe(): CollectionSubscription {\n    const whereClause = this.getWhereClauseForAlias()\n\n    if (whereClause) {\n      const whereExpression = convertToBasicExpression(whereClause, this.alias)\n\n      if (whereExpression) {\n        return this.subscribeToChanges(whereExpression)\n      }\n\n      throw new WhereClauseConversionError(this.collectionId, this.alias)\n    }\n\n    return this.subscribeToChanges()\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let subscription: CollectionSubscription\n    const orderByInfo = this.getOrderByInfo()\n    if (orderByInfo) {\n      subscription = this.subscribeToOrderedChanges(\n        whereExpression,\n        orderByInfo\n      )\n    } else {\n      // If the source alias is lazy then we should not include the initial state\n      const includeInitialState = !this.collectionConfigBuilder.isLazyAlias(\n        this.alias\n      )\n\n      subscription = this.subscribeToMatchingChanges(\n        whereExpression,\n        includeInitialState\n      )\n    }\n\n    const trackLoadPromise = () => {\n      // Guard against duplicate transitions\n      if (!this.subscriptionLoadingPromises.has(subscription)) {\n        let resolve: () => void\n        const promise = new Promise<void>((res) => {\n          resolve = res\n        })\n\n        this.subscriptionLoadingPromises.set(subscription, {\n          resolve: resolve!,\n        })\n        this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(\n          promise\n        )\n      }\n    }\n\n    // It can be that we are not yet subscribed when the first `loadSubset` call happens (i.e. the initial query).\n    // So we also check the status here and if it's `loadingSubset` then we track the load promise\n    if (subscription.status === `loadingSubset`) {\n      trackLoadPromise()\n    }\n\n    // Subscribe to subscription status changes to propagate loading state\n    const statusUnsubscribe = subscription.on(`status:change`, (event) => {\n      if (event.status === `loadingSubset`) {\n        trackLoadPromise()\n      } else {\n        // status is 'ready'\n        const deferred = this.subscriptionLoadingPromises.get(subscription)\n        if (deferred) {\n          // Clear the map entry FIRST (before resolving)\n          this.subscriptionLoadingPromises.delete(subscription)\n          deferred.resolve()\n        }\n      }\n    })\n\n    const unsubscribe = () => {\n      // If subscription has a pending promise, resolve it before unsubscribing\n      const deferred = this.subscriptionLoadingPromises.get(subscription)\n      if (deferred) {\n        // Clear the map entry FIRST (before resolving)\n        this.subscriptionLoadingPromises.delete(subscription)\n        deferred.resolve()\n      }\n\n      statusUnsubscribe()\n      subscription.unsubscribe()\n    }\n    // currentSyncState is always defined when subscribe() is called\n    // (called during sync session setup)\n    this.collectionConfigBuilder.currentSyncState!.unsubscribeCallbacks.add(\n      unsubscribe\n    )\n    return subscription\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    // currentSyncState and input are always defined when this method is called\n    // (only called from active subscriptions during a sync session)\n    const input =\n      this.collectionConfigBuilder.currentSyncState!.inputs[this.alias]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to schedule a graph run even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `scheduleGraphRun`\n    this.collectionConfigBuilder.scheduleGraphRun(dataLoader, {\n      alias: this.alias,\n    })\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    includeInitialState: boolean = false\n  ) {\n    const sendChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      this.sendChangesToPipeline(changes)\n    }\n\n    const subscription = this.collection.subscribeChanges(sendChanges, {\n      includeInitialState,\n      whereExpression,\n    })\n\n    return subscription\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    orderByInfo: OrderByOptimizationInfo\n  ) {\n    const { orderBy, offset, limit, index } = orderByInfo\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      const splittedChanges = splitUpdates(changes)\n      this.sendChangesToPipelineWithTracking(splittedChanges, subscription)\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const subscription = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    subscription.setOrderByIndex(index)\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.alias\n    )\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    subscription.requestLimitedSnapshot({\n      limit: offset + limit,\n      orderBy: normalizedOrderBy,\n    })\n\n    return subscription\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded(subscription: CollectionSubscription) {\n    const orderByInfo = this.getOrderByInfo()\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    if (n > 0) {\n      this.loadNextItems(n, subscription)\n    }\n    return true\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    subscription: CollectionSubscription\n  ) {\n    const orderByInfo = this.getOrderByInfo()\n    if (!orderByInfo) {\n      this.sendChangesToPipeline(changes)\n      return\n    }\n\n    const trackedChanges = this.trackSentValues(changes, orderByInfo.comparator)\n\n    // Cache the loadMoreIfNeeded callback on the subscription using a symbol property.\n    // This ensures we pass the same function instance to the scheduler each time,\n    // allowing it to deduplicate callbacks when multiple changes arrive during a transaction.\n    type SubscriptionWithLoader = CollectionSubscription & {\n      [loadMoreCallbackSymbol]?: () => boolean\n    }\n\n    const subscriptionWithLoader = subscription as SubscriptionWithLoader\n\n    subscriptionWithLoader[loadMoreCallbackSymbol] ??=\n      this.loadMoreIfNeeded.bind(this, subscription)\n\n    this.sendChangesToPipeline(\n      trackedChanges,\n      subscriptionWithLoader[loadMoreCallbackSymbol]\n    )\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number, subscription: CollectionSubscription) {\n    const orderByInfo = this.getOrderByInfo()\n    if (!orderByInfo) {\n      return\n    }\n    const { orderBy, valueExtractorForRawRow } = orderByInfo\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.alias\n    )\n\n    // Take the `n` items after the biggest sent value\n    subscription.requestLimitedSnapshot({\n      orderBy: normalizedOrderBy,\n      limit: n,\n      minValue: biggestSentValue,\n    })\n  }\n\n  private getWhereClauseForAlias(): BasicExpression<boolean> | undefined {\n    const sourceWhereClausesCache =\n      this.collectionConfigBuilder.sourceWhereClausesCache\n    if (!sourceWhereClausesCache) {\n      return undefined\n    }\n    return sourceWhereClausesCache.get(this.alias)\n  }\n\n  private getOrderByInfo(): OrderByOptimizationInfo | undefined {\n    const info =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n    if (info && info.alias === this.alias) {\n      return info\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n\n  if (multiSetArray.length !== 0) {\n    input.sendData(new MultiSet(multiSetArray))\n  }\n\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n"],"names":[],"mappings":";;;AAeA,MAAM,yBAAyB,OAAO;AAAA,EACpC;AACF;AAEO,MAAM,qBAGX;AAAA,EAUA,YACU,OACA,cACA,YACA,yBACR;AAJQ,SAAA,QAAA;AACA,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,0BAAA;AAZV,SAAQ,UAAe;AAGvB,SAAQ,kDAAkC,IAAA;AAAA,EAUvC;AAAA,EAEH,YAAoC;AAClC,UAAM,cAAc,KAAK,uBAAA;AAEzB,QAAI,aAAa;AACf,YAAM,kBAAkB,yBAAyB,aAAa,KAAK,KAAK;AAExE,UAAI,iBAAiB;AACnB,eAAO,KAAK,mBAAmB,eAAe;AAAA,MAChD;AAEA,YAAM,IAAI,2BAA2B,KAAK,cAAc,KAAK,KAAK;AAAA,IACpE;AAEA,WAAO,KAAK,mBAAA;AAAA,EACd;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,aAAa;AACf,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ,OAAO;AAEL,YAAM,sBAAsB,CAAC,KAAK,wBAAwB;AAAA,QACxD,KAAK;AAAA,MAAA;AAGP,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,mBAAmB,MAAM;AAE7B,UAAI,CAAC,KAAK,4BAA4B,IAAI,YAAY,GAAG;AACvD,YAAI;AACJ,cAAM,UAAU,IAAI,QAAc,CAAC,QAAQ;AACzC,oBAAU;AAAA,QACZ,CAAC;AAED,aAAK,4BAA4B,IAAI,cAAc;AAAA,UACjD;AAAA,QAAA,CACD;AACD,aAAK,wBAAwB,oBAAqB,MAAM;AAAA,UACtD;AAAA,QAAA;AAAA,MAEJ;AAAA,IACF;AAIA,QAAI,aAAa,WAAW,iBAAiB;AAC3C,uBAAA;AAAA,IACF;AAGA,UAAM,oBAAoB,aAAa,GAAG,iBAAiB,CAAC,UAAU;AACpE,UAAI,MAAM,WAAW,iBAAiB;AACpC,yBAAA;AAAA,MACF,OAAO;AAEL,cAAM,WAAW,KAAK,4BAA4B,IAAI,YAAY;AAClE,YAAI,UAAU;AAEZ,eAAK,4BAA4B,OAAO,YAAY;AACpD,mBAAS,QAAA;AAAA,QACX;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,cAAc,MAAM;AAExB,YAAM,WAAW,KAAK,4BAA4B,IAAI,YAAY;AAClE,UAAI,UAAU;AAEZ,aAAK,4BAA4B,OAAO,YAAY;AACpD,iBAAS,QAAA;AAAA,MACX;AAEA,wBAAA;AACA,mBAAa,YAAA;AAAA,IACf;AAGA,SAAK,wBAAwB,iBAAkB,qBAAqB;AAAA,MAClE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,UACA;AAGA,UAAM,QACJ,KAAK,wBAAwB,iBAAkB,OAAO,KAAK,KAAK;AAClE,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB,iBAAiB,YAAY;AAAA,MACxD,OAAO,KAAK;AAAA,IAAA,CACb;AAAA,EACH;AAAA,EAEQ,2BACN,iBACA,sBAA+B,OAC/B;AACA,UAAM,cAAc,CAClB,YACG;AACH,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAEA,UAAM,eAAe,KAAK,WAAW,iBAAiB,aAAa;AAAA,MACjE;AAAA,MACA;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA,aACA;AACA,UAAM,EAAE,SAAS,QAAQ,OAAO,UAAU;AAE1C,UAAM,qBAAqB,CACzB,YACG;AAEH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,WAAK,kCAAkC,iBAAiB,YAAY;AAAA,IACtE;AAIA,UAAM,eAAe,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACxE;AAAA,IAAA,CACD;AAED,iBAAa,gBAAgB,KAAK;AAGlC,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAKP,iBAAa,uBAAuB;AAAA,MAClC,OAAO,SAAS;AAAA,MAChB,SAAS;AAAA,IAAA,CACV;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAiB,cAAsC;AACrD,UAAM,cAAc,KAAK,eAAA;AAEzB,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,IAAI,GAAG;AACT,WAAK,cAAc,GAAG,YAAY;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,kCACN,SACA,cACA;AACA,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,CAAC,aAAa;AAChB,WAAK,sBAAsB,OAAO;AAClC;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,YAAY,UAAU;AAS3E,UAAM,yBAAyB;AAE/B,2BAAuB,sBAAsB,MAC3C,KAAK,iBAAiB,KAAK,MAAM,YAAY;AAE/C,SAAK;AAAA,MACH;AAAA,MACA,uBAAuB,sBAAsB;AAAA,IAAA;AAAA,EAEjD;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW,cAAsC;AACrE,UAAM,cAAc,KAAK,eAAA;AACzB,QAAI,CAAC,aAAa;AAChB;AAAA,IACF;AACA,UAAM,EAAE,SAAS,wBAAA,IAA4B;AAC7C,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAGJ,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAIP,iBAAa,uBAAuB;AAAA,MAClC,SAAS;AAAA,MACT,OAAO;AAAA,MACP,UAAU;AAAA,IAAA,CACX;AAAA,EACH;AAAA,EAEQ,yBAA+D;AACrE,UAAM,0BACJ,KAAK,wBAAwB;AAC/B,QAAI,CAAC,yBAAyB;AAC5B,aAAO;AAAA,IACT;AACA,WAAO,wBAAwB,IAAI,KAAK,KAAK;AAAA,EAC/C;AAAA,EAEQ,iBAAsD;AAC5D,UAAM,OACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,QAAI,QAAQ,KAAK,UAAU,KAAK,OAAO;AACrC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAAA,EAC5C;AAEA,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tanstack/db",
   "description": "A reactive client store for building super fast apps on sync",
-  "version": "0.4.17",
+  "version": "0.4.18",
   "dependencies": {
     "@standard-schema/spec": "^1.0.0",
     "@tanstack/pacer": "^0.1.0",

package/src/collection/changes.ts

@@ -108,7 +108,7 @@ export class CollectionChangesManager<
     })
 
     if (options.includeInitialState) {
-      subscription.requestSnapshot()
+      subscription.requestSnapshot({ trackLoadSubsetPromise: false })
     }
 
     // Add to batched listeners

package/src/collection/subscription.ts

@@ -1,5 +1,5 @@
 import { ensureIndexForExpression } from "../indexes/auto-index.js"
-import { and, gt, lt } from "../query/builder/functions.js"
+import { and, eq, gt, lt } from "../query/builder/functions.js"
 import { Value } from "../query/ir.js"
 import { EventEmitter } from "../event-emitter.js"
 import {
@@ -20,6 +20,7 @@ import type { CollectionImpl } from "./index.js"
 type RequestSnapshotOptions = {
   where?: BasicExpression<boolean>
   optimizedOnly?: boolean
+  trackLoadSubsetPromise?: boolean
 }
 
 type RequestLimitedSnapshotOptions = {
@@ -197,7 +198,10 @@ export class CollectionSubscription
       subscription: this,
     })
 
-    this.trackLoadSubsetPromise(syncResult)
+    const trackLoadSubsetPromise = opts?.trackLoadSubsetPromise ?? true
+    if (trackLoadSubsetPromise) {
+      this.trackLoadSubsetPromise(syncResult)
+    }
 
     // Also load data immediately from the collection
     const snapshot = this.collection.currentStateAsChanges(stateOpts)
@@ -218,10 +222,12 @@ export class CollectionSubscription
   }
 
   /**
-   * Sends a snapshot that is limited to the first `limit` rows that fulfill the `where` clause and are bigger than `minValue`.
+   * Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to `minValue`.
    * Requires a range index to be set with `setOrderByIndex` prior to calling this method.
    * It uses that range index to load the items in the order of the index.
-   * Note: it does not send keys that have already been sent before.
+   * Note 1: it may load more rows than the provided LIMIT because it loads all values equal to `minValue` + limit values greater than `minValue`.
+   *         This is needed to ensure that it does not accidentally skip duplicate values when the limit falls in the middle of some duplicated values.
+   * Note 2: it does not send keys that have already been sent before.
    */
   requestLimitedSnapshot({
     orderBy,
@@ -257,12 +263,49 @@ export class CollectionSubscription
 
     let biggestObservedValue = minValue
     const changes: Array<ChangeMessage<any, string | number>> = []
-    let keys: Array<string | number> = index.take(limit, minValue, filterFn)
+
+    // If we have a minValue we need to handle the case
+    // where there might be duplicate values equal to minValue that we need to include
+    // because we can have data like this: [1, 2, 3, 3, 3, 4, 5]
+    // so if minValue is 3 then the previous snapshot may not have included all 3s
+    // e.g. if it was offset 0 and limit 3 it would only have loaded the first 3
+    //      so we load all rows equal to minValue first, to be sure we don't skip any duplicate values
+    let keys: Array<string | number> = []
+    if (minValue !== undefined) {
+      // First, get all items with the same value as minValue
+      const { expression } = orderBy[0]!
+      const allRowsWithMinValue = this.collection.currentStateAsChanges({
+        where: eq(expression, new Value(minValue)),
+      })
+
+      if (allRowsWithMinValue) {
+        const keysWithMinValue = allRowsWithMinValue
+          .map((change) => change.key)
+          .filter((key) => !this.sentKeys.has(key) && filterFn(key))
+
+        // Add items with the minValue first
+        keys.push(...keysWithMinValue)
+
+        // Then get items greater than minValue
+        const keysGreaterThanMin = index.take(
+          limit - keys.length,
+          minValue,
+          filterFn
+        )
+        keys.push(...keysGreaterThanMin)
+      } else {
+        keys = index.take(limit, minValue, filterFn)
+      }
+    } else {
+      keys = index.take(limit, minValue, filterFn)
+    }
 
     const valuesNeeded = () => Math.max(limit - changes.length, 0)
     const collectionExhausted = () => keys.length === 0
 
     while (valuesNeeded() > 0 && !collectionExhausted()) {
+      const insertedKeys = new Set<string | number>() // Track keys we add to `changes` in this iteration
+
       for (const key of keys) {
         const value = this.collection.get(key)!
         changes.push({
@@ -271,6 +314,7 @@ export class CollectionSubscription
           value,
         })
         biggestObservedValue = value
+        insertedKeys.add(key) // Track this key
       }
 
       keys = index.take(valuesNeeded(), biggestObservedValue, filterFn)
@@ -296,9 +340,41 @@ export class CollectionSubscription
       subscription: this,
     })
 
-    this.trackLoadSubsetPromise(syncResult)
+    // Make parallel loadSubset calls for values equal to minValue and values greater than minValue
+    const promises: Array<Promise<void>> = []
+
+    // First promise: load all values equal to minValue
+    if (typeof minValue !== `undefined`) {
+      const { expression } = orderBy[0]!
+      const exactValueFilter = eq(expression, new Value(minValue))
+
+      const equalValueResult = this.collection._sync.loadSubset({
+        where: exactValueFilter,
+        subscription: this,
+      })
+
+      if (equalValueResult instanceof Promise) {
+        promises.push(equalValueResult)
+      }
+    }
+
+    // Second promise: load values greater than minValue
+    if (syncResult instanceof Promise) {
+      promises.push(syncResult)
+    }
+
+    // Track the combined promise
+    if (promises.length > 0) {
+      const combinedPromise = Promise.all(promises).then(() => {})
+      this.trackLoadSubsetPromise(combinedPromise)
+    } else {
+      this.trackLoadSubsetPromise(syncResult)
+    }
   }
 
+  // TODO: also add similar test but that checks that it can also load it from the collection's loadSubset function
+  //       and that that also works properly (i.e. does not skip duplicate values)
+
   /**
    * Filters and flips changes for keys that have not been sent yet.
    * Deletes are filtered out for keys that have not been sent yet.

package/src/errors.ts

@@ -404,6 +404,22 @@ export class CollectionInputNotFoundError extends QueryCompilationError {
   }
 }
 
+/**
+ * Error thrown when a subquery uses the same alias as its parent query.
+ * This causes issues because parent and subquery would share the same input streams,
+ * leading to empty results or incorrect data (aggregation cross-leaking).
+ */
+export class DuplicateAliasInSubqueryError extends QueryCompilationError {
+  constructor(alias: string, parentAliases: Array<string>) {
+    super(
+      `Subquery uses alias "${alias}" which is already used in the parent query. ` +
+        `Each alias must be unique across parent and subquery contexts. ` +
+        `Parent query aliases: ${parentAliases.join(`, `)}. ` +
+        `Please rename "${alias}" in either the parent query or subquery to avoid conflicts.`
+    )
+  }
+}
+
 export class UnsupportedFromTypeError extends QueryCompilationError {
   constructor(type: string) {
     super(`Unsupported FROM type: ${type}`)

package/src/query/compiler/index.ts

@@ -3,6 +3,7 @@ import { optimizeQuery } from "../optimizer.js"
 import {
   CollectionInputNotFoundError,
   DistinctRequiresSelectError,
+  DuplicateAliasInSubqueryError,
   HavingRequiresGroupByError,
   LimitOffsetRequireOrderByError,
   UnsupportedFromTypeError,
@@ -99,6 +100,11 @@ export function compileQuery(
     return cachedResult
   }
 
+  // Validate the raw query BEFORE optimization to check user's original structure.
+  // This must happen before optimization because the optimizer may create internal
+  // subqueries (e.g., for predicate pushdown) that reuse aliases, which is fine.
+  validateQueryStructure(rawQuery)
+
   // Optimize the query before compilation
   const { optimizedQuery: query, sourceWhereClauses } = optimizeQuery(rawQuery)
 
@@ -375,6 +381,74 @@ export function compileQuery(
   return compilationResult
 }
 
+/**
+ * Collects aliases used for DIRECT collection references (not subqueries).
+ * Used to validate that subqueries don't reuse parent query collection aliases.
+ * Only direct CollectionRef aliases matter - QueryRef aliases don't cause conflicts.
+ */
+function collectDirectCollectionAliases(query: QueryIR): Set<string> {
+  const aliases = new Set<string>()
+
+  // Collect FROM alias only if it's a direct collection reference
+  if (query.from.type === `collectionRef`) {
+    aliases.add(query.from.alias)
+  }
+
+  // Collect JOIN aliases only for direct collection references
+  if (query.join) {
+    for (const joinClause of query.join) {
+      if (joinClause.from.type === `collectionRef`) {
+        aliases.add(joinClause.from.alias)
+      }
+    }
+  }
+
+  return aliases
+}
+
+/**
+ * Validates the structure of a query and its subqueries.
+ * Checks that subqueries don't reuse collection aliases from parent queries.
+ * This must be called on the RAW query before optimization.
+ */
+function validateQueryStructure(
+  query: QueryIR,
+  parentCollectionAliases: Set<string> = new Set()
+): void {
+  // Collect direct collection aliases from this query level
+  const currentLevelAliases = collectDirectCollectionAliases(query)
+
+  // Check if any current alias conflicts with parent aliases
+  for (const alias of currentLevelAliases) {
+    if (parentCollectionAliases.has(alias)) {
+      throw new DuplicateAliasInSubqueryError(
+        alias,
+        Array.from(parentCollectionAliases)
+      )
+    }
+  }
+
+  // Combine parent and current aliases for checking nested subqueries
+  const combinedAliases = new Set([
+    ...parentCollectionAliases,
+    ...currentLevelAliases,
+  ])
+
+  // Recursively validate FROM subquery
+  if (query.from.type === `queryRef`) {
+    validateQueryStructure(query.from.query, combinedAliases)
+  }
+
+  // Recursively validate JOIN subqueries
+  if (query.join) {
+    for (const joinClause of query.join) {
+      if (joinClause.from.type === `queryRef`) {
+        validateQueryStructure(joinClause.from.query, combinedAliases)
+      }
+    }
+  }
+}
+
 /**
  * Processes the FROM clause, handling direct collection references and subqueries.
  * Populates `aliasToCollectionId` and `aliasRemapping` for per-alias subscription tracking.

package/src/query/compiler/order-by.ts

@@ -179,13 +179,11 @@ export function processOrderBy(
           orderByOptimizationInfo
 
         setSizeCallback = (getSize: () => number) => {
-          optimizableOrderByCollections[followRefCollection.id] = {
-            ...optimizableOrderByCollections[followRefCollection.id]!,
-            dataNeeded: () => {
+          optimizableOrderByCollections[followRefCollection.id]![`dataNeeded`] =
+            () => {
               const size = getSize()
               return Math.max(0, orderByOptimizationInfo!.limit - size)
-            },
-          }
+            }
         }
       }
     }

package/src/query/live/collection-subscriber.ts

@@ -73,29 +73,33 @@ export class CollectionSubscriber<
       )
     }
 
+    const trackLoadPromise = () => {
+      // Guard against duplicate transitions
+      if (!this.subscriptionLoadingPromises.has(subscription)) {
+        let resolve: () => void
+        const promise = new Promise<void>((res) => {
+          resolve = res
+        })
+
+        this.subscriptionLoadingPromises.set(subscription, {
+          resolve: resolve!,
+        })
+        this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
+          promise
+        )
+      }
+    }
+
+    // It can be that we are not yet subscribed when the first `loadSubset` call happens (i.e. the initial query).
+    // So we also check the status here and if it's `loadingSubset` then we track the load promise
+    if (subscription.status === `loadingSubset`) {
+      trackLoadPromise()
+    }
+
     // Subscribe to subscription status changes to propagate loading state
     const statusUnsubscribe = subscription.on(`status:change`, (event) => {
-      // TODO: For now we are setting this loading state whenever the subscription
-      // status changes to 'loadingSubset'. But we have discussed it only happening
-      // when the the live query has it's offset/limit changed, and that triggers the
-      // subscription to request a snapshot. This will require more work to implement,
-      // and builds on https://github.com/TanStack/db/pull/663 which this PR
-      // does not yet depend on.
       if (event.status === `loadingSubset`) {
-        // Guard against duplicate transitions
-        if (!this.subscriptionLoadingPromises.has(subscription)) {
-          let resolve: () => void
-          const promise = new Promise<void>((res) => {
-            resolve = res
-          })
-
-          this.subscriptionLoadingPromises.set(subscription, {
-            resolve: resolve!,
-          })
-          this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
-            promise
-          )
-        }
+        trackLoadPromise()
       } else {
         // status is 'ready'
         const deferred = this.subscriptionLoadingPromises.get(subscription)
@@ -176,30 +180,14 @@ export class CollectionSubscriber<
     whereExpression: BasicExpression<boolean> | undefined,
     orderByInfo: OrderByOptimizationInfo
   ) {
-    const { orderBy, offset, limit, comparator, dataNeeded, index } =
-      orderByInfo
+    const { orderBy, offset, limit, index } = orderByInfo
 
     const sendChangesInRange = (
       changes: Iterable<ChangeMessage<any, string | number>>
     ) => {
       // Split live updates into a delete of the old value and an insert of the new value
-      // and filter out changes that are bigger than the biggest value we've sent so far
-      // because they can't affect the topK (and if later we need more data, we will dynamically load more data)
       const splittedChanges = splitUpdates(changes)
-      let filteredChanges = splittedChanges
-      if (dataNeeded && dataNeeded() === 0) {
-        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue
-        // because they can never make it into the topK.
-        // However, if the topK isn't full yet, we need to also send changes > maxSentValue
-        // because they will make it into the topK
-        filteredChanges = filterChangesSmallerOrEqualToMax(
-          splittedChanges,
-          comparator,
-          this.biggest
-        )
-      }
-
-      this.sendChangesToPipelineWithTracking(filteredChanges, subscription)
+      this.sendChangesToPipelineWithTracking(splittedChanges, subscription)
     }
 
     // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far
@@ -395,37 +383,3 @@ function* splitUpdates<
     }
   }
 }
-
-function* filterChanges<
-  T extends object = Record<string, unknown>,
-  TKey extends string | number = string | number,
->(
-  changes: Iterable<ChangeMessage<T, TKey>>,
-  f: (change: ChangeMessage<T, TKey>) => boolean
-): Generator<ChangeMessage<T, TKey>> {
-  for (const change of changes) {
-    if (f(change)) {
-      yield change
-    }
-  }
-}
-
-/**
- * Filters changes to only include those that are smaller or equal to the provided max value
- * @param changes - Iterable of changes to filter
- * @param comparator - Comparator function to use for filtering
- * @param maxValue - Range to filter changes within (range boundaries are exclusive)
- * @returns Iterable of changes that fall within the range
- */
-function* filterChangesSmallerOrEqualToMax<
-  T extends object = Record<string, unknown>,
-  TKey extends string | number = string | number,
->(
-  changes: Iterable<ChangeMessage<T, TKey>>,
-  comparator: (a: any, b: any) => number,
-  maxValue: any
-): Generator<ChangeMessage<T, TKey>> {
-  yield* filterChanges(changes, (change) => {
-    return !maxValue || comparator(change.value, maxValue) <= 0
-  })
-}