@tanstack/db 0.4.5 → 0.4.6

package/dist/esm/query/live/collection-subscriber.js

@@ -1,5 +1,5 @@
 import { MultiSet } from "@tanstack/db-ivm";
-import { convertToBasicExpression } from "../compiler/expressions.js";
+import { convertToBasicExpression, convertOrderByToBasicExpression } from "../compiler/expressions.js";
 class CollectionSubscriber {
   constructor(collectionId, collection, config, syncState, collectionConfigBuilder) {
     this.collectionId = collectionId;
@@ -8,17 +8,17 @@ class CollectionSubscriber {
     this.syncState = syncState;
     this.collectionConfigBuilder = collectionConfigBuilder;
     this.biggest = void 0;
-  }
-  subscribe() {
-    const collectionAlias = findCollectionAlias(
+    this.collectionAlias = findCollectionAlias(
       this.collectionId,
       this.collectionConfigBuilder.query
     );
-    const whereClause = this.getWhereClauseFromAlias(collectionAlias);
+  }
+  subscribe() {
+    const whereClause = this.getWhereClauseFromAlias(this.collectionAlias);
     if (whereClause) {
       const whereExpression = convertToBasicExpression(
         whereClause,
-        collectionAlias
+        this.collectionAlias
       );
       if (whereExpression) {
         return this.subscribeToChanges(whereExpression);
@@ -76,7 +76,7 @@ class CollectionSubscriber {
     return subscription;
   }
   subscribeToOrderedChanges(whereExpression) {
-    const { offset, limit, comparator, dataNeeded, index } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    const { orderBy, offset, limit, comparator, dataNeeded, index } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
     const sendChangesInRange = (changes) => {
       const splittedChanges = splitUpdates(changes);
       let filteredChanges = splittedChanges;
@@ -93,8 +93,13 @@ class CollectionSubscriber {
       whereExpression
     });
     subscription.setOrderByIndex(index);
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    );
     subscription.requestLimitedSnapshot({
-      limit: offset + limit
+      limit: offset + limit,
+      orderBy: normalizedOrderBy
     });
     return subscription;
   }
@@ -129,10 +134,15 @@ class CollectionSubscriber {
   // Loads the next `n` items from the collection
   // starting from the biggest item it has sent
   loadNextItems(n, subscription) {
-    const { valueExtractorForRawRow } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
+    const { orderBy, valueExtractorForRawRow } = this.collectionConfigBuilder.optimizableOrderByCollections[this.collectionId];
     const biggestSentRow = this.biggest;
     const biggestSentValue = biggestSentRow ? valueExtractorForRawRow(biggestSentRow) : biggestSentRow;
+    const normalizedOrderBy = convertOrderByToBasicExpression(
+      orderBy,
+      this.collectionAlias
+    );
     subscription.requestLimitedSnapshot({
+      orderBy: normalizedOrderBy,
       limit: n,
       minValue: biggestSentValue
     });

package/dist/esm/query/live/collection-subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection/index.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\nimport type { CollectionSubscription } from \"../../collection/subscription.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe(): CollectionSubscription {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        return this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      return this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let subscription: CollectionSubscription\n    if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      subscription = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      // If the collection is lazy then we should not include the initial state\n      const includeInitialState =\n        !this.collectionConfigBuilder.lazyCollections.has(this.collectionId)\n\n      subscription = this.subscribeToMatchingChanges(\n        whereExpression,\n        includeInitialState\n      )\n    }\n    const unsubscribe = () => {\n      subscription.unsubscribe()\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n    return subscription\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    includeInitialState: boolean = false\n  ) {\n    const sendChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      this.sendChangesToPipeline(changes)\n    }\n\n    const subscription = this.collection.subscribeChanges(sendChanges, {\n      includeInitialState,\n      whereExpression,\n    })\n\n    return subscription\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK (and if later we need more data, we will dynamically load more data)\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n\n      this.sendChangesToPipelineWithTracking(filteredChanges, subscription)\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const subscription = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    subscription.setOrderByIndex(index)\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    subscription.requestLimitedSnapshot({\n      limit: offset + limit,\n    })\n\n    return subscription\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded(subscription: CollectionSubscription) {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    if (n > 0) {\n      this.loadNextItems(n, subscription)\n    }\n    return true\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    subscription: CollectionSubscription\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(\n      trackedChanges,\n      this.loadMoreIfNeeded.bind(this, subscription)\n    )\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number, subscription: CollectionSubscription) {\n    const { valueExtractorForRawRow } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    subscription.requestLimitedSnapshot({\n      limit: n,\n      minValue: biggestSentValue,\n    })\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n\n  if (multiSetArray.length !== 0) {\n    input.sendData(new MultiSet(multiSetArray))\n  }\n\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller or equal to the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;AAWO,MAAM,qBAGX;AAAA,EAIA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAPV,SAAQ,UAAe;AAAA,EAQpB;AAAA,EAEH,YAAoC;AAClC,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,eAAO,KAAK,mBAAmB,eAAe;AAAA,MAChD,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,aAAO,KAAK,mBAAA;AAAA,IACd;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,qBAAe,KAAK,0BAA0B,eAAe;AAAA,IAC/D,OAAO;AAEL,YAAM,sBACJ,CAAC,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY;AAErE,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,cAAc,MAAM;AACxB,mBAAa,YAAA;AAAA,IACf;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AACnD,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEQ,2BACN,iBACA,sBAA+B,OAC/B;AACA,UAAM,cAAc,CAClB,YACG;AACH,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAEA,UAAM,eAAe,KAAK,WAAW,iBAAiB,aAAa;AAAA,MACjE;AAAA,MACA;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,YAAY,UAC7C,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AAEA,WAAK,kCAAkC,iBAAiB,YAAY;AAAA,IACtE;AAIA,UAAM,eAAe,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACxE;AAAA,IAAA,CACD;AAED,iBAAa,gBAAgB,KAAK;AAIlC,iBAAa,uBAAuB;AAAA,MAClC,OAAO,SAAS;AAAA,IAAA,CACjB;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAiB,cAAsC;AACrD,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,IAAI,GAAG;AACT,WAAK,cAAc,GAAG,YAAY;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,kCACN,SACA,cACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK;AAAA,MACH;AAAA,MACA,KAAK,iBAAiB,KAAK,MAAM,YAAY;AAAA,IAAA;AAAA,EAEjD;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW,cAAsC;AACrE,UAAM,EAAE,wBAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,iBAAa,uBAAuB;AAAA,MAClC,OAAO;AAAA,MACP,UAAU;AAAA,IAAA,CACX;AAAA,EACH;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;AAEpB,MACE,MAAM,MAAM,SAAS,mBACrB,MAAM,KAAK,YAAY,OAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,UACE,WAAW,MAAM,SAAS,mBAC1B,WAAW,KAAK,YAAY,OAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAAA,EAC5C;AAEA,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}
+{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport {\n  convertOrderByToBasicExpression,\n  convertToBasicExpression,\n} from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection/index.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\nimport type { CollectionSubscription } from \"../../collection/subscription.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  private collectionAlias: string\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {\n    this.collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )!\n  }\n\n  subscribe(): CollectionSubscription {\n    const whereClause = this.getWhereClauseFromAlias(this.collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        this.collectionAlias\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        return this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      return this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let subscription: CollectionSubscription\n    if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      subscription = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      // If the collection is lazy then we should not include the initial state\n      const includeInitialState =\n        !this.collectionConfigBuilder.lazyCollections.has(this.collectionId)\n\n      subscription = this.subscribeToMatchingChanges(\n        whereExpression,\n        includeInitialState\n      )\n    }\n    const unsubscribe = () => {\n      subscription.unsubscribe()\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n    return subscription\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined,\n    includeInitialState: boolean = false\n  ) {\n    const sendChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      this.sendChangesToPipeline(changes)\n    }\n\n    const subscription = this.collection.subscribeChanges(sendChanges, {\n      includeInitialState,\n      whereExpression,\n    })\n\n    return subscription\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { orderBy, offset, limit, comparator, dataNeeded, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK (and if later we need more data, we will dynamically load more data)\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n\n      this.sendChangesToPipelineWithTracking(filteredChanges, subscription)\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const subscription = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    subscription.setOrderByIndex(index)\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.collectionAlias\n    )\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    subscription.requestLimitedSnapshot({\n      limit: offset + limit,\n      orderBy: normalizedOrderBy,\n    })\n\n    return subscription\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded(subscription: CollectionSubscription) {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    if (n > 0) {\n      this.loadNextItems(n, subscription)\n    }\n    return true\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    subscription: CollectionSubscription\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(\n      trackedChanges,\n      this.loadMoreIfNeeded.bind(this, subscription)\n    )\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number, subscription: CollectionSubscription) {\n    const { orderBy, valueExtractorForRawRow } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n\n    // Normalize the orderBy clauses such that the references are relative to the collection\n    const normalizedOrderBy = convertOrderByToBasicExpression(\n      orderBy,\n      this.collectionAlias\n    )\n\n    // Take the `n` items after the biggest sent value\n    subscription.requestLimitedSnapshot({\n      orderBy: normalizedOrderBy,\n      limit: n,\n      minValue: biggestSentValue,\n    })\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n\n  if (multiSetArray.length !== 0) {\n    input.sendData(new MultiSet(multiSetArray))\n  }\n\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller or equal to the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;AAcO,MAAM,qBAGX;AAAA,EAMA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AATV,SAAQ,UAAe;AAWrB,SAAK,kBAAkB;AAAA,MACrB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAAA,EAEjC;AAAA,EAEA,YAAoC;AAClC,UAAM,cAAc,KAAK,wBAAwB,KAAK,eAAe;AAErE,QAAI,aAAa;AAEf,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA,KAAK;AAAA,MAAA;AAGP,UAAI,iBAAiB;AAEnB,eAAO,KAAK,mBAAmB,eAAe;AAAA,MAChD,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,aAAO,KAAK,mBAAA;AAAA,IACd;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,qBAAe,KAAK,0BAA0B,eAAe;AAAA,IAC/D,OAAO;AAEL,YAAM,sBACJ,CAAC,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY;AAErE,qBAAe,KAAK;AAAA,QAClB;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,cAAc,MAAM;AACxB,mBAAa,YAAA;AAAA,IACf;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AACnD,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEQ,2BACN,iBACA,sBAA+B,OAC/B;AACA,UAAM,cAAc,CAClB,YACG;AACH,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAEA,UAAM,eAAe,KAAK,WAAW,iBAAiB,aAAa;AAAA,MACjE;AAAA,MACA;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,SAAS,QAAQ,OAAO,YAAY,YAAY,MAAA,IACtD,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AAEA,WAAK,kCAAkC,iBAAiB,YAAY;AAAA,IACtE;AAIA,UAAM,eAAe,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACxE;AAAA,IAAA,CACD;AAED,iBAAa,gBAAgB,KAAK;AAGlC,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAKP,iBAAa,uBAAuB;AAAA,MAClC,OAAO,SAAS;AAAA,MAChB,SAAS;AAAA,IAAA,CACV;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAiB,cAAsC;AACrD,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,IAAI,GAAG;AACT,WAAK,cAAc,GAAG,YAAY;AAAA,IACpC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,kCACN,SACA,cACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK;AAAA,MACH;AAAA,MACA,KAAK,iBAAiB,KAAK,MAAM,YAAY;AAAA,IAAA;AAAA,EAEjD;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW,cAAsC;AACrE,UAAM,EAAE,SAAS,4BACf,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAGJ,UAAM,oBAAoB;AAAA,MACxB;AAAA,MACA,KAAK;AAAA,IAAA;AAIP,iBAAa,uBAAuB;AAAA,MAClC,SAAS;AAAA,MACT,OAAO;AAAA,MACP,UAAU;AAAA,IAAA,CACX;AAAA,EACH;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;AAEpB,MACE,MAAM,MAAM,SAAS,mBACrB,MAAM,KAAK,YAAY,OAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,UACE,WAAW,MAAM,SAAS,mBAC1B,WAAW,KAAK,YAAY,OAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAAA,EAC5C;AAEA,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}

package/dist/esm/types.d.ts

@@ -92,6 +92,16 @@ type Value<TExtensions = never> = string | number | boolean | bigint | null | TE
 };
 export type Row<TExtensions = never> = Record<string, Value<TExtensions>>;
 export type OperationType = `insert` | `update` | `delete`;
+export type OnLoadMoreOptions = {
+    where?: BasicExpression<boolean>;
+    orderBy?: OrderBy;
+    limit?: number;
+};
+export type CleanupFn = () => void;
+export type SyncConfigRes = {
+    cleanup?: CleanupFn;
+    onLoadMore?: (options: OnLoadMoreOptions) => void | Promise<void>;
+};
 export interface SyncConfig<T extends object = Record<string, unknown>, TKey extends string | number = string | number> {
     sync: (params: {
         collection: Collection<T, TKey, any, any, any>;
@@ -100,7 +110,7 @@ export interface SyncConfig<T extends object = Record<string, unknown>, TKey ext
         commit: () => void;
         markReady: () => void;
         truncate: () => void;
-    }) => void;
+    }) => void | CleanupFn | SyncConfigRes;
     /**
      * Get the sync metadata for insert operations
      * @returns Record containing relation information

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tanstack/db",
   "description": "A reactive client store for building super fast apps on sync",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "dependencies": {
     "@standard-schema/spec": "^1.0.0",
     "@tanstack/db-ivm": "0.1.9"

package/src/collection/change-events.ts

@@ -20,7 +20,10 @@ import type { BasicExpression } from "../query/ir.js"
 export interface CollectionLike<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-> extends Pick<Collection<T, TKey>, `get` | `has` | `entries` | `indexes`> {}
+> extends Pick<
+    Collection<T, TKey>,
+    `get` | `has` | `entries` | `indexes` | `id`
+  > {}
 
 /**
  * Returns the current state of the collection as an array of changes
@@ -109,7 +112,7 @@ export function currentStateAsChanges<
   } catch (error) {
     // If anything goes wrong with the where clause, fall back to full scan
     console.warn(
-      `Error processing where clause, falling back to full scan:`,
+      `${collection.id ? `[${collection.id}] ` : ``}Error processing where clause, falling back to full scan:`,
       error
     )
 

package/src/collection/index.ts

@@ -25,6 +25,7 @@ import type {
   InferSchemaOutput,
   InsertConfig,
   NonSingleResult,
+  OnLoadMoreOptions,
   OperationConfig,
   SingleResult,
   SubscribeChangesOptions,
@@ -362,6 +363,18 @@ export class CollectionImpl<
     this._sync.startSync()
   }
 
+  /**
+   * Requests the sync layer to load more data.
+   * @param options Options to control what data is being loaded
+   * @returns If data loading is asynchronous, this method returns a promise that resolves when the data is loaded.
+   *          If data loading is synchronous, the data is loaded when the method returns.
+   */
+  public syncMore(options: OnLoadMoreOptions): void | Promise<void> {
+    if (this._sync.syncOnLoadMoreFn) {
+      return this._sync.syncOnLoadMoreFn(options)
+    }
+  }
+
   /**
    * Preload the collection data by starting sync if not already started
    * Multiple concurrent calls will share the same promise

package/src/collection/lifecycle.ts

@@ -111,7 +111,10 @@ export class CollectionLifecycleManager<
     if (newStatus === `ready` && !this.indexes.isIndexesResolved) {
       // Resolve indexes asynchronously without blocking
       this.indexes.resolveAllIndexes().catch((error) => {
-        console.warn(`Failed to resolve indexes:`, error)
+        console.warn(
+          `${this.config.id ? `[${this.config.id}] ` : ``}Failed to resolve indexes:`,
+          error
+        )
       })
     }
 

package/src/collection/subscription.ts

@@ -1,10 +1,11 @@
 import { ensureIndexForExpression } from "../indexes/auto-index.js"
-import { and } from "../query/builder/functions.js"
+import { and, gt, lt } from "../query/builder/functions.js"
+import { Value } from "../query/ir.js"
 import {
   createFilterFunctionFromExpression,
   createFilteredCallback,
 } from "./change-events.js"
-import type { BasicExpression } from "../query/ir.js"
+import type { BasicExpression, OrderBy } from "../query/ir.js"
 import type { IndexInterface } from "../indexes/base-index.js"
 import type { ChangeMessage } from "../types.js"
 import type { CollectionImpl } from "./index.js"
@@ -15,8 +16,9 @@ type RequestSnapshotOptions = {
 }
 
 type RequestLimitedSnapshotOptions = {
-  minValue?: any
+  orderBy: OrderBy
   limit: number
+  minValue?: any
 }
 
 type CollectionSubscriptionOptions = {
@@ -117,6 +119,13 @@ export class CollectionSubscription {
       this.loadedInitialState = true
     }
 
+    // Request the sync layer to load more data
+    // don't await it, we will load the data into the collection when it comes in
+    this.collection.syncMore({
+      where: stateOpts.where,
+    })
+
+    // Also load data immediately from the collection
     const snapshot = this.collection.currentStateAsChanges(stateOpts)
 
     if (snapshot === undefined) {
@@ -140,7 +149,11 @@ export class CollectionSubscription {
    * It uses that range index to load the items in the order of the index.
    * Note: it does not send keys that have already been sent before.
    */
-  requestLimitedSnapshot({ limit, minValue }: RequestLimitedSnapshotOptions) {
+  requestLimitedSnapshot({
+    orderBy,
+    limit,
+    minValue,
+  }: RequestLimitedSnapshotOptions) {
     if (!limit) throw new Error(`limit is required`)
 
     if (!this.orderByIndex) {
@@ -190,6 +203,23 @@ export class CollectionSubscription {
     }
 
     this.callback(changes)
+
+    let whereWithValueFilter = where
+    if (typeof minValue !== `undefined`) {
+      // Only request data that we haven't seen yet (i.e. is bigger than the minValue)
+      const { expression, compareOptions } = orderBy[0]!
+      const operator = compareOptions.direction === `asc` ? gt : lt
+      const valueFilter = operator(expression, new Value(minValue))
+      whereWithValueFilter = where ? and(where, valueFilter) : valueFilter
+    }
+
+    // Request the sync layer to load more data
+    // don't await it, we will load the data into the collection when it comes in
+    this.collection.syncMore({
+      where: whereWithValueFilter,
+      limit,
+      orderBy,
+    })
   }
 
   /**

package/src/collection/sync.ts

@@ -9,7 +9,13 @@ import {
 } from "../errors"
 import { deepEquals } from "../utils"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
-import type { ChangeMessage, CollectionConfig } from "../types"
+import type {
+  ChangeMessage,
+  CleanupFn,
+  CollectionConfig,
+  OnLoadMoreOptions,
+  SyncConfigRes,
+} from "../types"
 import type { CollectionImpl } from "./index.js"
 import type { CollectionStateManager } from "./state"
 import type { CollectionLifecycleManager } from "./lifecycle"
@@ -28,6 +34,9 @@ export class CollectionSyncManager<
 
   public preloadPromise: Promise<void> | null = null
   public syncCleanupFn: (() => void) | null = null
+  public syncOnLoadMoreFn:
+    | ((options: OnLoadMoreOptions) => void | Promise<void>)
+    | null = null
 
   /**
    * Creates a new CollectionSyncManager instance
@@ -52,7 +61,6 @@ export class CollectionSyncManager<
    * This is called when the collection is first accessed or preloaded
    */
   public startSync(): void {
-    const state = this.state
     if (
       this.lifecycle.status !== `idle` &&
       this.lifecycle.status !== `cleaned-up`
@@ -63,120 +71,125 @@ export class CollectionSyncManager<
     this.lifecycle.setStatus(`loading`)
 
     try {
-      const cleanupFn = this.config.sync.sync({
-        collection: this.collection,
-        begin: () => {
-          state.pendingSyncedTransactions.push({
-            committed: false,
-            operations: [],
-            deletedKeys: new Set(),
-          })
-        },
-        write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-          const key = this.config.getKey(messageWithoutKey.value)
-
-          let messageType = messageWithoutKey.type
-
-          // Check if an item with this key already exists when inserting
-          if (messageWithoutKey.type === `insert`) {
-            const insertingIntoExistingSynced = state.syncedData.has(key)
-            const hasPendingDeleteForKey =
-              pendingTransaction.deletedKeys.has(key)
-            const isTruncateTransaction = pendingTransaction.truncate === true
-            // Allow insert after truncate in the same transaction even if it existed in syncedData
-            if (
-              insertingIntoExistingSynced &&
-              !hasPendingDeleteForKey &&
-              !isTruncateTransaction
-            ) {
-              const existingValue = state.syncedData.get(key)
+      const syncRes = normalizeSyncFnResult(
+        this.config.sync.sync({
+          collection: this.collection,
+          begin: () => {
+            this.state.pendingSyncedTransactions.push({
+              committed: false,
+              operations: [],
+              deletedKeys: new Set(),
+            })
+          },
+          write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+            const key = this.config.getKey(messageWithoutKey.value)
+
+            let messageType = messageWithoutKey.type
+
+            // Check if an item with this key already exists when inserting
+            if (messageWithoutKey.type === `insert`) {
+              const insertingIntoExistingSynced = this.state.syncedData.has(key)
+              const hasPendingDeleteForKey =
+                pendingTransaction.deletedKeys.has(key)
+              const isTruncateTransaction = pendingTransaction.truncate === true
+              // Allow insert after truncate in the same transaction even if it existed in syncedData
               if (
-                existingValue !== undefined &&
-                deepEquals(existingValue, messageWithoutKey.value)
+                insertingIntoExistingSynced &&
+                !hasPendingDeleteForKey &&
+                !isTruncateTransaction
               ) {
-                // The "insert" is an echo of a value we already have locally.
-                // Treat it as an update so we preserve optimistic intent without
-                // throwing a duplicate-key error during reconciliation.
-                messageType = `update`
-              } else {
-                throw new DuplicateKeySyncError(key, this.id)
+                const existingValue = this.state.syncedData.get(key)
+                if (
+                  existingValue !== undefined &&
+                  deepEquals(existingValue, messageWithoutKey.value)
+                ) {
+                  // The "insert" is an echo of a value we already have locally.
+                  // Treat it as an update so we preserve optimistic intent without
+                  // throwing a duplicate-key error during reconciliation.
+                  messageType = `update`
+                } else {
+                  throw new DuplicateKeySyncError(key, this.id)
+                }
               }
             }
-          }
-
-          const message: ChangeMessage<TOutput> = {
-            ...messageWithoutKey,
-            type: messageType,
-            key,
-          }
-          pendingTransaction.operations.push(message)
-
-          if (messageType === `delete`) {
-            pendingTransaction.deletedKeys.add(key)
-          }
-        },
-        commit: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionCommitError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedError()
-          }
-
-          pendingTransaction.committed = true
-
-          // Update status to initialCommit when transitioning from loading
-          // This indicates we're in the process of committing the first transaction
-          if (this.lifecycle.status === `loading`) {
-            this.lifecycle.setStatus(`initialCommit`)
-          }
-
-          state.commitPendingTransactions()
-        },
-        markReady: () => {
-          this.lifecycle.markReady()
-        },
-        truncate: () => {
-          const pendingTransaction =
-            state.pendingSyncedTransactions[
-              state.pendingSyncedTransactions.length - 1
-            ]
-          if (!pendingTransaction) {
-            throw new NoPendingSyncTransactionWriteError()
-          }
-          if (pendingTransaction.committed) {
-            throw new SyncTransactionAlreadyCommittedWriteError()
-          }
-
-          // Clear all operations from the current transaction
-          pendingTransaction.operations = []
-          pendingTransaction.deletedKeys.clear()
-
-          // Mark the transaction as a truncate operation. During commit, this triggers:
-          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
-          // - Clearing of syncedData/syncedMetadata
-          // - Subsequent synced ops applied on the fresh base
-          // - Finally, optimistic mutations re-applied on top (single batch)
-          pendingTransaction.truncate = true
-        },
-      })
+
+            const message: ChangeMessage<TOutput> = {
+              ...messageWithoutKey,
+              type: messageType,
+              key,
+            }
+            pendingTransaction.operations.push(message)
+
+            if (messageType === `delete`) {
+              pendingTransaction.deletedKeys.add(key)
+            }
+          },
+          commit: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionCommitError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedError()
+            }
+
+            pendingTransaction.committed = true
+
+            // Update status to initialCommit when transitioning from loading
+            // This indicates we're in the process of committing the first transaction
+            if (this.lifecycle.status === `loading`) {
+              this.lifecycle.setStatus(`initialCommit`)
+            }
+
+            this.state.commitPendingTransactions()
+          },
+          markReady: () => {
+            this.lifecycle.markReady()
+          },
+          truncate: () => {
+            const pendingTransaction =
+              this.state.pendingSyncedTransactions[
+                this.state.pendingSyncedTransactions.length - 1
+              ]
+            if (!pendingTransaction) {
+              throw new NoPendingSyncTransactionWriteError()
+            }
+            if (pendingTransaction.committed) {
+              throw new SyncTransactionAlreadyCommittedWriteError()
+            }
+
+            // Clear all operations from the current transaction
+            pendingTransaction.operations = []
+            pendingTransaction.deletedKeys.clear()
+
+            // Mark the transaction as a truncate operation. During commit, this triggers:
+            // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
+            // - Clearing of syncedData/syncedMetadata
+            // - Subsequent synced ops applied on the fresh base
+            // - Finally, optimistic mutations re-applied on top (single batch)
+            pendingTransaction.truncate = true
+          },
+        })
+      )
 
       // Store cleanup function if provided
-      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null
+      this.syncCleanupFn = syncRes?.cleanup ?? null
+
+      // Store onLoadMore function if provided
+      this.syncOnLoadMoreFn = syncRes?.onLoadMore ?? null
     } catch (error) {
       this.lifecycle.setStatus(`error`)
       throw error
@@ -225,6 +238,18 @@ export class CollectionSyncManager<
     return this.preloadPromise
   }
 
+  /**
+   * Requests the sync layer to load more data.
+   * @param options Options to control what data is being loaded
+   * @returns If data loading is asynchronous, this method returns a promise that resolves when the data is loaded.
+   *          If data loading is synchronous, the data is loaded when the method returns.
+   */
+  public syncMore(options: OnLoadMoreOptions): void | Promise<void> {
+    if (this.syncOnLoadMoreFn) {
+      return this.syncOnLoadMoreFn(options)
+    }
+  }
+
   public cleanup(): void {
     try {
       if (this.syncCleanupFn) {
@@ -248,3 +273,15 @@ export class CollectionSyncManager<
     this.preloadPromise = null
   }
 }
+
+function normalizeSyncFnResult(result: void | CleanupFn | SyncConfigRes) {
+  if (typeof result === `function`) {
+    return { cleanup: result }
+  }
+
+  if (typeof result === `object`) {
+    return result
+  }
+
+  return undefined
+}

package/src/index.ts

@@ -1,4 +1,8 @@
 // Re-export all public APIs
+// Re-export IR types under their own namespace
+// because custom collections need access to the IR types
+import * as IR from "./query/ir.js"
+
 export * from "./collection/index.js"
 export * from "./SortedMap"
 export * from "./transactions"
@@ -18,3 +22,4 @@ export { type IndexOptions } from "./indexes/index-options.js"
 
 // Re-export some stuff explicitly to ensure the type & value is exported
 export type { Collection } from "./collection/index.js"
+export { IR }