@tanstack/db 0.1.1 → 0.1.4

package/dist/cjs/query/compiler/joins.cjs

@@ -2,32 +2,60 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const dbIvm = require("@tanstack/db-ivm");
 const errors = require("../../errors.cjs");
+const indexOptimization = require("../../utils/index-optimization.cjs");
+const autoIndex = require("../../indexes/auto-index.cjs");
 const evaluators = require("./evaluators.cjs");
 const index = require("./index.cjs");
-function processJoins(pipeline, joinClauses, tables, mainTableAlias, allInputs, cache, queryMapping) {
+function processJoins(pipeline, joinClauses, tables, mainTableId, mainTableAlias, allInputs, cache, queryMapping, collections, callbacks, lazyCollections, optimizableOrderByCollections, rawQuery) {
   let resultPipeline = pipeline;
   for (const joinClause of joinClauses) {
     resultPipeline = processJoin(
       resultPipeline,
       joinClause,
       tables,
+      mainTableId,
       mainTableAlias,
       allInputs,
       cache,
-      queryMapping
+      queryMapping,
+      collections,
+      callbacks,
+      lazyCollections,
+      optimizableOrderByCollections,
+      rawQuery
     );
   }
   return resultPipeline;
 }
-function processJoin(pipeline, joinClause, tables, mainTableAlias, allInputs, cache, queryMapping) {
-  const { alias: joinedTableAlias, input: joinedInput } = processJoinSource(
+function processJoin(pipeline, joinClause, tables, mainTableId, mainTableAlias, allInputs, cache, queryMapping, collections, callbacks, lazyCollections, optimizableOrderByCollections, rawQuery) {
+  const {
+    alias: joinedTableAlias,
+    input: joinedInput,
+    collectionId: joinedCollectionId
+  } = processJoinSource(
     joinClause.from,
     allInputs,
+    collections,
+    callbacks,
+    lazyCollections,
+    optimizableOrderByCollections,
     cache,
     queryMapping
   );
   tables[joinedTableAlias] = joinedInput;
-  const joinType = joinClause.type === `cross` ? `inner` : joinClause.type === `outer` ? `full` : joinClause.type;
+  const mainCollection = collections[mainTableId];
+  const joinedCollection = collections[joinedCollectionId];
+  if (!mainCollection) {
+    throw new errors.JoinCollectionNotFoundError(mainTableId);
+  }
+  if (!joinedCollection) {
+    throw new errors.JoinCollectionNotFoundError(joinedCollectionId);
+  }
+  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(
+    joinClause.type,
+    mainCollection,
+    joinedCollection
+  );
   const { mainExpr, joinedExpr } = analyzeJoinExpressions(
     joinClause.left,
     joinClause.right,
@@ -36,24 +64,72 @@ function processJoin(pipeline, joinClause, tables, mainTableAlias, allInputs, ca
   );
   const compiledMainExpr = evaluators.compileExpression(mainExpr);
   const compiledJoinedExpr = evaluators.compileExpression(joinedExpr);
-  const mainPipeline = pipeline.pipe(
+  let mainPipeline = pipeline.pipe(
     dbIvm.map(([currentKey, namespacedRow]) => {
       const mainKey = compiledMainExpr(namespacedRow);
       return [mainKey, [currentKey, namespacedRow]];
     })
   );
-  const joinedPipeline = joinedInput.pipe(
+  let joinedPipeline = joinedInput.pipe(
     dbIvm.map(([currentKey, row]) => {
       const namespacedRow = { [joinedTableAlias]: row };
       const joinedKey = compiledJoinedExpr(namespacedRow);
       return [joinedKey, [currentKey, namespacedRow]];
     })
   );
-  if (![`inner`, `left`, `right`, `full`].includes(joinType)) {
+  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {
     throw new errors.UnsupportedJoinTypeError(joinClause.type);
   }
+  if (activeCollection) {
+    lazyCollections.add(lazyCollection.id);
+    const activePipeline = activeCollection === `main` ? mainPipeline : joinedPipeline;
+    let index$1;
+    const lazyCollectionJoinExpr = activeCollection === `main` ? joinedExpr : mainExpr;
+    const activeColl = activeCollection === `main` ? collections[mainTableId] : lazyCollection;
+    const followRefResult = index.followRef(
+      rawQuery,
+      lazyCollectionJoinExpr,
+      activeColl
+    );
+    const followRefCollection = followRefResult.collection;
+    const fieldName = followRefResult.path[0];
+    if (fieldName) {
+      autoIndex.ensureIndexForField(fieldName, followRefResult.path, followRefCollection);
+    }
+    let deoptimized = false;
+    const activePipelineWithLoading = activePipeline.pipe(
+      dbIvm.tap(([joinKey, _]) => {
+        if (deoptimized) {
+          return;
+        }
+        index$1 ?? (index$1 = indexOptimization.findIndexForField(
+          followRefCollection.indexes,
+          followRefResult.path
+        ));
+        const collectionCallbacks = callbacks[lazyCollection.id];
+        if (!collectionCallbacks) {
+          throw new Error(
+            `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`
+          );
+        }
+        const { loadKeys, loadInitialState } = collectionCallbacks;
+        if (index$1 && index$1.supports(`eq`)) {
+          const matchingKeys = index$1.lookup(`eq`, joinKey);
+          loadKeys(matchingKeys);
+        } else {
+          deoptimized = true;
+          loadInitialState();
+        }
+      })
+    );
+    if (activeCollection === `main`) {
+      mainPipeline = activePipelineWithLoading;
+    } else {
+      joinedPipeline = activePipelineWithLoading;
+    }
+  }
   return mainPipeline.pipe(
-    dbIvm.join(joinedPipeline, joinType),
+    dbIvm.join(joinedPipeline, joinClause.type),
     dbIvm.consolidate(),
     processJoinResults(joinClause.type)
   );
@@ -101,20 +177,24 @@ function getTableAliasFromExpression(expr) {
       return null;
   }
 }
-function processJoinSource(from, allInputs, cache, queryMapping) {
+function processJoinSource(from, allInputs, collections, callbacks, lazyCollections, optimizableOrderByCollections, cache, queryMapping) {
   switch (from.type) {
     case `collectionRef`: {
       const input = allInputs[from.collection.id];
       if (!input) {
         throw new errors.CollectionInputNotFoundError(from.collection.id);
       }
-      return { alias: from.alias, input };
+      return { alias: from.alias, input, collectionId: from.collection.id };
     }
     case `queryRef`: {
       const originalQuery = queryMapping.get(from.query) || from.query;
       const subQueryResult = index.compileQuery(
         originalQuery,
         allInputs,
+        collections,
+        callbacks,
+        lazyCollections,
+        optimizableOrderByCollections,
         cache,
         queryMapping
       );
@@ -125,7 +205,11 @@ function processJoinSource(from, allInputs, cache, queryMapping) {
           return [key, value];
         })
       );
-      return { alias: from.alias, input: extractedInput };
+      return {
+        alias: from.alias,
+        input: extractedInput,
+        collectionId: subQueryResult.collectionId
+      };
     }
     default:
       throw new errors.UnsupportedJoinSourceTypeError(from.type);
@@ -169,5 +253,20 @@ function processJoinResults(joinType) {
     );
   };
 }
+function getActiveAndLazyCollections(joinType, leftCollection, rightCollection) {
+  if (leftCollection.id === rightCollection.id) {
+    return { activeCollection: void 0, lazyCollection: void 0 };
+  }
+  switch (joinType) {
+    case `left`:
+      return { activeCollection: `main`, lazyCollection: rightCollection };
+    case `right`:
+      return { activeCollection: `joined`, lazyCollection: leftCollection };
+    case `inner`:
+      return leftCollection.size < rightCollection.size ? { activeCollection: `main`, lazyCollection: rightCollection } : { activeCollection: `joined`, lazyCollection: leftCollection };
+    default:
+      return { activeCollection: void 0, lazyCollection: void 0 };
+  }
+}
 exports.processJoins = processJoins;
 //# sourceMappingURL=joins.cjs.map

package/dist/cjs/query/compiler/joins.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"joins.cjs","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  InvalidJoinConditionWrongTablesError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery } from \"./index.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  QueryRef,\n} from \"../ir.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const { alias: joinedTableAlias, input: joinedInput } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  // Convert join type to D2 join type\n  const joinType: JoinType =\n    joinClause.type === `cross`\n      ? `inner`\n      : joinClause.type === `outer`\n        ? `full`\n        : (joinClause.type as JoinType)\n\n  // Analyze which table each expression refers to and swap if necessary\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    mainTableAlias,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  const mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  const joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinType)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (main table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  mainTableAlias: string,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to main table and right refers to joined table, keep as is\n  if (\n    leftTableAlias === mainTableAlias &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to main table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias === mainTableAlias\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias || `unknown`)\n  }\n\n  // If one expression doesn't refer to either table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    throw new InvalidJoinConditionTableMismatchError(\n      mainTableAlias,\n      joinedTableAlias\n    )\n  }\n\n  // If expressions refer to tables not involved in this join, this is an invalid join\n  throw new InvalidJoinConditionWrongTablesError(\n    leftTableAlias,\n    rightTableAlias,\n    mainTableAlias,\n    joinedTableAlias\n  )\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return { alias: from.alias, input: extractedInput as KeyedStream }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n"],"names":["compileExpression","map","UnsupportedJoinTypeError","joinOperator","consolidate","InvalidJoinConditionSameTableError","InvalidJoinConditionTableMismatchError","InvalidJoinConditionWrongTablesError","CollectionInputNotFoundError","compileQuery","UnsupportedJoinSourceTypeError","filter"],"mappings":";;;;;;AAiCO,SAAS,aACd,UACA,aACA,QACA,gBACA,WACA,OACA,cAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,gBACA,WACA,OACA,cAC0B;AAE1B,QAAM,EAAE,OAAO,kBAAkB,OAAO,gBAAgB;AAAA,IACtD,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAG3B,QAAM,WACJ,WAAW,SAAS,UAChB,UACA,WAAW,SAAS,UAClB,SACC,WAAW;AAGpB,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmBA,WAAAA,kBAAkB,QAAQ;AACnD,QAAM,qBAAqBA,WAAAA,kBAAkB,UAAU;AAGvD,QAAM,eAAe,SAAS;AAAA,IAC5BC,MAAAA,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,QAAM,iBAAiB,YAAY;AAAA,IACjCA,MAAAA,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,QAAQ,GAAG;AAC1D,UAAM,IAAIC,OAAAA,yBAAyB,WAAW,IAAI;AAAA,EACpD;AACA,SAAO,aAAa;AAAA,IAClBC,MAAAA,KAAa,gBAAgB,QAAQ;AAAA,IACrCC,kBAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,gBACA,kBAC4D;AAC5D,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,mBAAmB,kBACnB,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,oBAAoB,gBACpB;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAIC,OAAAA,mCAAmC,kBAAkB,SAAS;AAAA,EAC1E;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AACvC,UAAM,IAAIC,OAAAA;AAAAA,MACR;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAGA,QAAM,IAAIC,OAAAA;AAAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,OACA,cACuC;AACvC,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAIC,OAAAA,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,MAAA;AAAA,IAC9B;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiBC,MAAAA;AAAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnCR,MAAAA,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,eAAA;AAAA,IACrC;AAAA,IACA;AACE,YAAM,IAAIS,OAAAA,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEdC,MAAAA,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACDV,MAAAA,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;;"}
+{"version":3,"file":"joins.cjs","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n  tap,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  InvalidJoinConditionWrongTablesError,\n  JoinCollectionNotFoundError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery, followRef } from \"./index.js\"\nimport type { OrderByOptimizationInfo } from \"./order-by.js\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  PropRef,\n  QueryIR,\n  QueryRef,\n} from \"../ir.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\n\nexport type LoadKeysFn = (key: Set<string | number>) => void\nexport type LazyCollectionCallbacks = {\n  loadKeys: LoadKeysFn\n  loadInitialState: () => void\n}\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableId,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping,\n      collections,\n      callbacks,\n      lazyCollections,\n      optimizableOrderByCollections,\n      rawQuery\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const {\n    alias: joinedTableAlias,\n    input: joinedInput,\n    collectionId: joinedCollectionId,\n  } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    collections,\n    callbacks,\n    lazyCollections,\n    optimizableOrderByCollections,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  const mainCollection = collections[mainTableId]\n  const joinedCollection = collections[joinedCollectionId]\n\n  if (!mainCollection) {\n    throw new JoinCollectionNotFoundError(mainTableId)\n  }\n\n  if (!joinedCollection) {\n    throw new JoinCollectionNotFoundError(joinedCollectionId)\n  }\n\n  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(\n    joinClause.type,\n    mainCollection,\n    joinedCollection\n  )\n\n  // Analyze which table each expression refers to and swap if necessary\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    mainTableAlias,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  let mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  let joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n\n  if (activeCollection) {\n    // This join can be optimized by having the active collection\n    // dynamically load keys into the lazy collection\n    // based on the value of the joinKey and by looking up\n    // matching rows in the index of the lazy collection\n\n    // Mark the lazy collection as lazy\n    // this Set is passed by the liveQueryCollection to the compiler\n    // such that the liveQueryCollection can check it after compilation\n    // to know which collections are lazy collections\n    lazyCollections.add(lazyCollection.id)\n\n    const activePipeline =\n      activeCollection === `main` ? mainPipeline : joinedPipeline\n\n    let index: BaseIndex<string | number> | undefined\n\n    const lazyCollectionJoinExpr =\n      activeCollection === `main`\n        ? (joinedExpr as PropRef)\n        : (mainExpr as PropRef)\n\n    const activeColl =\n      activeCollection === `main` ? collections[mainTableId]! : lazyCollection\n\n    const followRefResult = followRef(\n      rawQuery,\n      lazyCollectionJoinExpr,\n      activeColl\n    )!\n    const followRefCollection = followRefResult.collection\n\n    const fieldName = followRefResult.path[0]\n    if (fieldName) {\n      ensureIndexForField(fieldName, followRefResult.path, followRefCollection)\n    }\n\n    let deoptimized = false\n\n    const activePipelineWithLoading: IStreamBuilder<\n      [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]\n    > = activePipeline.pipe(\n      tap(([joinKey, _]) => {\n        if (deoptimized) {\n          return\n        }\n\n        // Find the index for the path we join on\n        // we need to find the index inside the map operator\n        // because the indexes are only available after the initial sync\n        // so we can't fetch it during compilation\n        index ??= findIndexForField(\n          followRefCollection.indexes,\n          followRefResult.path\n        )\n\n        // The `callbacks` object is passed by the liveQueryCollection to the compiler.\n        // It contains a function to lazy load keys for each lazy collection\n        // as well as a function to switch back to a regular collection\n        // (useful when there's no index for available for lazily loading the collection)\n        const collectionCallbacks = callbacks[lazyCollection.id]\n        if (!collectionCallbacks) {\n          throw new Error(\n            `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`\n          )\n        }\n\n        const { loadKeys, loadInitialState } = collectionCallbacks\n\n        if (index && index.supports(`eq`)) {\n          // Use the index to fetch the PKs of the rows in the lazy collection\n          // that match this row from the active collection based on the value of the joinKey\n          const matchingKeys = index.lookup(`eq`, joinKey)\n          // Inform the lazy collection that those keys need to be loaded\n          loadKeys(matchingKeys)\n        } else {\n          // We can't optimize the join because there is no index for the join key\n          // on the lazy collection, so we load the initial state\n          deoptimized = true\n          loadInitialState()\n        }\n      })\n    )\n\n    if (activeCollection === `main`) {\n      mainPipeline = activePipelineWithLoading\n    } else {\n      joinedPipeline = activePipelineWithLoading\n    }\n  }\n\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinClause.type as JoinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (main table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  mainTableAlias: string,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to main table and right refers to joined table, keep as is\n  if (\n    leftTableAlias === mainTableAlias &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to main table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias === mainTableAlias\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias || `unknown`)\n  }\n\n  // If one expression doesn't refer to either table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    throw new InvalidJoinConditionTableMismatchError(\n      mainTableAlias,\n      joinedTableAlias\n    )\n  }\n\n  // If expressions refer to tables not involved in this join, this is an invalid join\n  throw new InvalidJoinConditionWrongTablesError(\n    leftTableAlias,\n    rightTableAlias,\n    mainTableAlias,\n    joinedTableAlias\n  )\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream; collectionId: string } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input, collectionId: from.collection.id }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        collections,\n        callbacks,\n        lazyCollections,\n        optimizableOrderByCollections,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return {\n        alias: from.alias,\n        input: extractedInput as KeyedStream,\n        collectionId: subQueryResult.collectionId,\n      }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n\n/**\n * Returns the active and lazy collections for a join clause.\n * The active collection is the one that we need to fully iterate over\n * and it can be the main table (i.e. left collection) or the joined table (i.e. right collection).\n * The lazy collection is the one that we should join-in lazily based on matches in the active collection.\n * @param joinClause - The join clause to analyze\n * @param leftCollection - The left collection\n * @param rightCollection - The right collection\n * @returns The active and lazy collections. They are undefined if we need to loop over both collections (i.e. both are active)\n */\nfunction getActiveAndLazyCollections(\n  joinType: JoinClause[`type`],\n  leftCollection: Collection,\n  rightCollection: Collection\n):\n  | { activeCollection: `main` | `joined`; lazyCollection: Collection }\n  | { activeCollection: undefined; lazyCollection: undefined } {\n  if (leftCollection.id === rightCollection.id) {\n    // We can't apply this optimization if there's only one collection\n    // because `liveQueryCollection` will detect that the collection is lazy\n    // and treat it lazily (because the collection is shared)\n    // and thus it will not load any keys because both sides of the join\n    // will be handled lazily\n    return { activeCollection: undefined, lazyCollection: undefined }\n  }\n\n  switch (joinType) {\n    case `left`:\n      return { activeCollection: `main`, lazyCollection: rightCollection }\n    case `right`:\n      return { activeCollection: `joined`, lazyCollection: leftCollection }\n    case `inner`:\n      // The smallest collection should be the active collection\n      // and the biggest collection should be lazy\n      return leftCollection.size < rightCollection.size\n        ? { activeCollection: `main`, lazyCollection: rightCollection }\n        : { activeCollection: `joined`, lazyCollection: leftCollection }\n    default:\n      return { activeCollection: undefined, lazyCollection: undefined }\n  }\n}\n"],"names":["JoinCollectionNotFoundError","compileExpression","map","UnsupportedJoinTypeError","index","followRef","ensureIndexForField","tap","findIndexForField","joinOperator","consolidate","InvalidJoinConditionSameTableError","InvalidJoinConditionTableMismatchError","InvalidJoinConditionWrongTablesError","CollectionInputNotFoundError","compileQuery","UnsupportedJoinSourceTypeError","filter"],"mappings":";;;;;;;;AAgDO,SAAS,aACd,UACA,aACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAE1B,QAAM;AAAA,IACJ,OAAO;AAAA,IACP,OAAO;AAAA,IACP,cAAc;AAAA,EAAA,IACZ;AAAA,IACF,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAE3B,QAAM,iBAAiB,YAAY,WAAW;AAC9C,QAAM,mBAAmB,YAAY,kBAAkB;AAEvD,MAAI,CAAC,gBAAgB;AACnB,UAAM,IAAIA,OAAAA,4BAA4B,WAAW;AAAA,EACnD;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAIA,OAAAA,4BAA4B,kBAAkB;AAAA,EAC1D;AAEA,QAAM,EAAE,kBAAkB,eAAA,IAAmB;AAAA,IAC3C,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmBC,WAAAA,kBAAkB,QAAQ;AACnD,QAAM,qBAAqBA,WAAAA,kBAAkB,UAAU;AAGvD,MAAI,eAAe,SAAS;AAAA,IAC1BC,MAAAA,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,MAAI,iBAAiB,YAAY;AAAA,IAC/BA,MAAAA,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,WAAW,IAAI,GAAG;AACjE,UAAM,IAAIC,OAAAA,yBAAyB,WAAW,IAAI;AAAA,EACpD;AAEA,MAAI,kBAAkB;AAUpB,oBAAgB,IAAI,eAAe,EAAE;AAErC,UAAM,iBACJ,qBAAqB,SAAS,eAAe;AAE/C,QAAIC;AAEJ,UAAM,yBACJ,qBAAqB,SAChB,aACA;AAEP,UAAM,aACJ,qBAAqB,SAAS,YAAY,WAAW,IAAK;AAE5D,UAAM,kBAAkBC,MAAAA;AAAAA,MACtB;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,sBAAsB,gBAAgB;AAE5C,UAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,QAAI,WAAW;AACbC,gBAAAA,oBAAoB,WAAW,gBAAgB,MAAM,mBAAmB;AAAA,IAC1E;AAEA,QAAI,cAAc;AAElB,UAAM,4BAEF,eAAe;AAAA,MACjBC,MAAAA,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM;AACpB,YAAI,aAAa;AACf;AAAA,QACF;AAMAH,8BAAUI,kBAAAA;AAAAA,UACR,oBAAoB;AAAA,UACpB,gBAAgB;AAAA,QAAA;AAOlB,cAAM,sBAAsB,UAAU,eAAe,EAAE;AACvD,YAAI,CAAC,qBAAqB;AACxB,gBAAM,IAAI;AAAA,YACR;AAAA,UAAA;AAAA,QAEJ;AAEA,cAAM,EAAE,UAAU,iBAAA,IAAqB;AAEvC,YAAIJ,WAASA,QAAM,SAAS,IAAI,GAAG;AAGjC,gBAAM,eAAeA,QAAM,OAAO,MAAM,OAAO;AAE/C,mBAAS,YAAY;AAAA,QACvB,OAAO;AAGL,wBAAc;AACd,2BAAA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IAAA;AAGH,QAAI,qBAAqB,QAAQ;AAC/B,qBAAe;AAAA,IACjB,OAAO;AACL,uBAAiB;AAAA,IACnB;AAAA,EACF;AAEA,SAAO,aAAa;AAAA,IAClBK,WAAa,gBAAgB,WAAW,IAAgB;AAAA,IACxDC,kBAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,gBACA,kBAC4D;AAC5D,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,mBAAmB,kBACnB,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,oBAAoB,gBACpB;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAIC,OAAAA,mCAAmC,kBAAkB,SAAS;AAAA,EAC1E;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AACvC,UAAM,IAAIC,OAAAA;AAAAA,MACR;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAGA,QAAM,IAAIC,OAAAA;AAAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,aACA,WACA,iBACA,+BACA,OACA,cAC6D;AAC7D,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAIC,OAAAA,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,cAAc,KAAK,WAAW,GAAA;AAAA,IACnE;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiBC,MAAAA;AAAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnCb,MAAAA,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO;AAAA,QACL,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,cAAc,eAAe;AAAA,MAAA;AAAA,IAEjC;AAAA,IACA;AACE,YAAM,IAAIc,OAAAA,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEdC,MAAAA,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACDf,MAAAA,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAYA,SAAS,4BACP,UACA,gBACA,iBAG6D;AAC7D,MAAI,eAAe,OAAO,gBAAgB,IAAI;AAM5C,WAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EACxD;AAEA,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA;AAAA,IACrD,KAAK;AACH,aAAO,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACvD,KAAK;AAGH,aAAO,eAAe,OAAO,gBAAgB,OACzC,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA,IAC5C,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACpD;AACE,aAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EAAU;AAEtE;;"}

package/dist/cjs/query/compiler/joins.d.cts

@@ -1,7 +1,14 @@
-import { JoinClause } from '../ir.js';
+import { OrderByOptimizationInfo } from './order-by.js';
+import { JoinClause, QueryIR } from '../ir.js';
+import { Collection } from '../../collection.js';
 import { KeyedStream, NamespacedAndKeyedStream } from '../../types.js';
 import { QueryCache, QueryMapping } from './types.js';
+export type LoadKeysFn = (key: Set<string | number>) => void;
+export type LazyCollectionCallbacks = {
+    loadKeys: LoadKeysFn;
+    loadInitialState: () => void;
+};
 /**
  * Processes all join clauses in a query
  */
-export declare function processJoins(pipeline: NamespacedAndKeyedStream, joinClauses: Array<JoinClause>, tables: Record<string, KeyedStream>, mainTableAlias: string, allInputs: Record<string, KeyedStream>, cache: QueryCache, queryMapping: QueryMapping): NamespacedAndKeyedStream;
+export declare function processJoins(pipeline: NamespacedAndKeyedStream, joinClauses: Array<JoinClause>, tables: Record<string, KeyedStream>, mainTableId: string, mainTableAlias: string, allInputs: Record<string, KeyedStream>, cache: QueryCache, queryMapping: QueryMapping, collections: Record<string, Collection>, callbacks: Record<string, LazyCollectionCallbacks>, lazyCollections: Set<string>, optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>, rawQuery: QueryIR): NamespacedAndKeyedStream;

package/dist/cjs/query/compiler/order-by.cjs

@@ -2,11 +2,15 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const dbIvm = require("@tanstack/db-ivm");
 const comparison = require("../../utils/comparison.cjs");
+const ir = require("../ir.cjs");
+const autoIndex = require("../../indexes/auto-index.cjs");
+const indexOptimization = require("../../utils/index-optimization.cjs");
 const evaluators = require("./evaluators.cjs");
-function processOrderBy(pipeline, orderByClause, limit, offset) {
+const index = require("./index.cjs");
+function processOrderBy(rawQuery, pipeline, orderByClause, collection, optimizableOrderByCollections, limit, offset) {
   const compiledOrderBy = orderByClause.map((clause) => ({
     compiledExpression: evaluators.compileExpression(clause.expression),
-    direction: clause.direction
+    compareOptions: clause.compareOptions
   }));
   const valueExtractor = (row) => {
     const orderByContext = { ...row };
@@ -23,34 +27,87 @@ function processOrderBy(pipeline, orderByClause, limit, offset) {
     }
     return null;
   };
-  const makeComparator = () => {
-    return (a, b) => {
-      if (orderByClause.length > 1) {
-        const arrayA = a;
-        const arrayB = b;
-        for (let i = 0; i < orderByClause.length; i++) {
-          const direction = orderByClause[i].direction;
-          const compareFn = direction === `desc` ? comparison.descComparator : comparison.ascComparator;
-          const result = compareFn(arrayA[i], arrayB[i]);
-          if (result !== 0) {
-            return result;
-          }
+  const compare = (a, b) => {
+    if (orderByClause.length > 1) {
+      const arrayA = a;
+      const arrayB = b;
+      for (let i = 0; i < orderByClause.length; i++) {
+        const clause = orderByClause[i];
+        const compareFn = comparison.makeComparator(clause.compareOptions);
+        const result = compareFn(arrayA[i], arrayB[i]);
+        if (result !== 0) {
+          return result;
         }
-        return arrayA.length - arrayB.length;
-      }
-      if (orderByClause.length === 1) {
-        const direction = orderByClause[0].direction;
-        return direction === `desc` ? comparison.descComparator(a, b) : comparison.ascComparator(a, b);
       }
-      return comparison.ascComparator(a, b);
-    };
+      return arrayA.length - arrayB.length;
+    }
+    if (orderByClause.length === 1) {
+      const clause = orderByClause[0];
+      const compareFn = comparison.makeComparator(clause.compareOptions);
+      return compareFn(a, b);
+    }
+    return comparison.defaultComparator(a, b);
   };
-  const comparator = makeComparator();
+  let setSizeCallback;
+  if (limit && orderByClause.length === 1) {
+    const clause = orderByClause[0];
+    const orderByExpression = clause.expression;
+    if (orderByExpression.type === `ref`) {
+      const followRefResult = index.followRef(
+        rawQuery,
+        orderByExpression,
+        collection
+      );
+      const followRefCollection = followRefResult.collection;
+      const fieldName = followRefResult.path[0];
+      if (fieldName) {
+        autoIndex.ensureIndexForField(
+          fieldName,
+          followRefResult.path,
+          followRefCollection,
+          compare
+        );
+      }
+      const valueExtractorForRawRow = evaluators.compileExpression(
+        new ir.PropRef(followRefResult.path),
+        true
+      );
+      const comparator = (a, b) => {
+        const extractedA = a ? valueExtractorForRawRow(a) : a;
+        const extractedB = b ? valueExtractorForRawRow(b) : b;
+        return compare(extractedA, extractedB);
+      };
+      const index$1 = indexOptimization.findIndexForField(
+        followRefCollection.indexes,
+        followRefResult.path
+      );
+      if (index$1 && index$1.supports(`gt`)) {
+        const orderByOptimizationInfo = {
+          offset: offset ?? 0,
+          limit,
+          comparator,
+          valueExtractorForRawRow,
+          index: index$1
+        };
+        optimizableOrderByCollections[followRefCollection.id] = orderByOptimizationInfo;
+        setSizeCallback = (getSize) => {
+          optimizableOrderByCollections[followRefCollection.id] = {
+            ...optimizableOrderByCollections[followRefCollection.id],
+            dataNeeded: () => {
+              const size = getSize();
+              return Math.max(0, limit - size);
+            }
+          };
+        };
+      }
+    }
+  }
   return pipeline.pipe(
     dbIvm.orderByWithFractionalIndex(valueExtractor, {
       limit,
       offset,
-      comparator
+      comparator: compare,
+      setSizeCallback
     })
     // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format
   );

package/dist/cjs/query/compiler/order-by.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"order-by.cjs","sources":["../../../../src/query/compiler/order-by.ts"],"sourcesContent":["import { orderByWithFractionalIndex } from \"@tanstack/db-ivm\"\nimport { ascComparator, descComparator } from \"../../utils/comparison.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport type { OrderByClause } from \"../ir.js\"\nimport type { NamespacedAndKeyedStream, NamespacedRow } from \"../../types.js\"\nimport type { IStreamBuilder, KeyValue } from \"@tanstack/db-ivm\"\n\n/**\n * Processes the ORDER BY clause\n * Works with the new structure that has both namespaced row data and __select_results\n * Always uses fractional indexing and adds the index as __ordering_index to the result\n */\nexport function processOrderBy(\n  pipeline: NamespacedAndKeyedStream,\n  orderByClause: Array<OrderByClause>,\n  limit?: number,\n  offset?: number\n): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>> {\n  // Pre-compile all order by expressions\n  const compiledOrderBy = orderByClause.map((clause) => ({\n    compiledExpression: compileExpression(clause.expression),\n    direction: clause.direction,\n  }))\n\n  // Create a value extractor function for the orderBy operator\n  const valueExtractor = (row: NamespacedRow & { __select_results?: any }) => {\n    // For ORDER BY expressions, we need to provide access to both:\n    // 1. The original namespaced row data (for direct table column references)\n    // 2. The __select_results (for SELECT alias references)\n\n    // Create a merged context for expression evaluation\n    const orderByContext = { ...row }\n\n    // If there are select results, merge them at the top level for alias access\n    if (row.__select_results) {\n      // Add select results as top-level properties for alias access\n      Object.assign(orderByContext, row.__select_results)\n    }\n\n    if (orderByClause.length > 1) {\n      // For multiple orderBy columns, create a composite key\n      return compiledOrderBy.map((compiled) =>\n        compiled.compiledExpression(orderByContext)\n      )\n    } else if (orderByClause.length === 1) {\n      // For a single orderBy column, use the value directly\n      const compiled = compiledOrderBy[0]!\n      return compiled.compiledExpression(orderByContext)\n    }\n\n    // Default case - no ordering\n    return null\n  }\n\n  // Create a multi-property comparator that respects the order and direction of each property\n  const makeComparator = () => {\n    return (a: unknown, b: unknown) => {\n      // If we're comparing arrays (multiple properties), compare each property in order\n      if (orderByClause.length > 1) {\n        const arrayA = a as Array<unknown>\n        const arrayB = b as Array<unknown>\n        for (let i = 0; i < orderByClause.length; i++) {\n          const direction = orderByClause[i]!.direction\n          const compareFn =\n            direction === `desc` ? descComparator : ascComparator\n          const result = compareFn(arrayA[i], arrayB[i])\n          if (result !== 0) {\n            return result\n          }\n        }\n        return arrayA.length - arrayB.length\n      }\n\n      // Single property comparison\n      if (orderByClause.length === 1) {\n        const direction = orderByClause[0]!.direction\n        return direction === `desc` ? descComparator(a, b) : ascComparator(a, b)\n      }\n\n      return ascComparator(a, b)\n    }\n  }\n\n  const comparator = makeComparator()\n\n  // Use fractional indexing and return the tuple [value, index]\n  return pipeline.pipe(\n    orderByWithFractionalIndex(valueExtractor, {\n      limit,\n      offset,\n      comparator,\n    })\n    // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format\n  )\n}\n"],"names":["compileExpression","descComparator","ascComparator","orderByWithFractionalIndex"],"mappings":";;;;;AAYO,SAAS,eACd,UACA,eACA,OACA,QAC4D;AAE5D,QAAM,kBAAkB,cAAc,IAAI,CAAC,YAAY;AAAA,IACrD,oBAAoBA,WAAAA,kBAAkB,OAAO,UAAU;AAAA,IACvD,WAAW,OAAO;AAAA,EAAA,EAClB;AAGF,QAAM,iBAAiB,CAAC,QAAoD;AAM1E,UAAM,iBAAiB,EAAE,GAAG,IAAA;AAG5B,QAAI,IAAI,kBAAkB;AAExB,aAAO,OAAO,gBAAgB,IAAI,gBAAgB;AAAA,IACpD;AAEA,QAAI,cAAc,SAAS,GAAG;AAE5B,aAAO,gBAAgB;AAAA,QAAI,CAAC,aAC1B,SAAS,mBAAmB,cAAc;AAAA,MAAA;AAAA,IAE9C,WAAW,cAAc,WAAW,GAAG;AAErC,YAAM,WAAW,gBAAgB,CAAC;AAClC,aAAO,SAAS,mBAAmB,cAAc;AAAA,IACnD;AAGA,WAAO;AAAA,EACT;AAGA,QAAM,iBAAiB,MAAM;AAC3B,WAAO,CAAC,GAAY,MAAe;AAEjC,UAAI,cAAc,SAAS,GAAG;AAC5B,cAAM,SAAS;AACf,cAAM,SAAS;AACf,iBAAS,IAAI,GAAG,IAAI,cAAc,QAAQ,KAAK;AAC7C,gBAAM,YAAY,cAAc,CAAC,EAAG;AACpC,gBAAM,YACJ,cAAc,SAASC,WAAAA,iBAAiBC,WAAAA;AAC1C,gBAAM,SAAS,UAAU,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AAC7C,cAAI,WAAW,GAAG;AAChB,mBAAO;AAAA,UACT;AAAA,QACF;AACA,eAAO,OAAO,SAAS,OAAO;AAAA,MAChC;AAGA,UAAI,cAAc,WAAW,GAAG;AAC9B,cAAM,YAAY,cAAc,CAAC,EAAG;AACpC,eAAO,cAAc,SAASD,WAAAA,eAAe,GAAG,CAAC,IAAIC,WAAAA,cAAc,GAAG,CAAC;AAAA,MACzE;AAEA,aAAOA,WAAAA,cAAc,GAAG,CAAC;AAAA,IAC3B;AAAA,EACF;AAEA,QAAM,aAAa,eAAA;AAGnB,SAAO,SAAS;AAAA,IACdC,MAAAA,2BAA2B,gBAAgB;AAAA,MACzC;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AAAA;AAAA,EAAA;AAGL;;"}
+{"version":3,"file":"order-by.cjs","sources":["../../../../src/query/compiler/order-by.ts"],"sourcesContent":["import { orderByWithFractionalIndex } from \"@tanstack/db-ivm\"\nimport { defaultComparator, makeComparator } from \"../../utils/comparison.js\"\nimport { PropRef } from \"../ir.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { followRef } from \"./index.js\"\nimport type { CompiledSingleRowExpression } from \"./evaluators.js\"\nimport type { OrderByClause, QueryIR } from \"../ir.js\"\nimport type { NamespacedAndKeyedStream, NamespacedRow } from \"../../types.js\"\nimport type { IStreamBuilder, KeyValue } from \"@tanstack/db-ivm\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\nimport type { Collection } from \"../../collection.js\"\n\nexport type OrderByOptimizationInfo = {\n  offset: number\n  limit: number\n  comparator: (\n    a: Record<string, unknown> | null | undefined,\n    b: Record<string, unknown> | null | undefined\n  ) => number\n  valueExtractorForRawRow: (row: Record<string, unknown>) => any\n  index: BaseIndex<string | number>\n  dataNeeded?: () => number\n}\n\n/**\n * Processes the ORDER BY clause\n * Works with the new structure that has both namespaced row data and __select_results\n * Always uses fractional indexing and adds the index as __ordering_index to the result\n */\nexport function processOrderBy(\n  rawQuery: QueryIR,\n  pipeline: NamespacedAndKeyedStream,\n  orderByClause: Array<OrderByClause>,\n  collection: Collection,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  limit?: number,\n  offset?: number\n): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>> {\n  // Pre-compile all order by expressions\n  const compiledOrderBy = orderByClause.map((clause) => ({\n    compiledExpression: compileExpression(clause.expression),\n    compareOptions: clause.compareOptions,\n  }))\n\n  // Create a value extractor function for the orderBy operator\n  const valueExtractor = (row: NamespacedRow & { __select_results?: any }) => {\n    // For ORDER BY expressions, we need to provide access to both:\n    // 1. The original namespaced row data (for direct table column references)\n    // 2. The __select_results (for SELECT alias references)\n\n    // Create a merged context for expression evaluation\n    const orderByContext = { ...row }\n\n    // If there are select results, merge them at the top level for alias access\n    if (row.__select_results) {\n      // Add select results as top-level properties for alias access\n      Object.assign(orderByContext, row.__select_results)\n    }\n\n    if (orderByClause.length > 1) {\n      // For multiple orderBy columns, create a composite key\n      return compiledOrderBy.map((compiled) =>\n        compiled.compiledExpression(orderByContext)\n      )\n    } else if (orderByClause.length === 1) {\n      // For a single orderBy column, use the value directly\n      const compiled = compiledOrderBy[0]!\n      return compiled.compiledExpression(orderByContext)\n    }\n\n    // Default case - no ordering\n    return null\n  }\n\n  // Create a multi-property comparator that respects the order and direction of each property\n  const compare = (a: unknown, b: unknown) => {\n    // If we're comparing arrays (multiple properties), compare each property in order\n    if (orderByClause.length > 1) {\n      const arrayA = a as Array<unknown>\n      const arrayB = b as Array<unknown>\n      for (let i = 0; i < orderByClause.length; i++) {\n        const clause = orderByClause[i]!\n        const compareFn = makeComparator(clause.compareOptions)\n        const result = compareFn(arrayA[i], arrayB[i])\n        if (result !== 0) {\n          return result\n        }\n      }\n      return arrayA.length - arrayB.length\n    }\n\n    // Single property comparison\n    if (orderByClause.length === 1) {\n      const clause = orderByClause[0]!\n      const compareFn = makeComparator(clause.compareOptions)\n      return compareFn(a, b)\n    }\n\n    return defaultComparator(a, b)\n  }\n\n  let setSizeCallback: ((getSize: () => number) => void) | undefined\n\n  // Optimize the orderBy operator to lazily load elements\n  // by using the range index of the collection.\n  // Only for orderBy clause on a single column for now (no composite ordering)\n  if (limit && orderByClause.length === 1) {\n    const clause = orderByClause[0]!\n    const orderByExpression = clause.expression\n\n    if (orderByExpression.type === `ref`) {\n      const followRefResult = followRef(\n        rawQuery,\n        orderByExpression,\n        collection\n      )!\n\n      const followRefCollection = followRefResult.collection\n      const fieldName = followRefResult.path[0]\n      if (fieldName) {\n        ensureIndexForField(\n          fieldName,\n          followRefResult.path,\n          followRefCollection,\n          compare\n        )\n      }\n\n      const valueExtractorForRawRow = compileExpression(\n        new PropRef(followRefResult.path),\n        true\n      ) as CompiledSingleRowExpression\n\n      const comparator = (\n        a: Record<string, unknown> | null | undefined,\n        b: Record<string, unknown> | null | undefined\n      ) => {\n        const extractedA = a ? valueExtractorForRawRow(a) : a\n        const extractedB = b ? valueExtractorForRawRow(b) : b\n        return compare(extractedA, extractedB)\n      }\n\n      const index: BaseIndex<string | number> | undefined = findIndexForField(\n        followRefCollection.indexes,\n        followRefResult.path\n      )\n\n      if (index && index.supports(`gt`)) {\n        // We found an index that we can use to lazily load ordered data\n        const orderByOptimizationInfo = {\n          offset: offset ?? 0,\n          limit,\n          comparator,\n          valueExtractorForRawRow,\n          index,\n        }\n\n        optimizableOrderByCollections[followRefCollection.id] =\n          orderByOptimizationInfo\n\n        setSizeCallback = (getSize: () => number) => {\n          optimizableOrderByCollections[followRefCollection.id] = {\n            ...optimizableOrderByCollections[followRefCollection.id]!,\n            dataNeeded: () => {\n              const size = getSize()\n              return Math.max(0, limit - size)\n            },\n          }\n        }\n      }\n    }\n  }\n\n  // Use fractional indexing and return the tuple [value, index]\n  return pipeline.pipe(\n    orderByWithFractionalIndex(valueExtractor, {\n      limit,\n      offset,\n      comparator: compare,\n      setSizeCallback,\n    })\n    // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format\n  )\n}\n"],"names":["compileExpression","makeComparator","defaultComparator","followRef","ensureIndexForField","PropRef","index","findIndexForField","orderByWithFractionalIndex"],"mappings":";;;;;;;;;AA+BO,SAAS,eACd,UACA,UACA,eACA,YACA,+BACA,OACA,QAC4D;AAE5D,QAAM,kBAAkB,cAAc,IAAI,CAAC,YAAY;AAAA,IACrD,oBAAoBA,WAAAA,kBAAkB,OAAO,UAAU;AAAA,IACvD,gBAAgB,OAAO;AAAA,EAAA,EACvB;AAGF,QAAM,iBAAiB,CAAC,QAAoD;AAM1E,UAAM,iBAAiB,EAAE,GAAG,IAAA;AAG5B,QAAI,IAAI,kBAAkB;AAExB,aAAO,OAAO,gBAAgB,IAAI,gBAAgB;AAAA,IACpD;AAEA,QAAI,cAAc,SAAS,GAAG;AAE5B,aAAO,gBAAgB;AAAA,QAAI,CAAC,aAC1B,SAAS,mBAAmB,cAAc;AAAA,MAAA;AAAA,IAE9C,WAAW,cAAc,WAAW,GAAG;AAErC,YAAM,WAAW,gBAAgB,CAAC;AAClC,aAAO,SAAS,mBAAmB,cAAc;AAAA,IACnD;AAGA,WAAO;AAAA,EACT;AAGA,QAAM,UAAU,CAAC,GAAY,MAAe;AAE1C,QAAI,cAAc,SAAS,GAAG;AAC5B,YAAM,SAAS;AACf,YAAM,SAAS;AACf,eAAS,IAAI,GAAG,IAAI,cAAc,QAAQ,KAAK;AAC7C,cAAM,SAAS,cAAc,CAAC;AAC9B,cAAM,YAAYC,WAAAA,eAAe,OAAO,cAAc;AACtD,cAAM,SAAS,UAAU,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AAC7C,YAAI,WAAW,GAAG;AAChB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,aAAO,OAAO,SAAS,OAAO;AAAA,IAChC;AAGA,QAAI,cAAc,WAAW,GAAG;AAC9B,YAAM,SAAS,cAAc,CAAC;AAC9B,YAAM,YAAYA,WAAAA,eAAe,OAAO,cAAc;AACtD,aAAO,UAAU,GAAG,CAAC;AAAA,IACvB;AAEA,WAAOC,WAAAA,kBAAkB,GAAG,CAAC;AAAA,EAC/B;AAEA,MAAI;AAKJ,MAAI,SAAS,cAAc,WAAW,GAAG;AACvC,UAAM,SAAS,cAAc,CAAC;AAC9B,UAAM,oBAAoB,OAAO;AAEjC,QAAI,kBAAkB,SAAS,OAAO;AACpC,YAAM,kBAAkBC,MAAAA;AAAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAGF,YAAM,sBAAsB,gBAAgB;AAC5C,YAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,UAAI,WAAW;AACbC,kBAAAA;AAAAA,UACE;AAAA,UACA,gBAAgB;AAAA,UAChB;AAAA,UACA;AAAA,QAAA;AAAA,MAEJ;AAEA,YAAM,0BAA0BJ,WAAAA;AAAAA,QAC9B,IAAIK,GAAAA,QAAQ,gBAAgB,IAAI;AAAA,QAChC;AAAA,MAAA;AAGF,YAAM,aAAa,CACjB,GACA,MACG;AACH,cAAM,aAAa,IAAI,wBAAwB,CAAC,IAAI;AACpD,cAAM,aAAa,IAAI,wBAAwB,CAAC,IAAI;AACpD,eAAO,QAAQ,YAAY,UAAU;AAAA,MACvC;AAEA,YAAMC,UAAgDC,kBAAAA;AAAAA,QACpD,oBAAoB;AAAA,QACpB,gBAAgB;AAAA,MAAA;AAGlB,UAAID,WAASA,QAAM,SAAS,IAAI,GAAG;AAEjC,cAAM,0BAA0B;AAAA,UAC9B,QAAQ,UAAU;AAAA,UAClB;AAAA,UACA;AAAA,UACA;AAAA,UAAA,OACAA;AAAAA,QAAA;AAGF,sCAA8B,oBAAoB,EAAE,IAClD;AAEF,0BAAkB,CAAC,YAA0B;AAC3C,wCAA8B,oBAAoB,EAAE,IAAI;AAAA,YACtD,GAAG,8BAA8B,oBAAoB,EAAE;AAAA,YACvD,YAAY,MAAM;AAChB,oBAAM,OAAO,QAAA;AACb,qBAAO,KAAK,IAAI,GAAG,QAAQ,IAAI;AAAA,YACjC;AAAA,UAAA;AAAA,QAEJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,SAAO,SAAS;AAAA,IACdE,MAAAA,2BAA2B,gBAAgB;AAAA,MACzC;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ;AAAA,IAAA,CACD;AAAA;AAAA,EAAA;AAGL;;"}

package/dist/cjs/query/compiler/order-by.d.cts

@@ -1,9 +1,19 @@
-import { OrderByClause } from '../ir.js';
+import { OrderByClause, QueryIR } from '../ir.js';
 import { NamespacedAndKeyedStream, NamespacedRow } from '../../types.js';
 import { IStreamBuilder, KeyValue } from '@tanstack/db-ivm';
+import { BaseIndex } from '../../indexes/base-index.js';
+import { Collection } from '../../collection.js';
+export type OrderByOptimizationInfo = {
+    offset: number;
+    limit: number;
+    comparator: (a: Record<string, unknown> | null | undefined, b: Record<string, unknown> | null | undefined) => number;
+    valueExtractorForRawRow: (row: Record<string, unknown>) => any;
+    index: BaseIndex<string | number>;
+    dataNeeded?: () => number;
+};
 /**
  * Processes the ORDER BY clause
  * Works with the new structure that has both namespaced row data and __select_results
  * Always uses fractional indexing and adds the index as __ordering_index to the result
  */
-export declare function processOrderBy(pipeline: NamespacedAndKeyedStream, orderByClause: Array<OrderByClause>, limit?: number, offset?: number): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>>;
+export declare function processOrderBy(rawQuery: QueryIR, pipeline: NamespacedAndKeyedStream, orderByClause: Array<OrderByClause>, collection: Collection, optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>, limit?: number, offset?: number): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>>;

package/dist/cjs/query/ir.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"ir.cjs","sources":["../../../src/query/ir.ts"],"sourcesContent":["/*\nThis is the intermediate representation of the query.\n*/\n\nimport type { CollectionImpl } from \"../collection\"\nimport type { NamespacedRow } from \"../types\"\n\nexport interface QueryIR {\n  from: From\n  select?: Select\n  join?: Join\n  where?: Array<Where>\n  groupBy?: GroupBy\n  having?: Array<Having>\n  orderBy?: OrderBy\n  limit?: Limit\n  offset?: Offset\n  distinct?: true\n\n  // Functional variants\n  fnSelect?: (row: NamespacedRow) => any\n  fnWhere?: Array<(row: NamespacedRow) => any>\n  fnHaving?: Array<(row: NamespacedRow) => any>\n}\n\nexport type From = CollectionRef | QueryRef\n\nexport type Select = {\n  [alias: string]: BasicExpression | Aggregate\n}\n\nexport type Join = Array<JoinClause>\n\nexport interface JoinClause {\n  from: CollectionRef | QueryRef\n  type: `left` | `right` | `inner` | `outer` | `full` | `cross`\n  left: BasicExpression\n  right: BasicExpression\n}\n\nexport type Where = BasicExpression<boolean>\n\nexport type GroupBy = Array<BasicExpression>\n\nexport type Having = Where\n\nexport type OrderBy = Array<OrderByClause>\n\nexport type OrderByClause = {\n  expression: BasicExpression\n  direction: OrderByDirection\n}\n\nexport type OrderByDirection = `asc` | `desc`\n\nexport type Limit = number\n\nexport type Offset = number\n\n/* Expressions */\n\nabstract class BaseExpression<T = any> {\n  public abstract type: string\n  /** @internal - Type brand for TypeScript inference */\n  declare readonly __returnType: T\n}\n\nexport class CollectionRef extends BaseExpression {\n  public type = `collectionRef` as const\n  constructor(\n    public collection: CollectionImpl,\n    public alias: string\n  ) {\n    super()\n  }\n}\n\nexport class QueryRef extends BaseExpression {\n  public type = `queryRef` as const\n  constructor(\n    public query: QueryIR,\n    public alias: string\n  ) {\n    super()\n  }\n}\n\nexport class PropRef<T = any> extends BaseExpression<T> {\n  public type = `ref` as const\n  constructor(\n    public path: Array<string> // path to the property in the collection, with the alias as the first element\n  ) {\n    super()\n  }\n}\n\nexport class Value<T = any> extends BaseExpression<T> {\n  public type = `val` as const\n  constructor(\n    public value: T // any js value\n  ) {\n    super()\n  }\n}\n\nexport class Func<T = any> extends BaseExpression<T> {\n  public type = `func` as const\n  constructor(\n    public name: string, // such as eq, gt, lt, upper, lower, etc.\n    public args: Array<BasicExpression>\n  ) {\n    super()\n  }\n}\n\n// This is the basic expression type that is used in the majority of expression\n// builder callbacks (select, where, groupBy, having, orderBy, etc.)\n// it doesn't include aggregate functions as those are only used in the select clause\nexport type BasicExpression<T = any> = PropRef<T> | Value<T> | Func<T>\n\nexport class Aggregate<T = any> extends BaseExpression<T> {\n  public type = `agg` as const\n  constructor(\n    public name: string, // such as count, avg, sum, min, max, etc.\n    public args: Array<BasicExpression>\n  ) {\n    super()\n  }\n}\n"],"names":[],"mappings":";;AA6DA,MAAe,eAAwB;AAIvC;AAEO,MAAM,sBAAsB,eAAe;AAAA,EAEhD,YACS,YACA,OACP;AACA,UAAA;AAHO,SAAA,aAAA;AACA,SAAA,QAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAEO,MAAM,iBAAiB,eAAe;AAAA,EAE3C,YACS,OACA,OACP;AACA,UAAA;AAHO,SAAA,QAAA;AACA,SAAA,QAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAEO,MAAM,gBAAyB,eAAkB;AAAA,EAEtD,YACS,MACP;AACA,UAAA;AAFO,SAAA,OAAA;AAFT,SAAO,OAAO;AAAA,EAKd;AACF;AAEO,MAAM,cAAuB,eAAkB;AAAA,EAEpD,YACS,OACP;AACA,UAAA;AAFO,SAAA,QAAA;AAFT,SAAO,OAAO;AAAA,EAKd;AACF;AAEO,MAAM,aAAsB,eAAkB;AAAA,EAEnD,YACS,MACA,MACP;AACA,UAAA;AAHO,SAAA,OAAA;AACA,SAAA,OAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAOO,MAAM,kBAA2B,eAAkB;AAAA,EAExD,YACS,MACA,MACP;AACA,UAAA;AAHO,SAAA,OAAA;AACA,SAAA,OAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;;;;;;;"}
+{"version":3,"file":"ir.cjs","sources":["../../../src/query/ir.ts"],"sourcesContent":["/*\nThis is the intermediate representation of the query.\n*/\n\nimport type { CompareOptions } from \"./builder/types\"\nimport type { CollectionImpl } from \"../collection\"\nimport type { NamespacedRow } from \"../types\"\n\nexport interface QueryIR {\n  from: From\n  select?: Select\n  join?: Join\n  where?: Array<Where>\n  groupBy?: GroupBy\n  having?: Array<Having>\n  orderBy?: OrderBy\n  limit?: Limit\n  offset?: Offset\n  distinct?: true\n\n  // Functional variants\n  fnSelect?: (row: NamespacedRow) => any\n  fnWhere?: Array<(row: NamespacedRow) => any>\n  fnHaving?: Array<(row: NamespacedRow) => any>\n}\n\nexport type From = CollectionRef | QueryRef\n\nexport type Select = {\n  [alias: string]: BasicExpression | Aggregate\n}\n\nexport type Join = Array<JoinClause>\n\nexport interface JoinClause {\n  from: CollectionRef | QueryRef\n  type: `left` | `right` | `inner` | `outer` | `full` | `cross`\n  left: BasicExpression\n  right: BasicExpression\n}\n\nexport type Where = BasicExpression<boolean>\n\nexport type GroupBy = Array<BasicExpression>\n\nexport type Having = Where\n\nexport type OrderBy = Array<OrderByClause>\n\nexport type OrderByClause = {\n  expression: BasicExpression\n  compareOptions: CompareOptions\n}\n\nexport type OrderByDirection = `asc` | `desc`\n\nexport type Limit = number\n\nexport type Offset = number\n\n/* Expressions */\n\nabstract class BaseExpression<T = any> {\n  public abstract type: string\n  /** @internal - Type brand for TypeScript inference */\n  declare readonly __returnType: T\n}\n\nexport class CollectionRef extends BaseExpression {\n  public type = `collectionRef` as const\n  constructor(\n    public collection: CollectionImpl,\n    public alias: string\n  ) {\n    super()\n  }\n}\n\nexport class QueryRef extends BaseExpression {\n  public type = `queryRef` as const\n  constructor(\n    public query: QueryIR,\n    public alias: string\n  ) {\n    super()\n  }\n}\n\nexport class PropRef<T = any> extends BaseExpression<T> {\n  public type = `ref` as const\n  constructor(\n    public path: Array<string> // path to the property in the collection, with the alias as the first element\n  ) {\n    super()\n  }\n}\n\nexport class Value<T = any> extends BaseExpression<T> {\n  public type = `val` as const\n  constructor(\n    public value: T // any js value\n  ) {\n    super()\n  }\n}\n\nexport class Func<T = any> extends BaseExpression<T> {\n  public type = `func` as const\n  constructor(\n    public name: string, // such as eq, gt, lt, upper, lower, etc.\n    public args: Array<BasicExpression>\n  ) {\n    super()\n  }\n}\n\n// This is the basic expression type that is used in the majority of expression\n// builder callbacks (select, where, groupBy, having, orderBy, etc.)\n// it doesn't include aggregate functions as those are only used in the select clause\nexport type BasicExpression<T = any> = PropRef<T> | Value<T> | Func<T>\n\nexport class Aggregate<T = any> extends BaseExpression<T> {\n  public type = `agg` as const\n  constructor(\n    public name: string, // such as count, avg, sum, min, max, etc.\n    public args: Array<BasicExpression>\n  ) {\n    super()\n  }\n}\n"],"names":[],"mappings":";;AA8DA,MAAe,eAAwB;AAIvC;AAEO,MAAM,sBAAsB,eAAe;AAAA,EAEhD,YACS,YACA,OACP;AACA,UAAA;AAHO,SAAA,aAAA;AACA,SAAA,QAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAEO,MAAM,iBAAiB,eAAe;AAAA,EAE3C,YACS,OACA,OACP;AACA,UAAA;AAHO,SAAA,QAAA;AACA,SAAA,QAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAEO,MAAM,gBAAyB,eAAkB;AAAA,EAEtD,YACS,MACP;AACA,UAAA;AAFO,SAAA,OAAA;AAFT,SAAO,OAAO;AAAA,EAKd;AACF;AAEO,MAAM,cAAuB,eAAkB;AAAA,EAEpD,YACS,OACP;AACA,UAAA;AAFO,SAAA,QAAA;AAFT,SAAO,OAAO;AAAA,EAKd;AACF;AAEO,MAAM,aAAsB,eAAkB;AAAA,EAEnD,YACS,MACA,MACP;AACA,UAAA;AAHO,SAAA,OAAA;AACA,SAAA,OAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;AAOO,MAAM,kBAA2B,eAAkB;AAAA,EAExD,YACS,MACA,MACP;AACA,UAAA;AAHO,SAAA,OAAA;AACA,SAAA,OAAA;AAHT,SAAO,OAAO;AAAA,EAMd;AACF;;;;;;;"}

package/dist/cjs/query/ir.d.cts

@@ -1,3 +1,4 @@
+import { CompareOptions } from './builder/types.cjs';
 import { CollectionImpl } from '../collection.cjs';
 import { NamespacedRow } from '../types.cjs';
 export interface QueryIR {
@@ -32,7 +33,7 @@ export type Having = Where;
 export type OrderBy = Array<OrderByClause>;
 export type OrderByClause = {
     expression: BasicExpression;
-    direction: OrderByDirection;
+    compareOptions: CompareOptions;
 };
 export type OrderByDirection = `asc` | `desc`;
 export type Limit = number;