@tanstack/db 0.2.3 → 0.2.4

package/dist/cjs/query/builder/types.d.cts

@@ -345,7 +345,7 @@ type NonNull<T> = T extends null ? never : T;
  * ```
  */
 export type Ref<T = any> = {
-    [K in keyof T]: IsNonExactOptional<T[K]> extends true ? IsNonExactNullable<T[K]> extends true ? NonNullable<T[K]> extends Record<string, any> ? Ref<NonNullable<T[K]>> | undefined : RefLeaf<NonNullable<T[K]>> | undefined : NonUndefined<T[K]> extends Record<string, any> ? Ref<NonUndefined<T[K]>> | undefined : RefLeaf<NonUndefined<T[K]>> | undefined : IsNonExactNullable<T[K]> extends true ? NonNull<T[K]> extends Record<string, any> ? Ref<NonNull<T[K]>> | null : RefLeaf<NonNull<T[K]>> | null : T[K] extends Record<string, any> ? Ref<T[K]> : RefLeaf<T[K]>;
+    [K in keyof T]: IsNonExactOptional<T[K]> extends true ? IsNonExactNullable<T[K]> extends true ? IsPlainObject<NonNullable<T[K]>> extends true ? Ref<NonNullable<T[K]>> | undefined : RefLeaf<NonNullable<T[K]>> | undefined : IsPlainObject<NonUndefined<T[K]>> extends true ? Ref<NonUndefined<T[K]>> | undefined : RefLeaf<NonUndefined<T[K]>> | undefined : IsNonExactNullable<T[K]> extends true ? IsPlainObject<NonNull<T[K]>> extends true ? Ref<NonNull<T[K]>> | null : RefLeaf<NonNull<T[K]>> | null : IsPlainObject<T[K]> extends true ? Ref<T[K]> : RefLeaf<T[K]>;
 } & RefLeaf<T>;
 /**
  * Ref - The user-facing ref type with clean IDE display
@@ -564,4 +564,12 @@ export type WithResult<TContext extends Context, TResult> = Prettify<Omit<TConte
 export type Prettify<T> = {
     [K in keyof T]: T[K];
 } & {};
+/**
+ * IsPlainObject - Utility type to check if T is a plain object
+ */
+type IsPlainObject<T> = T extends unknown ? T extends object ? T extends ReadonlyArray<any> ? false : T extends JsBuiltIns ? false : true : false : false;
+/**
+ * JsBuiltIns - List of JavaScript built-ins
+ */
+type JsBuiltIns = ArrayBuffer | ArrayBufferLike | AsyncGenerator<any, any, any> | BigInt64Array | BigUint64Array | DataView | Date | Error | Float32Array | Float64Array | Function | Generator<any, any, any> | Int16Array | Int32Array | Int8Array | Map<any, any> | Promise<any> | RegExp | Set<any> | string | Uint16Array | Uint32Array | Uint8Array | Uint8ClampedArray | WeakMap<any, any> | WeakSet<any>;
 export {};

package/dist/cjs/query/compiler/joins.cjs

@@ -105,7 +105,7 @@ function processJoin(pipeline, joinClause, tables, mainTableId, mainTableAlias,
       }
       let deoptimized = false;
       const activePipelineWithLoading = activePipeline.pipe(
-        dbIvm.tap(([joinKey, _]) => {
+        dbIvm.tap((data) => {
           if (deoptimized) {
             return;
           }
@@ -120,8 +120,9 @@ function processJoin(pipeline, joinClause, tables, mainTableId, mainTableAlias,
             );
           }
           const { loadKeys, loadInitialState } = collectionCallbacks;
-          if (index$1 && index$1.supports(`eq`)) {
-            const matchingKeys = index$1.lookup(`eq`, joinKey);
+          if (index$1 && index$1.supports(`in`)) {
+            const joinKeys = data.getInner().map(([[joinKey]]) => joinKey);
+            const matchingKeys = index$1.lookup(`in`, joinKeys);
             loadKeys(matchingKeys);
           } else {
             deoptimized = true;

package/dist/cjs/query/compiler/joins.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"joins.cjs","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n  tap,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinCondition,\n  InvalidJoinConditionLeftTableError,\n  InvalidJoinConditionRightTableError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  JoinCollectionNotFoundError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery, followRef } from \"./index.js\"\nimport type { OrderByOptimizationInfo } from \"./order-by.js\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  PropRef,\n  QueryIR,\n  QueryRef,\n} from \"../ir.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\n\nexport type LoadKeysFn = (key: Set<string | number>) => void\nexport type LazyCollectionCallbacks = {\n  loadKeys: LoadKeysFn\n  loadInitialState: () => void\n}\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableId,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping,\n      collections,\n      callbacks,\n      lazyCollections,\n      optimizableOrderByCollections,\n      rawQuery\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const {\n    alias: joinedTableAlias,\n    input: joinedInput,\n    collectionId: joinedCollectionId,\n  } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    collections,\n    callbacks,\n    lazyCollections,\n    optimizableOrderByCollections,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  const mainCollection = collections[mainTableId]\n  const joinedCollection = collections[joinedCollectionId]\n\n  if (!mainCollection) {\n    throw new JoinCollectionNotFoundError(mainTableId)\n  }\n\n  if (!joinedCollection) {\n    throw new JoinCollectionNotFoundError(joinedCollectionId)\n  }\n\n  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(\n    joinClause.type,\n    mainCollection,\n    joinedCollection\n  )\n\n  // Analyze which table each expression refers to and swap if necessary\n  const availableTableAliases = Object.keys(tables)\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    availableTableAliases,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  let mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  let joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n\n  if (activeCollection) {\n    // If the lazy collection comes from a subquery that has a limit and/or an offset clause\n    // then we need to deoptimize the join because we don't know which rows are in the result set\n    // since we simply lookup matching keys in the index but the index contains all rows\n    // (not just the ones that pass the limit and offset clauses)\n    const lazyFrom =\n      activeCollection === `main` ? joinClause.from : rawQuery.from\n    const limitedSubquery =\n      lazyFrom.type === `queryRef` &&\n      (lazyFrom.query.limit || lazyFrom.query.offset)\n\n    if (!limitedSubquery) {\n      // This join can be optimized by having the active collection\n      // dynamically load keys into the lazy collection\n      // based on the value of the joinKey and by looking up\n      // matching rows in the index of the lazy collection\n\n      // Mark the lazy collection as lazy\n      // this Set is passed by the liveQueryCollection to the compiler\n      // such that the liveQueryCollection can check it after compilation\n      // to know which collections are lazy collections\n      lazyCollections.add(lazyCollection.id)\n\n      const activePipeline =\n        activeCollection === `main` ? mainPipeline : joinedPipeline\n\n      let index: BaseIndex<string | number> | undefined\n\n      const lazyCollectionJoinExpr =\n        activeCollection === `main`\n          ? (joinedExpr as PropRef)\n          : (mainExpr as PropRef)\n\n      const followRefResult = followRef(\n        rawQuery,\n        lazyCollectionJoinExpr,\n        lazyCollection\n      )!\n      const followRefCollection = followRefResult.collection\n\n      const fieldName = followRefResult.path[0]\n      if (fieldName) {\n        ensureIndexForField(\n          fieldName,\n          followRefResult.path,\n          followRefCollection\n        )\n      }\n\n      let deoptimized = false\n\n      const activePipelineWithLoading: IStreamBuilder<\n        [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]\n      > = activePipeline.pipe(\n        tap(([joinKey, _]) => {\n          if (deoptimized) {\n            return\n          }\n\n          // Find the index for the path we join on\n          // we need to find the index inside the map operator\n          // because the indexes are only available after the initial sync\n          // so we can't fetch it during compilation\n          index ??= findIndexForField(\n            followRefCollection.indexes,\n            followRefResult.path\n          )\n\n          // The `callbacks` object is passed by the liveQueryCollection to the compiler.\n          // It contains a function to lazy load keys for each lazy collection\n          // as well as a function to switch back to a regular collection\n          // (useful when there's no index for available for lazily loading the collection)\n          const collectionCallbacks = callbacks[lazyCollection.id]\n          if (!collectionCallbacks) {\n            throw new Error(\n              `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`\n            )\n          }\n\n          const { loadKeys, loadInitialState } = collectionCallbacks\n\n          if (index && index.supports(`eq`)) {\n            // Use the index to fetch the PKs of the rows in the lazy collection\n            // that match this row from the active collection based on the value of the joinKey\n            const matchingKeys = index.lookup(`eq`, joinKey)\n            // Inform the lazy collection that those keys need to be loaded\n            loadKeys(matchingKeys)\n          } else {\n            // We can't optimize the join because there is no index for the join key\n            // on the lazy collection, so we load the initial state\n            deoptimized = true\n            loadInitialState()\n          }\n        })\n      )\n\n      if (activeCollection === `main`) {\n        mainPipeline = activePipelineWithLoading\n      } else {\n        joinedPipeline = activePipelineWithLoading\n      }\n    }\n  }\n\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinClause.type as JoinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (available table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  allAvailableTableAliases: Array<string>,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  // Filter out the joined table alias from the available table aliases\n  const availableTableAliases = allAvailableTableAliases.filter(\n    (alias) => alias !== joinedTableAlias\n  )\n\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to an available table and right refers to joined table, keep as is\n  if (\n    leftTableAlias &&\n    availableTableAliases.includes(leftTableAlias) &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to an available table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias &&\n    availableTableAliases.includes(rightTableAlias)\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If one expression doesn't refer to any table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    // For backward compatibility, use the first available table alias in error message\n    throw new InvalidJoinConditionTableMismatchError()\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias)\n  }\n\n  // Left side must refer to an available table\n  // This cannot happen with the query builder as there is no way to build a ref\n  // to an unavailable table, but just in case, but could happen with the IR\n  if (!availableTableAliases.includes(leftTableAlias)) {\n    throw new InvalidJoinConditionLeftTableError(leftTableAlias)\n  }\n\n  // Right side must refer to the joined table\n  if (rightTableAlias !== joinedTableAlias) {\n    throw new InvalidJoinConditionRightTableError(joinedTableAlias)\n  }\n\n  // This should not be reachable given the logic above, but just in case\n  throw new InvalidJoinCondition()\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream; collectionId: string } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input, collectionId: from.collection.id }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        collections,\n        callbacks,\n        lazyCollections,\n        optimizableOrderByCollections,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return {\n        alias: from.alias,\n        input: extractedInput as KeyedStream,\n        collectionId: subQueryResult.collectionId,\n      }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n\n/**\n * Returns the active and lazy collections for a join clause.\n * The active collection is the one that we need to fully iterate over\n * and it can be the main table (i.e. left collection) or the joined table (i.e. right collection).\n * The lazy collection is the one that we should join-in lazily based on matches in the active collection.\n * @param joinClause - The join clause to analyze\n * @param leftCollection - The left collection\n * @param rightCollection - The right collection\n * @returns The active and lazy collections. They are undefined if we need to loop over both collections (i.e. both are active)\n */\nfunction getActiveAndLazyCollections(\n  joinType: JoinClause[`type`],\n  leftCollection: Collection,\n  rightCollection: Collection\n):\n  | { activeCollection: `main` | `joined`; lazyCollection: Collection }\n  | { activeCollection: undefined; lazyCollection: undefined } {\n  if (leftCollection.id === rightCollection.id) {\n    // We can't apply this optimization if there's only one collection\n    // because `liveQueryCollection` will detect that the collection is lazy\n    // and treat it lazily (because the collection is shared)\n    // and thus it will not load any keys because both sides of the join\n    // will be handled lazily\n    return { activeCollection: undefined, lazyCollection: undefined }\n  }\n\n  switch (joinType) {\n    case `left`:\n      return { activeCollection: `main`, lazyCollection: rightCollection }\n    case `right`:\n      return { activeCollection: `joined`, lazyCollection: leftCollection }\n    case `inner`:\n      // The smallest collection should be the active collection\n      // and the biggest collection should be lazy\n      return leftCollection.size < rightCollection.size\n        ? { activeCollection: `main`, lazyCollection: rightCollection }\n        : { activeCollection: `joined`, lazyCollection: leftCollection }\n    default:\n      return { activeCollection: undefined, lazyCollection: undefined }\n  }\n}\n"],"names":["JoinCollectionNotFoundError","compileExpression","map","UnsupportedJoinTypeError","index","followRef","ensureIndexForField","tap","findIndexForField","joinOperator","consolidate","InvalidJoinConditionTableMismatchError","InvalidJoinConditionSameTableError","InvalidJoinConditionLeftTableError","InvalidJoinConditionRightTableError","InvalidJoinCondition","CollectionInputNotFoundError","compileQuery","UnsupportedJoinSourceTypeError","filter"],"mappings":";;;;;;;;AAkDO,SAAS,aACd,UACA,aACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAE1B,QAAM;AAAA,IACJ,OAAO;AAAA,IACP,OAAO;AAAA,IACP,cAAc;AAAA,EAAA,IACZ;AAAA,IACF,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAE3B,QAAM,iBAAiB,YAAY,WAAW;AAC9C,QAAM,mBAAmB,YAAY,kBAAkB;AAEvD,MAAI,CAAC,gBAAgB;AACnB,UAAM,IAAIA,OAAAA,4BAA4B,WAAW;AAAA,EACnD;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAIA,OAAAA,4BAA4B,kBAAkB;AAAA,EAC1D;AAEA,QAAM,EAAE,kBAAkB,eAAA,IAAmB;AAAA,IAC3C,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,wBAAwB,OAAO,KAAK,MAAM;AAChD,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmBC,WAAAA,kBAAkB,QAAQ;AACnD,QAAM,qBAAqBA,WAAAA,kBAAkB,UAAU;AAGvD,MAAI,eAAe,SAAS;AAAA,IAC1BC,MAAAA,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,MAAI,iBAAiB,YAAY;AAAA,IAC/BA,MAAAA,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,WAAW,IAAI,GAAG;AACjE,UAAM,IAAIC,OAAAA,yBAAyB,WAAW,IAAI;AAAA,EACpD;AAEA,MAAI,kBAAkB;AAKpB,UAAM,WACJ,qBAAqB,SAAS,WAAW,OAAO,SAAS;AAC3D,UAAM,kBACJ,SAAS,SAAS,eACjB,SAAS,MAAM,SAAS,SAAS,MAAM;AAE1C,QAAI,CAAC,iBAAiB;AAUpB,sBAAgB,IAAI,eAAe,EAAE;AAErC,YAAM,iBACJ,qBAAqB,SAAS,eAAe;AAE/C,UAAIC;AAEJ,YAAM,yBACJ,qBAAqB,SAChB,aACA;AAEP,YAAM,kBAAkBC,MAAAA;AAAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,sBAAsB,gBAAgB;AAE5C,YAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,UAAI,WAAW;AACbC,kBAAAA;AAAAA,UACE;AAAA,UACA,gBAAgB;AAAA,UAChB;AAAA,QAAA;AAAA,MAEJ;AAEA,UAAI,cAAc;AAElB,YAAM,4BAEF,eAAe;AAAA,QACjBC,MAAAA,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM;AACpB,cAAI,aAAa;AACf;AAAA,UACF;AAMAH,gCAAUI,kBAAAA;AAAAA,YACR,oBAAoB;AAAA,YACpB,gBAAgB;AAAA,UAAA;AAOlB,gBAAM,sBAAsB,UAAU,eAAe,EAAE;AACvD,cAAI,CAAC,qBAAqB;AACxB,kBAAM,IAAI;AAAA,cACR;AAAA,YAAA;AAAA,UAEJ;AAEA,gBAAM,EAAE,UAAU,iBAAA,IAAqB;AAEvC,cAAIJ,WAASA,QAAM,SAAS,IAAI,GAAG;AAGjC,kBAAM,eAAeA,QAAM,OAAO,MAAM,OAAO;AAE/C,qBAAS,YAAY;AAAA,UACvB,OAAO;AAGL,0BAAc;AACd,6BAAA;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MAAA;AAGH,UAAI,qBAAqB,QAAQ;AAC/B,uBAAe;AAAA,MACjB,OAAO;AACL,yBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,aAAa;AAAA,IAClBK,WAAa,gBAAgB,WAAW,IAAgB;AAAA,IACxDC,kBAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,0BACA,kBAC4D;AAE5D,QAAM,wBAAwB,yBAAyB;AAAA,IACrD,CAAC,UAAU,UAAU;AAAA,EAAA;AAGvB,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,kBACA,sBAAsB,SAAS,cAAc,KAC7C,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,mBACA,sBAAsB,SAAS,eAAe,GAC9C;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AAEvC,UAAM,IAAIC,OAAAA,uCAAA;AAAA,EACZ;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAIC,OAAAA,mCAAmC,cAAc;AAAA,EAC7D;AAKA,MAAI,CAAC,sBAAsB,SAAS,cAAc,GAAG;AACnD,UAAM,IAAIC,OAAAA,mCAAmC,cAAc;AAAA,EAC7D;AAGA,MAAI,oBAAoB,kBAAkB;AACxC,UAAM,IAAIC,OAAAA,oCAAoC,gBAAgB;AAAA,EAChE;AAGA,QAAM,IAAIC,OAAAA,qBAAA;AACZ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,aACA,WACA,iBACA,+BACA,OACA,cAC6D;AAC7D,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAIC,OAAAA,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,cAAc,KAAK,WAAW,GAAA;AAAA,IACnE;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiBC,MAAAA;AAAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnCf,MAAAA,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO;AAAA,QACL,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,cAAc,eAAe;AAAA,MAAA;AAAA,IAEjC;AAAA,IACA;AACE,YAAM,IAAIgB,OAAAA,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEdC,MAAAA,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACDjB,MAAAA,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAYA,SAAS,4BACP,UACA,gBACA,iBAG6D;AAC7D,MAAI,eAAe,OAAO,gBAAgB,IAAI;AAM5C,WAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EACxD;AAEA,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA;AAAA,IACrD,KAAK;AACH,aAAO,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACvD,KAAK;AAGH,aAAO,eAAe,OAAO,gBAAgB,OACzC,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA,IAC5C,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACpD;AACE,aAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EAAU;AAEtE;;"}
+{"version":3,"file":"joins.cjs","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n  tap,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinCondition,\n  InvalidJoinConditionLeftTableError,\n  InvalidJoinConditionRightTableError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  JoinCollectionNotFoundError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery, followRef } from \"./index.js\"\nimport type { OrderByOptimizationInfo } from \"./order-by.js\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  PropRef,\n  QueryIR,\n  QueryRef,\n} from \"../ir.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\n\nexport type LoadKeysFn = (key: Set<string | number>) => void\nexport type LazyCollectionCallbacks = {\n  loadKeys: LoadKeysFn\n  loadInitialState: () => void\n}\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableId,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping,\n      collections,\n      callbacks,\n      lazyCollections,\n      optimizableOrderByCollections,\n      rawQuery\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const {\n    alias: joinedTableAlias,\n    input: joinedInput,\n    collectionId: joinedCollectionId,\n  } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    collections,\n    callbacks,\n    lazyCollections,\n    optimizableOrderByCollections,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  const mainCollection = collections[mainTableId]\n  const joinedCollection = collections[joinedCollectionId]\n\n  if (!mainCollection) {\n    throw new JoinCollectionNotFoundError(mainTableId)\n  }\n\n  if (!joinedCollection) {\n    throw new JoinCollectionNotFoundError(joinedCollectionId)\n  }\n\n  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(\n    joinClause.type,\n    mainCollection,\n    joinedCollection\n  )\n\n  // Analyze which table each expression refers to and swap if necessary\n  const availableTableAliases = Object.keys(tables)\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    availableTableAliases,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  let mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  let joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n\n  if (activeCollection) {\n    // If the lazy collection comes from a subquery that has a limit and/or an offset clause\n    // then we need to deoptimize the join because we don't know which rows are in the result set\n    // since we simply lookup matching keys in the index but the index contains all rows\n    // (not just the ones that pass the limit and offset clauses)\n    const lazyFrom =\n      activeCollection === `main` ? joinClause.from : rawQuery.from\n    const limitedSubquery =\n      lazyFrom.type === `queryRef` &&\n      (lazyFrom.query.limit || lazyFrom.query.offset)\n\n    if (!limitedSubquery) {\n      // This join can be optimized by having the active collection\n      // dynamically load keys into the lazy collection\n      // based on the value of the joinKey and by looking up\n      // matching rows in the index of the lazy collection\n\n      // Mark the lazy collection as lazy\n      // this Set is passed by the liveQueryCollection to the compiler\n      // such that the liveQueryCollection can check it after compilation\n      // to know which collections are lazy collections\n      lazyCollections.add(lazyCollection.id)\n\n      const activePipeline =\n        activeCollection === `main` ? mainPipeline : joinedPipeline\n\n      let index: BaseIndex<string | number> | undefined\n\n      const lazyCollectionJoinExpr =\n        activeCollection === `main`\n          ? (joinedExpr as PropRef)\n          : (mainExpr as PropRef)\n\n      const followRefResult = followRef(\n        rawQuery,\n        lazyCollectionJoinExpr,\n        lazyCollection\n      )!\n      const followRefCollection = followRefResult.collection\n\n      const fieldName = followRefResult.path[0]\n      if (fieldName) {\n        ensureIndexForField(\n          fieldName,\n          followRefResult.path,\n          followRefCollection\n        )\n      }\n\n      let deoptimized = false\n\n      const activePipelineWithLoading: IStreamBuilder<\n        [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]\n      > = activePipeline.pipe(\n        tap((data) => {\n          if (deoptimized) {\n            return\n          }\n\n          // Find the index for the path we join on\n          // we need to find the index inside the map operator\n          // because the indexes are only available after the initial sync\n          // so we can't fetch it during compilation\n          index ??= findIndexForField(\n            followRefCollection.indexes,\n            followRefResult.path\n          )\n\n          // The `callbacks` object is passed by the liveQueryCollection to the compiler.\n          // It contains a function to lazy load keys for each lazy collection\n          // as well as a function to switch back to a regular collection\n          // (useful when there's no index for available for lazily loading the collection)\n          const collectionCallbacks = callbacks[lazyCollection.id]\n          if (!collectionCallbacks) {\n            throw new Error(\n              `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`\n            )\n          }\n\n          const { loadKeys, loadInitialState } = collectionCallbacks\n\n          if (index && index.supports(`in`)) {\n            // Use the index to fetch the PKs of the rows in the lazy collection\n            // that match this row from the active collection based on the value of the joinKey\n            const joinKeys = data.getInner().map(([[joinKey]]) => joinKey)\n            const matchingKeys = index.lookup(`in`, joinKeys)\n            // Inform the lazy collection that those keys need to be loaded\n            loadKeys(matchingKeys)\n          } else {\n            // We can't optimize the join because there is no index for the join key\n            // on the lazy collection, so we load the initial state\n            deoptimized = true\n            loadInitialState()\n          }\n        })\n      )\n\n      if (activeCollection === `main`) {\n        mainPipeline = activePipelineWithLoading\n      } else {\n        joinedPipeline = activePipelineWithLoading\n      }\n    }\n  }\n\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinClause.type as JoinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (available table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  allAvailableTableAliases: Array<string>,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  // Filter out the joined table alias from the available table aliases\n  const availableTableAliases = allAvailableTableAliases.filter(\n    (alias) => alias !== joinedTableAlias\n  )\n\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to an available table and right refers to joined table, keep as is\n  if (\n    leftTableAlias &&\n    availableTableAliases.includes(leftTableAlias) &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to an available table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias &&\n    availableTableAliases.includes(rightTableAlias)\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If one expression doesn't refer to any table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    // For backward compatibility, use the first available table alias in error message\n    throw new InvalidJoinConditionTableMismatchError()\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias)\n  }\n\n  // Left side must refer to an available table\n  // This cannot happen with the query builder as there is no way to build a ref\n  // to an unavailable table, but just in case, but could happen with the IR\n  if (!availableTableAliases.includes(leftTableAlias)) {\n    throw new InvalidJoinConditionLeftTableError(leftTableAlias)\n  }\n\n  // Right side must refer to the joined table\n  if (rightTableAlias !== joinedTableAlias) {\n    throw new InvalidJoinConditionRightTableError(joinedTableAlias)\n  }\n\n  // This should not be reachable given the logic above, but just in case\n  throw new InvalidJoinCondition()\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream; collectionId: string } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input, collectionId: from.collection.id }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        collections,\n        callbacks,\n        lazyCollections,\n        optimizableOrderByCollections,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return {\n        alias: from.alias,\n        input: extractedInput as KeyedStream,\n        collectionId: subQueryResult.collectionId,\n      }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n\n/**\n * Returns the active and lazy collections for a join clause.\n * The active collection is the one that we need to fully iterate over\n * and it can be the main table (i.e. left collection) or the joined table (i.e. right collection).\n * The lazy collection is the one that we should join-in lazily based on matches in the active collection.\n * @param joinClause - The join clause to analyze\n * @param leftCollection - The left collection\n * @param rightCollection - The right collection\n * @returns The active and lazy collections. They are undefined if we need to loop over both collections (i.e. both are active)\n */\nfunction getActiveAndLazyCollections(\n  joinType: JoinClause[`type`],\n  leftCollection: Collection,\n  rightCollection: Collection\n):\n  | { activeCollection: `main` | `joined`; lazyCollection: Collection }\n  | { activeCollection: undefined; lazyCollection: undefined } {\n  if (leftCollection.id === rightCollection.id) {\n    // We can't apply this optimization if there's only one collection\n    // because `liveQueryCollection` will detect that the collection is lazy\n    // and treat it lazily (because the collection is shared)\n    // and thus it will not load any keys because both sides of the join\n    // will be handled lazily\n    return { activeCollection: undefined, lazyCollection: undefined }\n  }\n\n  switch (joinType) {\n    case `left`:\n      return { activeCollection: `main`, lazyCollection: rightCollection }\n    case `right`:\n      return { activeCollection: `joined`, lazyCollection: leftCollection }\n    case `inner`:\n      // The smallest collection should be the active collection\n      // and the biggest collection should be lazy\n      return leftCollection.size < rightCollection.size\n        ? { activeCollection: `main`, lazyCollection: rightCollection }\n        : { activeCollection: `joined`, lazyCollection: leftCollection }\n    default:\n      return { activeCollection: undefined, lazyCollection: undefined }\n  }\n}\n"],"names":["JoinCollectionNotFoundError","compileExpression","map","UnsupportedJoinTypeError","index","followRef","ensureIndexForField","tap","findIndexForField","joinOperator","consolidate","InvalidJoinConditionTableMismatchError","InvalidJoinConditionSameTableError","InvalidJoinConditionLeftTableError","InvalidJoinConditionRightTableError","InvalidJoinCondition","CollectionInputNotFoundError","compileQuery","UnsupportedJoinSourceTypeError","filter"],"mappings":";;;;;;;;AAkDO,SAAS,aACd,UACA,aACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAE1B,QAAM;AAAA,IACJ,OAAO;AAAA,IACP,OAAO;AAAA,IACP,cAAc;AAAA,EAAA,IACZ;AAAA,IACF,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAE3B,QAAM,iBAAiB,YAAY,WAAW;AAC9C,QAAM,mBAAmB,YAAY,kBAAkB;AAEvD,MAAI,CAAC,gBAAgB;AACnB,UAAM,IAAIA,OAAAA,4BAA4B,WAAW;AAAA,EACnD;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAIA,OAAAA,4BAA4B,kBAAkB;AAAA,EAC1D;AAEA,QAAM,EAAE,kBAAkB,eAAA,IAAmB;AAAA,IAC3C,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,wBAAwB,OAAO,KAAK,MAAM;AAChD,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmBC,WAAAA,kBAAkB,QAAQ;AACnD,QAAM,qBAAqBA,WAAAA,kBAAkB,UAAU;AAGvD,MAAI,eAAe,SAAS;AAAA,IAC1BC,MAAAA,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,MAAI,iBAAiB,YAAY;AAAA,IAC/BA,MAAAA,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,WAAW,IAAI,GAAG;AACjE,UAAM,IAAIC,OAAAA,yBAAyB,WAAW,IAAI;AAAA,EACpD;AAEA,MAAI,kBAAkB;AAKpB,UAAM,WACJ,qBAAqB,SAAS,WAAW,OAAO,SAAS;AAC3D,UAAM,kBACJ,SAAS,SAAS,eACjB,SAAS,MAAM,SAAS,SAAS,MAAM;AAE1C,QAAI,CAAC,iBAAiB;AAUpB,sBAAgB,IAAI,eAAe,EAAE;AAErC,YAAM,iBACJ,qBAAqB,SAAS,eAAe;AAE/C,UAAIC;AAEJ,YAAM,yBACJ,qBAAqB,SAChB,aACA;AAEP,YAAM,kBAAkBC,MAAAA;AAAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,sBAAsB,gBAAgB;AAE5C,YAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,UAAI,WAAW;AACbC,kBAAAA;AAAAA,UACE;AAAA,UACA,gBAAgB;AAAA,UAChB;AAAA,QAAA;AAAA,MAEJ;AAEA,UAAI,cAAc;AAElB,YAAM,4BAEF,eAAe;AAAA,QACjBC,MAAAA,IAAI,CAAC,SAAS;AACZ,cAAI,aAAa;AACf;AAAA,UACF;AAMAH,gCAAUI,kBAAAA;AAAAA,YACR,oBAAoB;AAAA,YACpB,gBAAgB;AAAA,UAAA;AAOlB,gBAAM,sBAAsB,UAAU,eAAe,EAAE;AACvD,cAAI,CAAC,qBAAqB;AACxB,kBAAM,IAAI;AAAA,cACR;AAAA,YAAA;AAAA,UAEJ;AAEA,gBAAM,EAAE,UAAU,iBAAA,IAAqB;AAEvC,cAAIJ,WAASA,QAAM,SAAS,IAAI,GAAG;AAGjC,kBAAM,WAAW,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,OAAO;AAC7D,kBAAM,eAAeA,QAAM,OAAO,MAAM,QAAQ;AAEhD,qBAAS,YAAY;AAAA,UACvB,OAAO;AAGL,0BAAc;AACd,6BAAA;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MAAA;AAGH,UAAI,qBAAqB,QAAQ;AAC/B,uBAAe;AAAA,MACjB,OAAO;AACL,yBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,aAAa;AAAA,IAClBK,WAAa,gBAAgB,WAAW,IAAgB;AAAA,IACxDC,kBAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,0BACA,kBAC4D;AAE5D,QAAM,wBAAwB,yBAAyB;AAAA,IACrD,CAAC,UAAU,UAAU;AAAA,EAAA;AAGvB,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,kBACA,sBAAsB,SAAS,cAAc,KAC7C,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,mBACA,sBAAsB,SAAS,eAAe,GAC9C;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AAEvC,UAAM,IAAIC,OAAAA,uCAAA;AAAA,EACZ;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAIC,OAAAA,mCAAmC,cAAc;AAAA,EAC7D;AAKA,MAAI,CAAC,sBAAsB,SAAS,cAAc,GAAG;AACnD,UAAM,IAAIC,OAAAA,mCAAmC,cAAc;AAAA,EAC7D;AAGA,MAAI,oBAAoB,kBAAkB;AACxC,UAAM,IAAIC,OAAAA,oCAAoC,gBAAgB;AAAA,EAChE;AAGA,QAAM,IAAIC,OAAAA,qBAAA;AACZ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,aACA,WACA,iBACA,+BACA,OACA,cAC6D;AAC7D,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAIC,OAAAA,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,cAAc,KAAK,WAAW,GAAA;AAAA,IACnE;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiBC,MAAAA;AAAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnCf,MAAAA,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO;AAAA,QACL,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,cAAc,eAAe;AAAA,MAAA;AAAA,IAEjC;AAAA,IACA;AACE,YAAM,IAAIgB,OAAAA,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEdC,MAAAA,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACDjB,MAAAA,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAYA,SAAS,4BACP,UACA,gBACA,iBAG6D;AAC7D,MAAI,eAAe,OAAO,gBAAgB,IAAI;AAM5C,WAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EACxD;AAEA,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA;AAAA,IACrD,KAAK;AACH,aAAO,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACvD,KAAK;AAGH,aAAO,eAAe,OAAO,gBAAgB,OACzC,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA,IAC5C,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACpD;AACE,aAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EAAU;AAEtE;;"}

package/dist/cjs/query/live/collection-subscriber.cjs

@@ -83,14 +83,18 @@ class CollectionSubscriber {
     );
   }
   loadKeys(keys, filterFn) {
+    const changes = [];
     for (const key of keys) {
       if (this.sentKeys.has(key)) continue;
       const value = this.collection.get(key);
       if (value !== void 0 && filterFn(value)) {
         this.sentKeys.add(key);
-        this.sendChangesToPipeline([{ type: `insert`, key, value }]);
+        changes.push({ type: `insert`, key, value });
       }
     }
+    if (changes.length > 0) {
+      this.sendChangesToPipeline(changes);
+    }
   }
   subscribeToAllChanges(whereExpression) {
     const sendChangesToPipeline = this.sendChangesToPipeline.bind(this);

package/dist/cjs/query/live/collection-subscriber.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"collection-subscriber.cjs","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { createFilterFunctionFromExpression } from \"../../change-events.js\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the keys we've sent (needed for join and orderBy optimizations)\n  private sentKeys = new Set<string | number>()\n\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe() {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let unsubscribe: () => void\n    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {\n      unsubscribe = this.subscribeToMatchingChanges(whereExpression)\n    } else if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      unsubscribe = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      unsubscribe = this.subscribeToAllChanges(whereExpression)\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  // Wraps the sendChangesToPipeline function\n  // in order to turn `update`s into `insert`s\n  // for keys that have not been sent to the pipeline yet\n  // and filter out deletes for keys that have not been sent\n  private sendVisibleChangesToPipeline = (\n    changes: Array<ChangeMessage<any, string | number>>,\n    loadedInitialState: boolean\n  ) => {\n    if (loadedInitialState) {\n      // There was no index for the join key\n      // so we loaded the initial state\n      // so we can safely assume that the pipeline has seen all keys\n      return this.sendChangesToPipeline(changes)\n    }\n\n    const newChanges = []\n    for (const change of changes) {\n      let newChange = change\n      if (!this.sentKeys.has(change.key)) {\n        if (change.type === `update`) {\n          newChange = { ...change, type: `insert` }\n        } else if (change.type === `delete`) {\n          // filter out deletes for keys that have not been sent\n          continue\n        }\n        this.sentKeys.add(change.key)\n      }\n      newChanges.push(newChange)\n    }\n\n    return this.sendChangesToPipeline(newChanges)\n  }\n\n  private loadKeys(\n    keys: Iterable<string | number>,\n    filterFn: (item: object) => boolean\n  ) {\n    for (const key of keys) {\n      // Only load the key once\n      if (this.sentKeys.has(key)) continue\n\n      const value = this.collection.get(key)\n      if (value !== undefined && filterFn(value)) {\n        this.sentKeys.add(key)\n        this.sendChangesToPipeline([{ type: `insert`, key, value }])\n      }\n    }\n  }\n\n  private subscribeToAllChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this)\n    const unsubscribe = this.collection.subscribeChanges(\n      sendChangesToPipeline,\n      {\n        includeInitialState: true,\n        ...(whereExpression ? { whereExpression } : undefined),\n      }\n    )\n    return unsubscribe\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    // Flag to indicate we have send to whole initial state of the collection\n    // to the pipeline, this is set when there are no indexes that can be used\n    // to filter the changes and so the whole state was requested from the collection\n    let loadedInitialState = false\n\n    // Flag to indicate that we have started sending changes to the pipeline.\n    // This is set to true by either the first call to `loadKeys` or when the\n    // query requests the whole initial state in `loadInitialState`.\n    // Until that point we filter out all changes from subscription to the collection.\n    let sendChanges = false\n\n    const sendVisibleChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      // We filter out changes when sendChanges is false to ensure that we don't send\n      // any changes from the live subscription until the join operator requests either\n      // the initial state or its first key. This is needed otherwise it could receive\n      // changes which are then later subsumed by the initial state (and that would\n      // lead to weird bugs due to the data being received twice).\n      this.sendVisibleChangesToPipeline(\n        sendChanges ? changes : [],\n        loadedInitialState\n      )\n    }\n\n    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {\n      whereExpression,\n    })\n\n    // Create a function that loads keys from the collection\n    // into the query pipeline on demand\n    const filterFn = whereExpression\n      ? createFilterFunctionFromExpression(whereExpression)\n      : () => true\n    const loadKs = (keys: Set<string | number>) => {\n      sendChanges = true\n      return this.loadKeys(keys, filterFn)\n    }\n\n    // Store the functions to load keys and load initial state in the `lazyCollectionsCallbacks` map\n    // This is used by the join operator to dynamically load matching keys from the lazy collection\n    // or to get the full initial state of the collection if there's no index for the join key\n    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {\n      loadKeys: loadKs,\n      loadInitialState: () => {\n        // Make sure we only load the initial state once\n        if (loadedInitialState) return\n        loadedInitialState = true\n        sendChanges = true\n\n        const changes = this.collection.currentStateAsChanges({\n          whereExpression,\n        })\n        this.sendChangesToPipeline(changes)\n      },\n    }\n    return unsubscribe\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    this.loadNextItems(offset + limit)\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n      this.sendChangesToPipeline(\n        filteredChanges,\n        this.loadMoreIfNeeded.bind(this)\n      )\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    return unsubscribe\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded() {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load, just return true\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    let noMoreNextItems = false\n    if (n > 0) {\n      const loadedItems = this.loadNextItems(n)\n      noMoreNextItems = loadedItems === 0\n    }\n\n    // Indicate that we're done loading data if we didn't need to load more data\n    // or there's no more data to load\n    return n === 0 || noMoreNextItems\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this))\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number) {\n    const { valueExtractorForRawRow, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    const nextOrderedKeys = index.take(n, biggestSentValue)\n    const nextInserts: Array<ChangeMessage<any, string | number>> =\n      nextOrderedKeys.map((key) => {\n        return { type: `insert`, key, value: this.collection.get(key) }\n      })\n    this.sendChangesToPipelineWithTracking(nextInserts)\n    return nextInserts.length\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      this.sentKeys.add(change.key)\n\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n  input.sendData(new MultiSet(multiSetArray))\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller than the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":["convertToBasicExpression","createFilterFunctionFromExpression","MultiSet"],"mappings":";;;;;AAWO,MAAM,qBAGX;AAAA,EAOA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAVV,SAAQ,+BAAe,IAAA;AAGvB,SAAQ,UAAe;AAwFvB,SAAQ,+BAA+B,CACrC,SACA,uBACG;AACH,UAAI,oBAAoB;AAItB,eAAO,KAAK,sBAAsB,OAAO;AAAA,MAC3C;AAEA,YAAM,aAAa,CAAA;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,YAAY;AAChB,YAAI,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;AAClC,cAAI,OAAO,SAAS,UAAU;AAC5B,wBAAY,EAAE,GAAG,QAAQ,MAAM,SAAA;AAAA,UACjC,WAAW,OAAO,SAAS,UAAU;AAEnC;AAAA,UACF;AACA,eAAK,SAAS,IAAI,OAAO,GAAG;AAAA,QAC9B;AACA,mBAAW,KAAK,SAAS;AAAA,MAC3B;AAEA,aAAO,KAAK,sBAAsB,UAAU;AAAA,IAC9C;AAAA,EA3GG;AAAA,EAEH,YAAY;AACV,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkBA,YAAAA;AAAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,aAAK,mBAAmB,eAAe;AAAA,MACzC,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,WAAK,mBAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QAAI,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY,GAAG;AACvE,oBAAc,KAAK,2BAA2B,eAAe;AAAA,IAC/D,WACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,oBAAc,KAAK,0BAA0B,eAAe;AAAA,IAC9D,OAAO;AACL,oBAAc,KAAK,sBAAsB,eAAe;AAAA,IAC1D;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AAAA,EACrD;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAmCQ,SACN,MACA,UACA;AACA,eAAW,OAAO,MAAM;AAEtB,UAAI,KAAK,SAAS,IAAI,GAAG,EAAG;AAE5B,YAAM,QAAQ,KAAK,WAAW,IAAI,GAAG;AACrC,UAAI,UAAU,UAAa,SAAS,KAAK,GAAG;AAC1C,aAAK,SAAS,IAAI,GAAG;AACrB,aAAK,sBAAsB,CAAC,EAAE,MAAM,UAAU,KAAK,MAAA,CAAO,CAAC;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,sBACN,iBACA;AACA,UAAM,wBAAwB,KAAK,sBAAsB,KAAK,IAAI;AAClE,UAAM,cAAc,KAAK,WAAW;AAAA,MAClC;AAAA,MACA;AAAA,QACE,qBAAqB;AAAA,QACrB,GAAI,kBAAkB,EAAE,oBAAoB;AAAA,MAAA;AAAA,IAC9C;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,2BACN,iBACA;AAIA,QAAI,qBAAqB;AAMzB,QAAI,cAAc;AAElB,UAAM,qBAAqB,CACzB,YACG;AAMH,WAAK;AAAA,QACH,cAAc,UAAU,CAAA;AAAA,QACxB;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAID,UAAM,WAAW,kBACbC,aAAAA,mCAAmC,eAAe,IAClD,MAAM;AACV,UAAM,SAAS,CAAC,SAA+B;AAC7C,oBAAc;AACd,aAAO,KAAK,SAAS,MAAM,QAAQ;AAAA,IACrC;AAKA,SAAK,wBAAwB,yBAAyB,KAAK,YAAY,IAAI;AAAA,MACzE,UAAU;AAAA,MACV,kBAAkB,MAAM;AAEtB,YAAI,mBAAoB;AACxB,6BAAqB;AACrB,sBAAc;AAEd,cAAM,UAAU,KAAK,WAAW,sBAAsB;AAAA,UACpD;AAAA,QAAA,CACD;AACD,aAAK,sBAAsB,OAAO;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,WAAA,IACjC,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAIF,SAAK,cAAc,SAAS,KAAK;AAEjC,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AACA,WAAK;AAAA,QACH;AAAA,QACA,KAAK,iBAAiB,KAAK,IAAI;AAAA,MAAA;AAAA,IAEnC;AAIA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,mBAAmB;AACjB,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,kBAAkB;AACtB,QAAI,IAAI,GAAG;AACT,YAAM,cAAc,KAAK,cAAc,CAAC;AACxC,wBAAkB,gBAAgB;AAAA,IACpC;AAIA,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA,EAEQ,kCACN,SACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK,sBAAsB,gBAAgB,KAAK,iBAAiB,KAAK,IAAI,CAAC;AAAA,EAC7E;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW;AAC/B,UAAM,EAAE,yBAAyB,UAC/B,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,UAAM,kBAAkB,MAAM,KAAK,GAAG,gBAAgB;AACtD,UAAM,cACJ,gBAAgB,IAAI,CAAC,QAAQ;AAC3B,aAAO,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,WAAW,IAAI,GAAG,EAAA;AAAA,IAC9D,CAAC;AACH,SAAK,kCAAkC,WAAW;AAClD,WAAO,YAAY;AAAA,EACrB;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,WAAK,SAAS,IAAI,OAAO,GAAG;AAE5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;;AAEpB,QACE,WAAM,SAAN,mBAAY,UAAS,qBACrB,WAAM,KAAK,eAAX,mBAAuB,QAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,YACE,gBAAW,SAAX,mBAAiB,UAAS,qBAC1B,gBAAW,KAAK,eAAhB,mBAA4B,QAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,QAAM,SAAS,IAAIC,MAAAA,SAAS,aAAa,CAAC;AAC1C,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;;"}
+{"version":3,"file":"collection-subscriber.cjs","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { createFilterFunctionFromExpression } from \"../../change-events.js\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the keys we've sent (needed for join and orderBy optimizations)\n  private sentKeys = new Set<string | number>()\n\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe() {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let unsubscribe: () => void\n    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {\n      unsubscribe = this.subscribeToMatchingChanges(whereExpression)\n    } else if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      unsubscribe = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      unsubscribe = this.subscribeToAllChanges(whereExpression)\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  // Wraps the sendChangesToPipeline function\n  // in order to turn `update`s into `insert`s\n  // for keys that have not been sent to the pipeline yet\n  // and filter out deletes for keys that have not been sent\n  private sendVisibleChangesToPipeline = (\n    changes: Array<ChangeMessage<any, string | number>>,\n    loadedInitialState: boolean\n  ) => {\n    if (loadedInitialState) {\n      // There was no index for the join key\n      // so we loaded the initial state\n      // so we can safely assume that the pipeline has seen all keys\n      return this.sendChangesToPipeline(changes)\n    }\n\n    const newChanges = []\n    for (const change of changes) {\n      let newChange = change\n      if (!this.sentKeys.has(change.key)) {\n        if (change.type === `update`) {\n          newChange = { ...change, type: `insert` }\n        } else if (change.type === `delete`) {\n          // filter out deletes for keys that have not been sent\n          continue\n        }\n        this.sentKeys.add(change.key)\n      }\n      newChanges.push(newChange)\n    }\n\n    return this.sendChangesToPipeline(newChanges)\n  }\n\n  private loadKeys(\n    keys: Iterable<string | number>,\n    filterFn: (item: object) => boolean\n  ) {\n    const changes: Array<ChangeMessage<any, string | number>> = []\n    for (const key of keys) {\n      // Only load the key once\n      if (this.sentKeys.has(key)) continue\n\n      const value = this.collection.get(key)\n      if (value !== undefined && filterFn(value)) {\n        this.sentKeys.add(key)\n        changes.push({ type: `insert`, key, value })\n      }\n    }\n    if (changes.length > 0) {\n      this.sendChangesToPipeline(changes)\n    }\n  }\n\n  private subscribeToAllChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this)\n    const unsubscribe = this.collection.subscribeChanges(\n      sendChangesToPipeline,\n      {\n        includeInitialState: true,\n        ...(whereExpression ? { whereExpression } : undefined),\n      }\n    )\n    return unsubscribe\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    // Flag to indicate we have send to whole initial state of the collection\n    // to the pipeline, this is set when there are no indexes that can be used\n    // to filter the changes and so the whole state was requested from the collection\n    let loadedInitialState = false\n\n    // Flag to indicate that we have started sending changes to the pipeline.\n    // This is set to true by either the first call to `loadKeys` or when the\n    // query requests the whole initial state in `loadInitialState`.\n    // Until that point we filter out all changes from subscription to the collection.\n    let sendChanges = false\n\n    const sendVisibleChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      // We filter out changes when sendChanges is false to ensure that we don't send\n      // any changes from the live subscription until the join operator requests either\n      // the initial state or its first key. This is needed otherwise it could receive\n      // changes which are then later subsumed by the initial state (and that would\n      // lead to weird bugs due to the data being received twice).\n      this.sendVisibleChangesToPipeline(\n        sendChanges ? changes : [],\n        loadedInitialState\n      )\n    }\n\n    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {\n      whereExpression,\n    })\n\n    // Create a function that loads keys from the collection\n    // into the query pipeline on demand\n    const filterFn = whereExpression\n      ? createFilterFunctionFromExpression(whereExpression)\n      : () => true\n    const loadKs = (keys: Set<string | number>) => {\n      sendChanges = true\n      return this.loadKeys(keys, filterFn)\n    }\n\n    // Store the functions to load keys and load initial state in the `lazyCollectionsCallbacks` map\n    // This is used by the join operator to dynamically load matching keys from the lazy collection\n    // or to get the full initial state of the collection if there's no index for the join key\n    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {\n      loadKeys: loadKs,\n      loadInitialState: () => {\n        // Make sure we only load the initial state once\n        if (loadedInitialState) return\n        loadedInitialState = true\n        sendChanges = true\n\n        const changes = this.collection.currentStateAsChanges({\n          whereExpression,\n        })\n        this.sendChangesToPipeline(changes)\n      },\n    }\n    return unsubscribe\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    this.loadNextItems(offset + limit)\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n      this.sendChangesToPipeline(\n        filteredChanges,\n        this.loadMoreIfNeeded.bind(this)\n      )\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    return unsubscribe\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded() {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load, just return true\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    let noMoreNextItems = false\n    if (n > 0) {\n      const loadedItems = this.loadNextItems(n)\n      noMoreNextItems = loadedItems === 0\n    }\n\n    // Indicate that we're done loading data if we didn't need to load more data\n    // or there's no more data to load\n    return n === 0 || noMoreNextItems\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this))\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number) {\n    const { valueExtractorForRawRow, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    const nextOrderedKeys = index.take(n, biggestSentValue)\n    const nextInserts: Array<ChangeMessage<any, string | number>> =\n      nextOrderedKeys.map((key) => {\n        return { type: `insert`, key, value: this.collection.get(key) }\n      })\n    this.sendChangesToPipelineWithTracking(nextInserts)\n    return nextInserts.length\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      this.sentKeys.add(change.key)\n\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n  input.sendData(new MultiSet(multiSetArray))\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller than the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":["convertToBasicExpression","createFilterFunctionFromExpression","MultiSet"],"mappings":";;;;;AAWO,MAAM,qBAGX;AAAA,EAOA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAVV,SAAQ,+BAAe,IAAA;AAGvB,SAAQ,UAAe;AAwFvB,SAAQ,+BAA+B,CACrC,SACA,uBACG;AACH,UAAI,oBAAoB;AAItB,eAAO,KAAK,sBAAsB,OAAO;AAAA,MAC3C;AAEA,YAAM,aAAa,CAAA;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,YAAY;AAChB,YAAI,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;AAClC,cAAI,OAAO,SAAS,UAAU;AAC5B,wBAAY,EAAE,GAAG,QAAQ,MAAM,SAAA;AAAA,UACjC,WAAW,OAAO,SAAS,UAAU;AAEnC;AAAA,UACF;AACA,eAAK,SAAS,IAAI,OAAO,GAAG;AAAA,QAC9B;AACA,mBAAW,KAAK,SAAS;AAAA,MAC3B;AAEA,aAAO,KAAK,sBAAsB,UAAU;AAAA,IAC9C;AAAA,EA3GG;AAAA,EAEH,YAAY;AACV,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkBA,YAAAA;AAAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,aAAK,mBAAmB,eAAe;AAAA,MACzC,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,WAAK,mBAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QAAI,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY,GAAG;AACvE,oBAAc,KAAK,2BAA2B,eAAe;AAAA,IAC/D,WACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,oBAAc,KAAK,0BAA0B,eAAe;AAAA,IAC9D,OAAO;AACL,oBAAc,KAAK,sBAAsB,eAAe;AAAA,IAC1D;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AAAA,EACrD;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAmCQ,SACN,MACA,UACA;AACA,UAAM,UAAsD,CAAA;AAC5D,eAAW,OAAO,MAAM;AAEtB,UAAI,KAAK,SAAS,IAAI,GAAG,EAAG;AAE5B,YAAM,QAAQ,KAAK,WAAW,IAAI,GAAG;AACrC,UAAI,UAAU,UAAa,SAAS,KAAK,GAAG;AAC1C,aAAK,SAAS,IAAI,GAAG;AACrB,gBAAQ,KAAK,EAAE,MAAM,UAAU,KAAK,OAAO;AAAA,MAC7C;AAAA,IACF;AACA,QAAI,QAAQ,SAAS,GAAG;AACtB,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,sBACN,iBACA;AACA,UAAM,wBAAwB,KAAK,sBAAsB,KAAK,IAAI;AAClE,UAAM,cAAc,KAAK,WAAW;AAAA,MAClC;AAAA,MACA;AAAA,QACE,qBAAqB;AAAA,QACrB,GAAI,kBAAkB,EAAE,oBAAoB;AAAA,MAAA;AAAA,IAC9C;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,2BACN,iBACA;AAIA,QAAI,qBAAqB;AAMzB,QAAI,cAAc;AAElB,UAAM,qBAAqB,CACzB,YACG;AAMH,WAAK;AAAA,QACH,cAAc,UAAU,CAAA;AAAA,QACxB;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAID,UAAM,WAAW,kBACbC,aAAAA,mCAAmC,eAAe,IAClD,MAAM;AACV,UAAM,SAAS,CAAC,SAA+B;AAC7C,oBAAc;AACd,aAAO,KAAK,SAAS,MAAM,QAAQ;AAAA,IACrC;AAKA,SAAK,wBAAwB,yBAAyB,KAAK,YAAY,IAAI;AAAA,MACzE,UAAU;AAAA,MACV,kBAAkB,MAAM;AAEtB,YAAI,mBAAoB;AACxB,6BAAqB;AACrB,sBAAc;AAEd,cAAM,UAAU,KAAK,WAAW,sBAAsB;AAAA,UACpD;AAAA,QAAA,CACD;AACD,aAAK,sBAAsB,OAAO;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,WAAA,IACjC,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAIF,SAAK,cAAc,SAAS,KAAK;AAEjC,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AACA,WAAK;AAAA,QACH;AAAA,QACA,KAAK,iBAAiB,KAAK,IAAI;AAAA,MAAA;AAAA,IAEnC;AAIA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,mBAAmB;AACjB,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,kBAAkB;AACtB,QAAI,IAAI,GAAG;AACT,YAAM,cAAc,KAAK,cAAc,CAAC;AACxC,wBAAkB,gBAAgB;AAAA,IACpC;AAIA,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA,EAEQ,kCACN,SACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK,sBAAsB,gBAAgB,KAAK,iBAAiB,KAAK,IAAI,CAAC;AAAA,EAC7E;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW;AAC/B,UAAM,EAAE,yBAAyB,UAC/B,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,UAAM,kBAAkB,MAAM,KAAK,GAAG,gBAAgB;AACtD,UAAM,cACJ,gBAAgB,IAAI,CAAC,QAAQ;AAC3B,aAAO,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,WAAW,IAAI,GAAG,EAAA;AAAA,IAC9D,CAAC;AACH,SAAK,kCAAkC,WAAW;AAClD,WAAO,YAAY;AAAA,EACrB;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,WAAK,SAAS,IAAI,OAAO,GAAG;AAE5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;;AAEpB,QACE,WAAM,SAAN,mBAAY,UAAS,qBACrB,WAAM,KAAK,eAAX,mBAAuB,QAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,YACE,gBAAW,SAAX,mBAAiB,UAAS,qBAC1B,gBAAW,KAAK,eAAhB,mBAA4B,QAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,QAAM,SAAS,IAAIC,MAAAA,SAAS,aAAa,CAAC;AAC1C,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;;"}

package/dist/esm/query/builder/types.d.ts

@@ -345,7 +345,7 @@ type NonNull<T> = T extends null ? never : T;
  * ```
  */
 export type Ref<T = any> = {
-    [K in keyof T]: IsNonExactOptional<T[K]> extends true ? IsNonExactNullable<T[K]> extends true ? NonNullable<T[K]> extends Record<string, any> ? Ref<NonNullable<T[K]>> | undefined : RefLeaf<NonNullable<T[K]>> | undefined : NonUndefined<T[K]> extends Record<string, any> ? Ref<NonUndefined<T[K]>> | undefined : RefLeaf<NonUndefined<T[K]>> | undefined : IsNonExactNullable<T[K]> extends true ? NonNull<T[K]> extends Record<string, any> ? Ref<NonNull<T[K]>> | null : RefLeaf<NonNull<T[K]>> | null : T[K] extends Record<string, any> ? Ref<T[K]> : RefLeaf<T[K]>;
+    [K in keyof T]: IsNonExactOptional<T[K]> extends true ? IsNonExactNullable<T[K]> extends true ? IsPlainObject<NonNullable<T[K]>> extends true ? Ref<NonNullable<T[K]>> | undefined : RefLeaf<NonNullable<T[K]>> | undefined : IsPlainObject<NonUndefined<T[K]>> extends true ? Ref<NonUndefined<T[K]>> | undefined : RefLeaf<NonUndefined<T[K]>> | undefined : IsNonExactNullable<T[K]> extends true ? IsPlainObject<NonNull<T[K]>> extends true ? Ref<NonNull<T[K]>> | null : RefLeaf<NonNull<T[K]>> | null : IsPlainObject<T[K]> extends true ? Ref<T[K]> : RefLeaf<T[K]>;
 } & RefLeaf<T>;
 /**
  * Ref - The user-facing ref type with clean IDE display
@@ -564,4 +564,12 @@ export type WithResult<TContext extends Context, TResult> = Prettify<Omit<TConte
 export type Prettify<T> = {
     [K in keyof T]: T[K];
 } & {};
+/**
+ * IsPlainObject - Utility type to check if T is a plain object
+ */
+type IsPlainObject<T> = T extends unknown ? T extends object ? T extends ReadonlyArray<any> ? false : T extends JsBuiltIns ? false : true : false : false;
+/**
+ * JsBuiltIns - List of JavaScript built-ins
+ */
+type JsBuiltIns = ArrayBuffer | ArrayBufferLike | AsyncGenerator<any, any, any> | BigInt64Array | BigUint64Array | DataView | Date | Error | Float32Array | Float64Array | Function | Generator<any, any, any> | Int16Array | Int32Array | Int8Array | Map<any, any> | Promise<any> | RegExp | Set<any> | string | Uint16Array | Uint32Array | Uint8Array | Uint8ClampedArray | WeakMap<any, any> | WeakSet<any>;
 export {};

package/dist/esm/query/compiler/joins.js

@@ -103,7 +103,7 @@ function processJoin(pipeline, joinClause, tables, mainTableId, mainTableAlias,
       }
       let deoptimized = false;
       const activePipelineWithLoading = activePipeline.pipe(
-        tap(([joinKey, _]) => {
+        tap((data) => {
           if (deoptimized) {
             return;
           }
@@ -118,8 +118,9 @@ function processJoin(pipeline, joinClause, tables, mainTableId, mainTableAlias,
             );
           }
           const { loadKeys, loadInitialState } = collectionCallbacks;
-          if (index && index.supports(`eq`)) {
-            const matchingKeys = index.lookup(`eq`, joinKey);
+          if (index && index.supports(`in`)) {
+            const joinKeys = data.getInner().map(([[joinKey]]) => joinKey);
+            const matchingKeys = index.lookup(`in`, joinKeys);
             loadKeys(matchingKeys);
           } else {
             deoptimized = true;

package/dist/esm/query/compiler/joins.js.map

@@ -1 +1 @@
-{"version":3,"file":"joins.js","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n  tap,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinCondition,\n  InvalidJoinConditionLeftTableError,\n  InvalidJoinConditionRightTableError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  JoinCollectionNotFoundError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery, followRef } from \"./index.js\"\nimport type { OrderByOptimizationInfo } from \"./order-by.js\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  PropRef,\n  QueryIR,\n  QueryRef,\n} from \"../ir.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\n\nexport type LoadKeysFn = (key: Set<string | number>) => void\nexport type LazyCollectionCallbacks = {\n  loadKeys: LoadKeysFn\n  loadInitialState: () => void\n}\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableId,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping,\n      collections,\n      callbacks,\n      lazyCollections,\n      optimizableOrderByCollections,\n      rawQuery\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const {\n    alias: joinedTableAlias,\n    input: joinedInput,\n    collectionId: joinedCollectionId,\n  } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    collections,\n    callbacks,\n    lazyCollections,\n    optimizableOrderByCollections,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  const mainCollection = collections[mainTableId]\n  const joinedCollection = collections[joinedCollectionId]\n\n  if (!mainCollection) {\n    throw new JoinCollectionNotFoundError(mainTableId)\n  }\n\n  if (!joinedCollection) {\n    throw new JoinCollectionNotFoundError(joinedCollectionId)\n  }\n\n  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(\n    joinClause.type,\n    mainCollection,\n    joinedCollection\n  )\n\n  // Analyze which table each expression refers to and swap if necessary\n  const availableTableAliases = Object.keys(tables)\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    availableTableAliases,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  let mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  let joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n\n  if (activeCollection) {\n    // If the lazy collection comes from a subquery that has a limit and/or an offset clause\n    // then we need to deoptimize the join because we don't know which rows are in the result set\n    // since we simply lookup matching keys in the index but the index contains all rows\n    // (not just the ones that pass the limit and offset clauses)\n    const lazyFrom =\n      activeCollection === `main` ? joinClause.from : rawQuery.from\n    const limitedSubquery =\n      lazyFrom.type === `queryRef` &&\n      (lazyFrom.query.limit || lazyFrom.query.offset)\n\n    if (!limitedSubquery) {\n      // This join can be optimized by having the active collection\n      // dynamically load keys into the lazy collection\n      // based on the value of the joinKey and by looking up\n      // matching rows in the index of the lazy collection\n\n      // Mark the lazy collection as lazy\n      // this Set is passed by the liveQueryCollection to the compiler\n      // such that the liveQueryCollection can check it after compilation\n      // to know which collections are lazy collections\n      lazyCollections.add(lazyCollection.id)\n\n      const activePipeline =\n        activeCollection === `main` ? mainPipeline : joinedPipeline\n\n      let index: BaseIndex<string | number> | undefined\n\n      const lazyCollectionJoinExpr =\n        activeCollection === `main`\n          ? (joinedExpr as PropRef)\n          : (mainExpr as PropRef)\n\n      const followRefResult = followRef(\n        rawQuery,\n        lazyCollectionJoinExpr,\n        lazyCollection\n      )!\n      const followRefCollection = followRefResult.collection\n\n      const fieldName = followRefResult.path[0]\n      if (fieldName) {\n        ensureIndexForField(\n          fieldName,\n          followRefResult.path,\n          followRefCollection\n        )\n      }\n\n      let deoptimized = false\n\n      const activePipelineWithLoading: IStreamBuilder<\n        [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]\n      > = activePipeline.pipe(\n        tap(([joinKey, _]) => {\n          if (deoptimized) {\n            return\n          }\n\n          // Find the index for the path we join on\n          // we need to find the index inside the map operator\n          // because the indexes are only available after the initial sync\n          // so we can't fetch it during compilation\n          index ??= findIndexForField(\n            followRefCollection.indexes,\n            followRefResult.path\n          )\n\n          // The `callbacks` object is passed by the liveQueryCollection to the compiler.\n          // It contains a function to lazy load keys for each lazy collection\n          // as well as a function to switch back to a regular collection\n          // (useful when there's no index for available for lazily loading the collection)\n          const collectionCallbacks = callbacks[lazyCollection.id]\n          if (!collectionCallbacks) {\n            throw new Error(\n              `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`\n            )\n          }\n\n          const { loadKeys, loadInitialState } = collectionCallbacks\n\n          if (index && index.supports(`eq`)) {\n            // Use the index to fetch the PKs of the rows in the lazy collection\n            // that match this row from the active collection based on the value of the joinKey\n            const matchingKeys = index.lookup(`eq`, joinKey)\n            // Inform the lazy collection that those keys need to be loaded\n            loadKeys(matchingKeys)\n          } else {\n            // We can't optimize the join because there is no index for the join key\n            // on the lazy collection, so we load the initial state\n            deoptimized = true\n            loadInitialState()\n          }\n        })\n      )\n\n      if (activeCollection === `main`) {\n        mainPipeline = activePipelineWithLoading\n      } else {\n        joinedPipeline = activePipelineWithLoading\n      }\n    }\n  }\n\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinClause.type as JoinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (available table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  allAvailableTableAliases: Array<string>,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  // Filter out the joined table alias from the available table aliases\n  const availableTableAliases = allAvailableTableAliases.filter(\n    (alias) => alias !== joinedTableAlias\n  )\n\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to an available table and right refers to joined table, keep as is\n  if (\n    leftTableAlias &&\n    availableTableAliases.includes(leftTableAlias) &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to an available table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias &&\n    availableTableAliases.includes(rightTableAlias)\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If one expression doesn't refer to any table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    // For backward compatibility, use the first available table alias in error message\n    throw new InvalidJoinConditionTableMismatchError()\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias)\n  }\n\n  // Left side must refer to an available table\n  // This cannot happen with the query builder as there is no way to build a ref\n  // to an unavailable table, but just in case, but could happen with the IR\n  if (!availableTableAliases.includes(leftTableAlias)) {\n    throw new InvalidJoinConditionLeftTableError(leftTableAlias)\n  }\n\n  // Right side must refer to the joined table\n  if (rightTableAlias !== joinedTableAlias) {\n    throw new InvalidJoinConditionRightTableError(joinedTableAlias)\n  }\n\n  // This should not be reachable given the logic above, but just in case\n  throw new InvalidJoinCondition()\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream; collectionId: string } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input, collectionId: from.collection.id }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        collections,\n        callbacks,\n        lazyCollections,\n        optimizableOrderByCollections,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return {\n        alias: from.alias,\n        input: extractedInput as KeyedStream,\n        collectionId: subQueryResult.collectionId,\n      }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n\n/**\n * Returns the active and lazy collections for a join clause.\n * The active collection is the one that we need to fully iterate over\n * and it can be the main table (i.e. left collection) or the joined table (i.e. right collection).\n * The lazy collection is the one that we should join-in lazily based on matches in the active collection.\n * @param joinClause - The join clause to analyze\n * @param leftCollection - The left collection\n * @param rightCollection - The right collection\n * @returns The active and lazy collections. They are undefined if we need to loop over both collections (i.e. both are active)\n */\nfunction getActiveAndLazyCollections(\n  joinType: JoinClause[`type`],\n  leftCollection: Collection,\n  rightCollection: Collection\n):\n  | { activeCollection: `main` | `joined`; lazyCollection: Collection }\n  | { activeCollection: undefined; lazyCollection: undefined } {\n  if (leftCollection.id === rightCollection.id) {\n    // We can't apply this optimization if there's only one collection\n    // because `liveQueryCollection` will detect that the collection is lazy\n    // and treat it lazily (because the collection is shared)\n    // and thus it will not load any keys because both sides of the join\n    // will be handled lazily\n    return { activeCollection: undefined, lazyCollection: undefined }\n  }\n\n  switch (joinType) {\n    case `left`:\n      return { activeCollection: `main`, lazyCollection: rightCollection }\n    case `right`:\n      return { activeCollection: `joined`, lazyCollection: leftCollection }\n    case `inner`:\n      // The smallest collection should be the active collection\n      // and the biggest collection should be lazy\n      return leftCollection.size < rightCollection.size\n        ? { activeCollection: `main`, lazyCollection: rightCollection }\n        : { activeCollection: `joined`, lazyCollection: leftCollection }\n    default:\n      return { activeCollection: undefined, lazyCollection: undefined }\n  }\n}\n"],"names":["joinOperator"],"mappings":";;;;;;AAkDO,SAAS,aACd,UACA,aACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAE1B,QAAM;AAAA,IACJ,OAAO;AAAA,IACP,OAAO;AAAA,IACP,cAAc;AAAA,EAAA,IACZ;AAAA,IACF,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAE3B,QAAM,iBAAiB,YAAY,WAAW;AAC9C,QAAM,mBAAmB,YAAY,kBAAkB;AAEvD,MAAI,CAAC,gBAAgB;AACnB,UAAM,IAAI,4BAA4B,WAAW;AAAA,EACnD;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAI,4BAA4B,kBAAkB;AAAA,EAC1D;AAEA,QAAM,EAAE,kBAAkB,eAAA,IAAmB;AAAA,IAC3C,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,wBAAwB,OAAO,KAAK,MAAM;AAChD,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmB,kBAAkB,QAAQ;AACnD,QAAM,qBAAqB,kBAAkB,UAAU;AAGvD,MAAI,eAAe,SAAS;AAAA,IAC1B,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,MAAI,iBAAiB,YAAY;AAAA,IAC/B,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,WAAW,IAAI,GAAG;AACjE,UAAM,IAAI,yBAAyB,WAAW,IAAI;AAAA,EACpD;AAEA,MAAI,kBAAkB;AAKpB,UAAM,WACJ,qBAAqB,SAAS,WAAW,OAAO,SAAS;AAC3D,UAAM,kBACJ,SAAS,SAAS,eACjB,SAAS,MAAM,SAAS,SAAS,MAAM;AAE1C,QAAI,CAAC,iBAAiB;AAUpB,sBAAgB,IAAI,eAAe,EAAE;AAErC,YAAM,iBACJ,qBAAqB,SAAS,eAAe;AAE/C,UAAI;AAEJ,YAAM,yBACJ,qBAAqB,SAChB,aACA;AAEP,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,sBAAsB,gBAAgB;AAE5C,YAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,UAAI,WAAW;AACb;AAAA,UACE;AAAA,UACA,gBAAgB;AAAA,UAChB;AAAA,QAAA;AAAA,MAEJ;AAEA,UAAI,cAAc;AAElB,YAAM,4BAEF,eAAe;AAAA,QACjB,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM;AACpB,cAAI,aAAa;AACf;AAAA,UACF;AAMA,4BAAU;AAAA,YACR,oBAAoB;AAAA,YACpB,gBAAgB;AAAA,UAAA;AAOlB,gBAAM,sBAAsB,UAAU,eAAe,EAAE;AACvD,cAAI,CAAC,qBAAqB;AACxB,kBAAM,IAAI;AAAA,cACR;AAAA,YAAA;AAAA,UAEJ;AAEA,gBAAM,EAAE,UAAU,iBAAA,IAAqB;AAEvC,cAAI,SAAS,MAAM,SAAS,IAAI,GAAG;AAGjC,kBAAM,eAAe,MAAM,OAAO,MAAM,OAAO;AAE/C,qBAAS,YAAY;AAAA,UACvB,OAAO;AAGL,0BAAc;AACd,6BAAA;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MAAA;AAGH,UAAI,qBAAqB,QAAQ;AAC/B,uBAAe;AAAA,MACjB,OAAO;AACL,yBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,aAAa;AAAA,IAClBA,KAAa,gBAAgB,WAAW,IAAgB;AAAA,IACxD,YAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,0BACA,kBAC4D;AAE5D,QAAM,wBAAwB,yBAAyB;AAAA,IACrD,CAAC,UAAU,UAAU;AAAA,EAAA;AAGvB,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,kBACA,sBAAsB,SAAS,cAAc,KAC7C,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,mBACA,sBAAsB,SAAS,eAAe,GAC9C;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AAEvC,UAAM,IAAI,uCAAA;AAAA,EACZ;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAI,mCAAmC,cAAc;AAAA,EAC7D;AAKA,MAAI,CAAC,sBAAsB,SAAS,cAAc,GAAG;AACnD,UAAM,IAAI,mCAAmC,cAAc;AAAA,EAC7D;AAGA,MAAI,oBAAoB,kBAAkB;AACxC,UAAM,IAAI,oCAAoC,gBAAgB;AAAA,EAChE;AAGA,QAAM,IAAI,qBAAA;AACZ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,aACA,WACA,iBACA,+BACA,OACA,cAC6D;AAC7D,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAI,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,cAAc,KAAK,WAAW,GAAA;AAAA,IACnE;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiB;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnC,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO;AAAA,QACL,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,cAAc,eAAe;AAAA,MAAA;AAAA,IAEjC;AAAA,IACA;AACE,YAAM,IAAI,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEd,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACD,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAYA,SAAS,4BACP,UACA,gBACA,iBAG6D;AAC7D,MAAI,eAAe,OAAO,gBAAgB,IAAI;AAM5C,WAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EACxD;AAEA,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA;AAAA,IACrD,KAAK;AACH,aAAO,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACvD,KAAK;AAGH,aAAO,eAAe,OAAO,gBAAgB,OACzC,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA,IAC5C,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACpD;AACE,aAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EAAU;AAEtE;"}
+{"version":3,"file":"joins.js","sources":["../../../../src/query/compiler/joins.ts"],"sourcesContent":["import {\n  consolidate,\n  filter,\n  join as joinOperator,\n  map,\n  tap,\n} from \"@tanstack/db-ivm\"\nimport {\n  CollectionInputNotFoundError,\n  InvalidJoinCondition,\n  InvalidJoinConditionLeftTableError,\n  InvalidJoinConditionRightTableError,\n  InvalidJoinConditionSameTableError,\n  InvalidJoinConditionTableMismatchError,\n  JoinCollectionNotFoundError,\n  UnsupportedJoinSourceTypeError,\n  UnsupportedJoinTypeError,\n} from \"../../errors.js\"\nimport { findIndexForField } from \"../../utils/index-optimization.js\"\nimport { ensureIndexForField } from \"../../indexes/auto-index.js\"\nimport { compileExpression } from \"./evaluators.js\"\nimport { compileQuery, followRef } from \"./index.js\"\nimport type { OrderByOptimizationInfo } from \"./order-by.js\"\nimport type {\n  BasicExpression,\n  CollectionRef,\n  JoinClause,\n  PropRef,\n  QueryIR,\n  QueryRef,\n} from \"../ir.js\"\nimport type { IStreamBuilder, JoinType } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type {\n  KeyedStream,\n  NamespacedAndKeyedStream,\n  NamespacedRow,\n} from \"../../types.js\"\nimport type { QueryCache, QueryMapping } from \"./types.js\"\nimport type { BaseIndex } from \"../../indexes/base-index.js\"\n\nexport type LoadKeysFn = (key: Set<string | number>) => void\nexport type LazyCollectionCallbacks = {\n  loadKeys: LoadKeysFn\n  loadInitialState: () => void\n}\n\n/**\n * Processes all join clauses in a query\n */\nexport function processJoins(\n  pipeline: NamespacedAndKeyedStream,\n  joinClauses: Array<JoinClause>,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  let resultPipeline = pipeline\n\n  for (const joinClause of joinClauses) {\n    resultPipeline = processJoin(\n      resultPipeline,\n      joinClause,\n      tables,\n      mainTableId,\n      mainTableAlias,\n      allInputs,\n      cache,\n      queryMapping,\n      collections,\n      callbacks,\n      lazyCollections,\n      optimizableOrderByCollections,\n      rawQuery\n    )\n  }\n\n  return resultPipeline\n}\n\n/**\n * Processes a single join clause\n */\nfunction processJoin(\n  pipeline: NamespacedAndKeyedStream,\n  joinClause: JoinClause,\n  tables: Record<string, KeyedStream>,\n  mainTableId: string,\n  mainTableAlias: string,\n  allInputs: Record<string, KeyedStream>,\n  cache: QueryCache,\n  queryMapping: QueryMapping,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  rawQuery: QueryIR\n): NamespacedAndKeyedStream {\n  // Get the joined table alias and input stream\n  const {\n    alias: joinedTableAlias,\n    input: joinedInput,\n    collectionId: joinedCollectionId,\n  } = processJoinSource(\n    joinClause.from,\n    allInputs,\n    collections,\n    callbacks,\n    lazyCollections,\n    optimizableOrderByCollections,\n    cache,\n    queryMapping\n  )\n\n  // Add the joined table to the tables map\n  tables[joinedTableAlias] = joinedInput\n\n  const mainCollection = collections[mainTableId]\n  const joinedCollection = collections[joinedCollectionId]\n\n  if (!mainCollection) {\n    throw new JoinCollectionNotFoundError(mainTableId)\n  }\n\n  if (!joinedCollection) {\n    throw new JoinCollectionNotFoundError(joinedCollectionId)\n  }\n\n  const { activeCollection, lazyCollection } = getActiveAndLazyCollections(\n    joinClause.type,\n    mainCollection,\n    joinedCollection\n  )\n\n  // Analyze which table each expression refers to and swap if necessary\n  const availableTableAliases = Object.keys(tables)\n  const { mainExpr, joinedExpr } = analyzeJoinExpressions(\n    joinClause.left,\n    joinClause.right,\n    availableTableAliases,\n    joinedTableAlias\n  )\n\n  // Pre-compile the join expressions\n  const compiledMainExpr = compileExpression(mainExpr)\n  const compiledJoinedExpr = compileExpression(joinedExpr)\n\n  // Prepare the main pipeline for joining\n  let mainPipeline = pipeline.pipe(\n    map(([currentKey, namespacedRow]) => {\n      // Extract the join key from the main table expression\n      const mainKey = compiledMainExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [mainKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Prepare the joined pipeline\n  let joinedPipeline = joinedInput.pipe(\n    map(([currentKey, row]) => {\n      // Wrap the row in a namespaced structure\n      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }\n\n      // Extract the join key from the joined table expression\n      const joinedKey = compiledJoinedExpr(namespacedRow)\n\n      // Return [joinKey, [originalKey, namespacedRow]]\n      return [joinedKey, [currentKey, namespacedRow]] as [\n        unknown,\n        [string, typeof namespacedRow],\n      ]\n    })\n  )\n\n  // Apply the join operation\n  if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) {\n    throw new UnsupportedJoinTypeError(joinClause.type)\n  }\n\n  if (activeCollection) {\n    // If the lazy collection comes from a subquery that has a limit and/or an offset clause\n    // then we need to deoptimize the join because we don't know which rows are in the result set\n    // since we simply lookup matching keys in the index but the index contains all rows\n    // (not just the ones that pass the limit and offset clauses)\n    const lazyFrom =\n      activeCollection === `main` ? joinClause.from : rawQuery.from\n    const limitedSubquery =\n      lazyFrom.type === `queryRef` &&\n      (lazyFrom.query.limit || lazyFrom.query.offset)\n\n    if (!limitedSubquery) {\n      // This join can be optimized by having the active collection\n      // dynamically load keys into the lazy collection\n      // based on the value of the joinKey and by looking up\n      // matching rows in the index of the lazy collection\n\n      // Mark the lazy collection as lazy\n      // this Set is passed by the liveQueryCollection to the compiler\n      // such that the liveQueryCollection can check it after compilation\n      // to know which collections are lazy collections\n      lazyCollections.add(lazyCollection.id)\n\n      const activePipeline =\n        activeCollection === `main` ? mainPipeline : joinedPipeline\n\n      let index: BaseIndex<string | number> | undefined\n\n      const lazyCollectionJoinExpr =\n        activeCollection === `main`\n          ? (joinedExpr as PropRef)\n          : (mainExpr as PropRef)\n\n      const followRefResult = followRef(\n        rawQuery,\n        lazyCollectionJoinExpr,\n        lazyCollection\n      )!\n      const followRefCollection = followRefResult.collection\n\n      const fieldName = followRefResult.path[0]\n      if (fieldName) {\n        ensureIndexForField(\n          fieldName,\n          followRefResult.path,\n          followRefCollection\n        )\n      }\n\n      let deoptimized = false\n\n      const activePipelineWithLoading: IStreamBuilder<\n        [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]\n      > = activePipeline.pipe(\n        tap((data) => {\n          if (deoptimized) {\n            return\n          }\n\n          // Find the index for the path we join on\n          // we need to find the index inside the map operator\n          // because the indexes are only available after the initial sync\n          // so we can't fetch it during compilation\n          index ??= findIndexForField(\n            followRefCollection.indexes,\n            followRefResult.path\n          )\n\n          // The `callbacks` object is passed by the liveQueryCollection to the compiler.\n          // It contains a function to lazy load keys for each lazy collection\n          // as well as a function to switch back to a regular collection\n          // (useful when there's no index for available for lazily loading the collection)\n          const collectionCallbacks = callbacks[lazyCollection.id]\n          if (!collectionCallbacks) {\n            throw new Error(\n              `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`\n            )\n          }\n\n          const { loadKeys, loadInitialState } = collectionCallbacks\n\n          if (index && index.supports(`in`)) {\n            // Use the index to fetch the PKs of the rows in the lazy collection\n            // that match this row from the active collection based on the value of the joinKey\n            const joinKeys = data.getInner().map(([[joinKey]]) => joinKey)\n            const matchingKeys = index.lookup(`in`, joinKeys)\n            // Inform the lazy collection that those keys need to be loaded\n            loadKeys(matchingKeys)\n          } else {\n            // We can't optimize the join because there is no index for the join key\n            // on the lazy collection, so we load the initial state\n            deoptimized = true\n            loadInitialState()\n          }\n        })\n      )\n\n      if (activeCollection === `main`) {\n        mainPipeline = activePipelineWithLoading\n      } else {\n        joinedPipeline = activePipelineWithLoading\n      }\n    }\n  }\n\n  return mainPipeline.pipe(\n    joinOperator(joinedPipeline, joinClause.type as JoinType),\n    consolidate(),\n    processJoinResults(joinClause.type)\n  )\n}\n\n/**\n * Analyzes join expressions to determine which refers to which table\n * and returns them in the correct order (available table expression first, joined table expression second)\n */\nfunction analyzeJoinExpressions(\n  left: BasicExpression,\n  right: BasicExpression,\n  allAvailableTableAliases: Array<string>,\n  joinedTableAlias: string\n): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {\n  // Filter out the joined table alias from the available table aliases\n  const availableTableAliases = allAvailableTableAliases.filter(\n    (alias) => alias !== joinedTableAlias\n  )\n\n  const leftTableAlias = getTableAliasFromExpression(left)\n  const rightTableAlias = getTableAliasFromExpression(right)\n\n  // If left expression refers to an available table and right refers to joined table, keep as is\n  if (\n    leftTableAlias &&\n    availableTableAliases.includes(leftTableAlias) &&\n    rightTableAlias === joinedTableAlias\n  ) {\n    return { mainExpr: left, joinedExpr: right }\n  }\n\n  // If left expression refers to joined table and right refers to an available table, swap them\n  if (\n    leftTableAlias === joinedTableAlias &&\n    rightTableAlias &&\n    availableTableAliases.includes(rightTableAlias)\n  ) {\n    return { mainExpr: right, joinedExpr: left }\n  }\n\n  // If one expression doesn't refer to any table, this is an invalid join\n  if (!leftTableAlias || !rightTableAlias) {\n    // For backward compatibility, use the first available table alias in error message\n    throw new InvalidJoinConditionTableMismatchError()\n  }\n\n  // If both expressions refer to the same alias, this is an invalid join\n  if (leftTableAlias === rightTableAlias) {\n    throw new InvalidJoinConditionSameTableError(leftTableAlias)\n  }\n\n  // Left side must refer to an available table\n  // This cannot happen with the query builder as there is no way to build a ref\n  // to an unavailable table, but just in case, but could happen with the IR\n  if (!availableTableAliases.includes(leftTableAlias)) {\n    throw new InvalidJoinConditionLeftTableError(leftTableAlias)\n  }\n\n  // Right side must refer to the joined table\n  if (rightTableAlias !== joinedTableAlias) {\n    throw new InvalidJoinConditionRightTableError(joinedTableAlias)\n  }\n\n  // This should not be reachable given the logic above, but just in case\n  throw new InvalidJoinCondition()\n}\n\n/**\n * Extracts the table alias from a join expression\n */\nfunction getTableAliasFromExpression(expr: BasicExpression): string | null {\n  switch (expr.type) {\n    case `ref`:\n      // PropRef path has the table alias as the first element\n      return expr.path[0] || null\n    case `func`: {\n      // For function expressions, we need to check if all arguments refer to the same table\n      const tableAliases = new Set<string>()\n      for (const arg of expr.args) {\n        const alias = getTableAliasFromExpression(arg)\n        if (alias) {\n          tableAliases.add(alias)\n        }\n      }\n      // If all arguments refer to the same table, return that table alias\n      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null\n    }\n    default:\n      // Values (type='val') don't reference any table\n      return null\n  }\n}\n\n/**\n * Processes the join source (collection or sub-query)\n */\nfunction processJoinSource(\n  from: CollectionRef | QueryRef,\n  allInputs: Record<string, KeyedStream>,\n  collections: Record<string, Collection>,\n  callbacks: Record<string, LazyCollectionCallbacks>,\n  lazyCollections: Set<string>,\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>,\n  cache: QueryCache,\n  queryMapping: QueryMapping\n): { alias: string; input: KeyedStream; collectionId: string } {\n  switch (from.type) {\n    case `collectionRef`: {\n      const input = allInputs[from.collection.id]\n      if (!input) {\n        throw new CollectionInputNotFoundError(from.collection.id)\n      }\n      return { alias: from.alias, input, collectionId: from.collection.id }\n    }\n    case `queryRef`: {\n      // Find the original query for caching purposes\n      const originalQuery = queryMapping.get(from.query) || from.query\n\n      // Recursively compile the sub-query with cache\n      const subQueryResult = compileQuery(\n        originalQuery,\n        allInputs,\n        collections,\n        callbacks,\n        lazyCollections,\n        optimizableOrderByCollections,\n        cache,\n        queryMapping\n      )\n\n      // Extract the pipeline from the compilation result\n      const subQueryInput = subQueryResult.pipeline\n\n      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)\n      // We need to extract just the value for use in parent queries\n      const extractedInput = subQueryInput.pipe(\n        map((data: any) => {\n          const [key, [value, _orderByIndex]] = data\n          return [key, value] as [unknown, any]\n        })\n      )\n\n      return {\n        alias: from.alias,\n        input: extractedInput as KeyedStream,\n        collectionId: subQueryResult.collectionId,\n      }\n    }\n    default:\n      throw new UnsupportedJoinSourceTypeError((from as any).type)\n  }\n}\n\n/**\n * Processes the results of a join operation\n */\nfunction processJoinResults(joinType: string) {\n  return function (\n    pipeline: IStreamBuilder<\n      [\n        key: string,\n        [\n          [string, NamespacedRow] | undefined,\n          [string, NamespacedRow] | undefined,\n        ],\n      ]\n    >\n  ): NamespacedAndKeyedStream {\n    return pipeline.pipe(\n      // Process the join result and handle nulls\n      filter((result) => {\n        const [_key, [main, joined]] = result\n        const mainNamespacedRow = main?.[1]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Handle different join types\n        if (joinType === `inner`) {\n          return !!(mainNamespacedRow && joinedNamespacedRow)\n        }\n\n        if (joinType === `left`) {\n          return !!mainNamespacedRow\n        }\n\n        if (joinType === `right`) {\n          return !!joinedNamespacedRow\n        }\n\n        // For full joins, always include\n        return true\n      }),\n      map((result) => {\n        const [_key, [main, joined]] = result\n        const mainKey = main?.[0]\n        const mainNamespacedRow = main?.[1]\n        const joinedKey = joined?.[0]\n        const joinedNamespacedRow = joined?.[1]\n\n        // Merge the namespaced rows\n        const mergedNamespacedRow: NamespacedRow = {}\n\n        // Add main row data if it exists\n        if (mainNamespacedRow) {\n          Object.assign(mergedNamespacedRow, mainNamespacedRow)\n        }\n\n        // Add joined row data if it exists\n        if (joinedNamespacedRow) {\n          Object.assign(mergedNamespacedRow, joinedNamespacedRow)\n        }\n\n        // We create a composite key that combines the main and joined keys\n        const resultKey = `[${mainKey},${joinedKey}]`\n\n        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]\n      })\n    )\n  }\n}\n\n/**\n * Returns the active and lazy collections for a join clause.\n * The active collection is the one that we need to fully iterate over\n * and it can be the main table (i.e. left collection) or the joined table (i.e. right collection).\n * The lazy collection is the one that we should join-in lazily based on matches in the active collection.\n * @param joinClause - The join clause to analyze\n * @param leftCollection - The left collection\n * @param rightCollection - The right collection\n * @returns The active and lazy collections. They are undefined if we need to loop over both collections (i.e. both are active)\n */\nfunction getActiveAndLazyCollections(\n  joinType: JoinClause[`type`],\n  leftCollection: Collection,\n  rightCollection: Collection\n):\n  | { activeCollection: `main` | `joined`; lazyCollection: Collection }\n  | { activeCollection: undefined; lazyCollection: undefined } {\n  if (leftCollection.id === rightCollection.id) {\n    // We can't apply this optimization if there's only one collection\n    // because `liveQueryCollection` will detect that the collection is lazy\n    // and treat it lazily (because the collection is shared)\n    // and thus it will not load any keys because both sides of the join\n    // will be handled lazily\n    return { activeCollection: undefined, lazyCollection: undefined }\n  }\n\n  switch (joinType) {\n    case `left`:\n      return { activeCollection: `main`, lazyCollection: rightCollection }\n    case `right`:\n      return { activeCollection: `joined`, lazyCollection: leftCollection }\n    case `inner`:\n      // The smallest collection should be the active collection\n      // and the biggest collection should be lazy\n      return leftCollection.size < rightCollection.size\n        ? { activeCollection: `main`, lazyCollection: rightCollection }\n        : { activeCollection: `joined`, lazyCollection: leftCollection }\n    default:\n      return { activeCollection: undefined, lazyCollection: undefined }\n  }\n}\n"],"names":["joinOperator"],"mappings":";;;;;;AAkDO,SAAS,aACd,UACA,aACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAC1B,MAAI,iBAAiB;AAErB,aAAW,cAAc,aAAa;AACpC,qBAAiB;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,SAAO;AACT;AAKA,SAAS,YACP,UACA,YACA,QACA,aACA,gBACA,WACA,OACA,cACA,aACA,WACA,iBACA,+BACA,UAC0B;AAE1B,QAAM;AAAA,IACJ,OAAO;AAAA,IACP,OAAO;AAAA,IACP,cAAc;AAAA,EAAA,IACZ;AAAA,IACF,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAIF,SAAO,gBAAgB,IAAI;AAE3B,QAAM,iBAAiB,YAAY,WAAW;AAC9C,QAAM,mBAAmB,YAAY,kBAAkB;AAEvD,MAAI,CAAC,gBAAgB;AACnB,UAAM,IAAI,4BAA4B,WAAW;AAAA,EACnD;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAI,4BAA4B,kBAAkB;AAAA,EAC1D;AAEA,QAAM,EAAE,kBAAkB,eAAA,IAAmB;AAAA,IAC3C,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,wBAAwB,OAAO,KAAK,MAAM;AAChD,QAAM,EAAE,UAAU,WAAA,IAAe;AAAA,IAC/B,WAAW;AAAA,IACX,WAAW;AAAA,IACX;AAAA,IACA;AAAA,EAAA;AAIF,QAAM,mBAAmB,kBAAkB,QAAQ;AACnD,QAAM,qBAAqB,kBAAkB,UAAU;AAGvD,MAAI,eAAe,SAAS;AAAA,IAC1B,IAAI,CAAC,CAAC,YAAY,aAAa,MAAM;AAEnC,YAAM,UAAU,iBAAiB,aAAa;AAG9C,aAAO,CAAC,SAAS,CAAC,YAAY,aAAa,CAAC;AAAA,IAI9C,CAAC;AAAA,EAAA;AAIH,MAAI,iBAAiB,YAAY;AAAA,IAC/B,IAAI,CAAC,CAAC,YAAY,GAAG,MAAM;AAEzB,YAAM,gBAA+B,EAAE,CAAC,gBAAgB,GAAG,IAAA;AAG3D,YAAM,YAAY,mBAAmB,aAAa;AAGlD,aAAO,CAAC,WAAW,CAAC,YAAY,aAAa,CAAC;AAAA,IAIhD,CAAC;AAAA,EAAA;AAIH,MAAI,CAAC,CAAC,SAAS,QAAQ,SAAS,MAAM,EAAE,SAAS,WAAW,IAAI,GAAG;AACjE,UAAM,IAAI,yBAAyB,WAAW,IAAI;AAAA,EACpD;AAEA,MAAI,kBAAkB;AAKpB,UAAM,WACJ,qBAAqB,SAAS,WAAW,OAAO,SAAS;AAC3D,UAAM,kBACJ,SAAS,SAAS,eACjB,SAAS,MAAM,SAAS,SAAS,MAAM;AAE1C,QAAI,CAAC,iBAAiB;AAUpB,sBAAgB,IAAI,eAAe,EAAE;AAErC,YAAM,iBACJ,qBAAqB,SAAS,eAAe;AAE/C,UAAI;AAEJ,YAAM,yBACJ,qBAAqB,SAChB,aACA;AAEP,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,sBAAsB,gBAAgB;AAE5C,YAAM,YAAY,gBAAgB,KAAK,CAAC;AACxC,UAAI,WAAW;AACb;AAAA,UACE;AAAA,UACA,gBAAgB;AAAA,UAChB;AAAA,QAAA;AAAA,MAEJ;AAEA,UAAI,cAAc;AAElB,YAAM,4BAEF,eAAe;AAAA,QACjB,IAAI,CAAC,SAAS;AACZ,cAAI,aAAa;AACf;AAAA,UACF;AAMA,4BAAU;AAAA,YACR,oBAAoB;AAAA,YACpB,gBAAgB;AAAA,UAAA;AAOlB,gBAAM,sBAAsB,UAAU,eAAe,EAAE;AACvD,cAAI,CAAC,qBAAqB;AACxB,kBAAM,IAAI;AAAA,cACR;AAAA,YAAA;AAAA,UAEJ;AAEA,gBAAM,EAAE,UAAU,iBAAA,IAAqB;AAEvC,cAAI,SAAS,MAAM,SAAS,IAAI,GAAG;AAGjC,kBAAM,WAAW,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,OAAO;AAC7D,kBAAM,eAAe,MAAM,OAAO,MAAM,QAAQ;AAEhD,qBAAS,YAAY;AAAA,UACvB,OAAO;AAGL,0BAAc;AACd,6BAAA;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MAAA;AAGH,UAAI,qBAAqB,QAAQ;AAC/B,uBAAe;AAAA,MACjB,OAAO;AACL,yBAAiB;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,aAAa;AAAA,IAClBA,KAAa,gBAAgB,WAAW,IAAgB;AAAA,IACxD,YAAA;AAAA,IACA,mBAAmB,WAAW,IAAI;AAAA,EAAA;AAEtC;AAMA,SAAS,uBACP,MACA,OACA,0BACA,kBAC4D;AAE5D,QAAM,wBAAwB,yBAAyB;AAAA,IACrD,CAAC,UAAU,UAAU;AAAA,EAAA;AAGvB,QAAM,iBAAiB,4BAA4B,IAAI;AACvD,QAAM,kBAAkB,4BAA4B,KAAK;AAGzD,MACE,kBACA,sBAAsB,SAAS,cAAc,KAC7C,oBAAoB,kBACpB;AACA,WAAO,EAAE,UAAU,MAAM,YAAY,MAAA;AAAA,EACvC;AAGA,MACE,mBAAmB,oBACnB,mBACA,sBAAsB,SAAS,eAAe,GAC9C;AACA,WAAO,EAAE,UAAU,OAAO,YAAY,KAAA;AAAA,EACxC;AAGA,MAAI,CAAC,kBAAkB,CAAC,iBAAiB;AAEvC,UAAM,IAAI,uCAAA;AAAA,EACZ;AAGA,MAAI,mBAAmB,iBAAiB;AACtC,UAAM,IAAI,mCAAmC,cAAc;AAAA,EAC7D;AAKA,MAAI,CAAC,sBAAsB,SAAS,cAAc,GAAG;AACnD,UAAM,IAAI,mCAAmC,cAAc;AAAA,EAC7D;AAGA,MAAI,oBAAoB,kBAAkB;AACxC,UAAM,IAAI,oCAAoC,gBAAgB;AAAA,EAChE;AAGA,QAAM,IAAI,qBAAA;AACZ;AAKA,SAAS,4BAA4B,MAAsC;AACzE,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK;AAEH,aAAO,KAAK,KAAK,CAAC,KAAK;AAAA,IACzB,KAAK,QAAQ;AAEX,YAAM,mCAAmB,IAAA;AACzB,iBAAW,OAAO,KAAK,MAAM;AAC3B,cAAM,QAAQ,4BAA4B,GAAG;AAC7C,YAAI,OAAO;AACT,uBAAa,IAAI,KAAK;AAAA,QACxB;AAAA,MACF;AAEA,aAAO,aAAa,SAAS,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC,IAAK;AAAA,IAClE;AAAA,IACA;AAEE,aAAO;AAAA,EAAA;AAEb;AAKA,SAAS,kBACP,MACA,WACA,aACA,WACA,iBACA,+BACA,OACA,cAC6D;AAC7D,UAAQ,KAAK,MAAA;AAAA,IACX,KAAK,iBAAiB;AACpB,YAAM,QAAQ,UAAU,KAAK,WAAW,EAAE;AAC1C,UAAI,CAAC,OAAO;AACV,cAAM,IAAI,6BAA6B,KAAK,WAAW,EAAE;AAAA,MAC3D;AACA,aAAO,EAAE,OAAO,KAAK,OAAO,OAAO,cAAc,KAAK,WAAW,GAAA;AAAA,IACnE;AAAA,IACA,KAAK,YAAY;AAEf,YAAM,gBAAgB,aAAa,IAAI,KAAK,KAAK,KAAK,KAAK;AAG3D,YAAM,iBAAiB;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAIF,YAAM,gBAAgB,eAAe;AAIrC,YAAM,iBAAiB,cAAc;AAAA,QACnC,IAAI,CAAC,SAAc;AACjB,gBAAM,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,IAAI;AACtC,iBAAO,CAAC,KAAK,KAAK;AAAA,QACpB,CAAC;AAAA,MAAA;AAGH,aAAO;AAAA,QACL,OAAO,KAAK;AAAA,QACZ,OAAO;AAAA,QACP,cAAc,eAAe;AAAA,MAAA;AAAA,IAEjC;AAAA,IACA;AACE,YAAM,IAAI,+BAAgC,KAAa,IAAI;AAAA,EAAA;AAEjE;AAKA,SAAS,mBAAmB,UAAkB;AAC5C,SAAO,SACL,UAS0B;AAC1B,WAAO,SAAS;AAAA;AAAA,MAEd,OAAO,CAAC,WAAW;AACjB,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,oBAAoB,6BAAO;AACjC,cAAM,sBAAsB,iCAAS;AAGrC,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,EAAE,qBAAqB;AAAA,QACjC;AAEA,YAAI,aAAa,QAAQ;AACvB,iBAAO,CAAC,CAAC;AAAA,QACX;AAEA,YAAI,aAAa,SAAS;AACxB,iBAAO,CAAC,CAAC;AAAA,QACX;AAGA,eAAO;AAAA,MACT,CAAC;AAAA,MACD,IAAI,CAAC,WAAW;AACd,cAAM,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI;AAC/B,cAAM,UAAU,6BAAO;AACvB,cAAM,oBAAoB,6BAAO;AACjC,cAAM,YAAY,iCAAS;AAC3B,cAAM,sBAAsB,iCAAS;AAGrC,cAAM,sBAAqC,CAAA;AAG3C,YAAI,mBAAmB;AACrB,iBAAO,OAAO,qBAAqB,iBAAiB;AAAA,QACtD;AAGA,YAAI,qBAAqB;AACvB,iBAAO,OAAO,qBAAqB,mBAAmB;AAAA,QACxD;AAGA,cAAM,YAAY,IAAI,OAAO,IAAI,SAAS;AAE1C,eAAO,CAAC,WAAW,mBAAmB;AAAA,MACxC,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAYA,SAAS,4BACP,UACA,gBACA,iBAG6D;AAC7D,MAAI,eAAe,OAAO,gBAAgB,IAAI;AAM5C,WAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EACxD;AAEA,UAAQ,UAAA;AAAA,IACN,KAAK;AACH,aAAO,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA;AAAA,IACrD,KAAK;AACH,aAAO,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACvD,KAAK;AAGH,aAAO,eAAe,OAAO,gBAAgB,OACzC,EAAE,kBAAkB,QAAQ,gBAAgB,gBAAA,IAC5C,EAAE,kBAAkB,UAAU,gBAAgB,eAAA;AAAA,IACpD;AACE,aAAO,EAAE,kBAAkB,QAAW,gBAAgB,OAAA;AAAA,EAAU;AAEtE;"}

package/dist/esm/query/live/collection-subscriber.js

@@ -81,14 +81,18 @@ class CollectionSubscriber {
     );
   }
   loadKeys(keys, filterFn) {
+    const changes = [];
     for (const key of keys) {
       if (this.sentKeys.has(key)) continue;
       const value = this.collection.get(key);
       if (value !== void 0 && filterFn(value)) {
         this.sentKeys.add(key);
-        this.sendChangesToPipeline([{ type: `insert`, key, value }]);
+        changes.push({ type: `insert`, key, value });
       }
     }
+    if (changes.length > 0) {
+      this.sendChangesToPipeline(changes);
+    }
   }
   subscribeToAllChanges(whereExpression) {
     const sendChangesToPipeline = this.sendChangesToPipeline.bind(this);

package/dist/esm/query/live/collection-subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { createFilterFunctionFromExpression } from \"../../change-events.js\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the keys we've sent (needed for join and orderBy optimizations)\n  private sentKeys = new Set<string | number>()\n\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe() {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let unsubscribe: () => void\n    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {\n      unsubscribe = this.subscribeToMatchingChanges(whereExpression)\n    } else if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      unsubscribe = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      unsubscribe = this.subscribeToAllChanges(whereExpression)\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  // Wraps the sendChangesToPipeline function\n  // in order to turn `update`s into `insert`s\n  // for keys that have not been sent to the pipeline yet\n  // and filter out deletes for keys that have not been sent\n  private sendVisibleChangesToPipeline = (\n    changes: Array<ChangeMessage<any, string | number>>,\n    loadedInitialState: boolean\n  ) => {\n    if (loadedInitialState) {\n      // There was no index for the join key\n      // so we loaded the initial state\n      // so we can safely assume that the pipeline has seen all keys\n      return this.sendChangesToPipeline(changes)\n    }\n\n    const newChanges = []\n    for (const change of changes) {\n      let newChange = change\n      if (!this.sentKeys.has(change.key)) {\n        if (change.type === `update`) {\n          newChange = { ...change, type: `insert` }\n        } else if (change.type === `delete`) {\n          // filter out deletes for keys that have not been sent\n          continue\n        }\n        this.sentKeys.add(change.key)\n      }\n      newChanges.push(newChange)\n    }\n\n    return this.sendChangesToPipeline(newChanges)\n  }\n\n  private loadKeys(\n    keys: Iterable<string | number>,\n    filterFn: (item: object) => boolean\n  ) {\n    for (const key of keys) {\n      // Only load the key once\n      if (this.sentKeys.has(key)) continue\n\n      const value = this.collection.get(key)\n      if (value !== undefined && filterFn(value)) {\n        this.sentKeys.add(key)\n        this.sendChangesToPipeline([{ type: `insert`, key, value }])\n      }\n    }\n  }\n\n  private subscribeToAllChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this)\n    const unsubscribe = this.collection.subscribeChanges(\n      sendChangesToPipeline,\n      {\n        includeInitialState: true,\n        ...(whereExpression ? { whereExpression } : undefined),\n      }\n    )\n    return unsubscribe\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    // Flag to indicate we have send to whole initial state of the collection\n    // to the pipeline, this is set when there are no indexes that can be used\n    // to filter the changes and so the whole state was requested from the collection\n    let loadedInitialState = false\n\n    // Flag to indicate that we have started sending changes to the pipeline.\n    // This is set to true by either the first call to `loadKeys` or when the\n    // query requests the whole initial state in `loadInitialState`.\n    // Until that point we filter out all changes from subscription to the collection.\n    let sendChanges = false\n\n    const sendVisibleChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      // We filter out changes when sendChanges is false to ensure that we don't send\n      // any changes from the live subscription until the join operator requests either\n      // the initial state or its first key. This is needed otherwise it could receive\n      // changes which are then later subsumed by the initial state (and that would\n      // lead to weird bugs due to the data being received twice).\n      this.sendVisibleChangesToPipeline(\n        sendChanges ? changes : [],\n        loadedInitialState\n      )\n    }\n\n    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {\n      whereExpression,\n    })\n\n    // Create a function that loads keys from the collection\n    // into the query pipeline on demand\n    const filterFn = whereExpression\n      ? createFilterFunctionFromExpression(whereExpression)\n      : () => true\n    const loadKs = (keys: Set<string | number>) => {\n      sendChanges = true\n      return this.loadKeys(keys, filterFn)\n    }\n\n    // Store the functions to load keys and load initial state in the `lazyCollectionsCallbacks` map\n    // This is used by the join operator to dynamically load matching keys from the lazy collection\n    // or to get the full initial state of the collection if there's no index for the join key\n    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {\n      loadKeys: loadKs,\n      loadInitialState: () => {\n        // Make sure we only load the initial state once\n        if (loadedInitialState) return\n        loadedInitialState = true\n        sendChanges = true\n\n        const changes = this.collection.currentStateAsChanges({\n          whereExpression,\n        })\n        this.sendChangesToPipeline(changes)\n      },\n    }\n    return unsubscribe\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    this.loadNextItems(offset + limit)\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n      this.sendChangesToPipeline(\n        filteredChanges,\n        this.loadMoreIfNeeded.bind(this)\n      )\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    return unsubscribe\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded() {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load, just return true\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    let noMoreNextItems = false\n    if (n > 0) {\n      const loadedItems = this.loadNextItems(n)\n      noMoreNextItems = loadedItems === 0\n    }\n\n    // Indicate that we're done loading data if we didn't need to load more data\n    // or there's no more data to load\n    return n === 0 || noMoreNextItems\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this))\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number) {\n    const { valueExtractorForRawRow, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    const nextOrderedKeys = index.take(n, biggestSentValue)\n    const nextInserts: Array<ChangeMessage<any, string | number>> =\n      nextOrderedKeys.map((key) => {\n        return { type: `insert`, key, value: this.collection.get(key) }\n      })\n    this.sendChangesToPipelineWithTracking(nextInserts)\n    return nextInserts.length\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      this.sentKeys.add(change.key)\n\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n  input.sendData(new MultiSet(multiSetArray))\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller than the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;;AAWO,MAAM,qBAGX;AAAA,EAOA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAVV,SAAQ,+BAAe,IAAA;AAGvB,SAAQ,UAAe;AAwFvB,SAAQ,+BAA+B,CACrC,SACA,uBACG;AACH,UAAI,oBAAoB;AAItB,eAAO,KAAK,sBAAsB,OAAO;AAAA,MAC3C;AAEA,YAAM,aAAa,CAAA;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,YAAY;AAChB,YAAI,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;AAClC,cAAI,OAAO,SAAS,UAAU;AAC5B,wBAAY,EAAE,GAAG,QAAQ,MAAM,SAAA;AAAA,UACjC,WAAW,OAAO,SAAS,UAAU;AAEnC;AAAA,UACF;AACA,eAAK,SAAS,IAAI,OAAO,GAAG;AAAA,QAC9B;AACA,mBAAW,KAAK,SAAS;AAAA,MAC3B;AAEA,aAAO,KAAK,sBAAsB,UAAU;AAAA,IAC9C;AAAA,EA3GG;AAAA,EAEH,YAAY;AACV,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,aAAK,mBAAmB,eAAe;AAAA,MACzC,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,WAAK,mBAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QAAI,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY,GAAG;AACvE,oBAAc,KAAK,2BAA2B,eAAe;AAAA,IAC/D,WACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,oBAAc,KAAK,0BAA0B,eAAe;AAAA,IAC9D,OAAO;AACL,oBAAc,KAAK,sBAAsB,eAAe;AAAA,IAC1D;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AAAA,EACrD;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAmCQ,SACN,MACA,UACA;AACA,eAAW,OAAO,MAAM;AAEtB,UAAI,KAAK,SAAS,IAAI,GAAG,EAAG;AAE5B,YAAM,QAAQ,KAAK,WAAW,IAAI,GAAG;AACrC,UAAI,UAAU,UAAa,SAAS,KAAK,GAAG;AAC1C,aAAK,SAAS,IAAI,GAAG;AACrB,aAAK,sBAAsB,CAAC,EAAE,MAAM,UAAU,KAAK,MAAA,CAAO,CAAC;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,sBACN,iBACA;AACA,UAAM,wBAAwB,KAAK,sBAAsB,KAAK,IAAI;AAClE,UAAM,cAAc,KAAK,WAAW;AAAA,MAClC;AAAA,MACA;AAAA,QACE,qBAAqB;AAAA,QACrB,GAAI,kBAAkB,EAAE,oBAAoB;AAAA,MAAA;AAAA,IAC9C;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,2BACN,iBACA;AAIA,QAAI,qBAAqB;AAMzB,QAAI,cAAc;AAElB,UAAM,qBAAqB,CACzB,YACG;AAMH,WAAK;AAAA,QACH,cAAc,UAAU,CAAA;AAAA,QACxB;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAID,UAAM,WAAW,kBACb,mCAAmC,eAAe,IAClD,MAAM;AACV,UAAM,SAAS,CAAC,SAA+B;AAC7C,oBAAc;AACd,aAAO,KAAK,SAAS,MAAM,QAAQ;AAAA,IACrC;AAKA,SAAK,wBAAwB,yBAAyB,KAAK,YAAY,IAAI;AAAA,MACzE,UAAU;AAAA,MACV,kBAAkB,MAAM;AAEtB,YAAI,mBAAoB;AACxB,6BAAqB;AACrB,sBAAc;AAEd,cAAM,UAAU,KAAK,WAAW,sBAAsB;AAAA,UACpD;AAAA,QAAA,CACD;AACD,aAAK,sBAAsB,OAAO;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,WAAA,IACjC,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAIF,SAAK,cAAc,SAAS,KAAK;AAEjC,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AACA,WAAK;AAAA,QACH;AAAA,QACA,KAAK,iBAAiB,KAAK,IAAI;AAAA,MAAA;AAAA,IAEnC;AAIA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,mBAAmB;AACjB,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,kBAAkB;AACtB,QAAI,IAAI,GAAG;AACT,YAAM,cAAc,KAAK,cAAc,CAAC;AACxC,wBAAkB,gBAAgB;AAAA,IACpC;AAIA,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA,EAEQ,kCACN,SACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK,sBAAsB,gBAAgB,KAAK,iBAAiB,KAAK,IAAI,CAAC;AAAA,EAC7E;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW;AAC/B,UAAM,EAAE,yBAAyB,UAC/B,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,UAAM,kBAAkB,MAAM,KAAK,GAAG,gBAAgB;AACtD,UAAM,cACJ,gBAAgB,IAAI,CAAC,QAAQ;AAC3B,aAAO,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,WAAW,IAAI,GAAG,EAAA;AAAA,IAC9D,CAAC;AACH,SAAK,kCAAkC,WAAW;AAClD,WAAO,YAAY;AAAA,EACrB;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,WAAK,SAAS,IAAI,OAAO,GAAG;AAE5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;;AAEpB,QACE,WAAM,SAAN,mBAAY,UAAS,qBACrB,WAAM,KAAK,eAAX,mBAAuB,QAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,YACE,gBAAW,SAAX,mBAAiB,UAAS,qBAC1B,gBAAW,KAAK,eAAhB,mBAA4B,QAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,QAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAC1C,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}
+{"version":3,"file":"collection-subscriber.js","sources":["../../../../src/query/live/collection-subscriber.ts"],"sourcesContent":["import { MultiSet } from \"@tanstack/db-ivm\"\nimport { createFilterFunctionFromExpression } from \"../../change-events.js\"\nimport { convertToBasicExpression } from \"../compiler/expressions.js\"\nimport type { FullSyncState } from \"./types.js\"\nimport type { MultiSetArray, RootStreamBuilder } from \"@tanstack/db-ivm\"\nimport type { Collection } from \"../../collection.js\"\nimport type { ChangeMessage, SyncConfig } from \"../../types.js\"\nimport type { Context, GetResult } from \"../builder/types.js\"\nimport type { BasicExpression } from \"../ir.js\"\nimport type { CollectionConfigBuilder } from \"./collection-config-builder.js\"\n\nexport class CollectionSubscriber<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  // Keep track of the keys we've sent (needed for join and orderBy optimizations)\n  private sentKeys = new Set<string | number>()\n\n  // Keep track of the biggest value we've sent so far (needed for orderBy optimization)\n  private biggest: any = undefined\n\n  constructor(\n    private collectionId: string,\n    private collection: Collection,\n    private config: Parameters<SyncConfig<TResult>[`sync`]>[0],\n    private syncState: FullSyncState,\n    private collectionConfigBuilder: CollectionConfigBuilder<TContext, TResult>\n  ) {}\n\n  subscribe() {\n    const collectionAlias = findCollectionAlias(\n      this.collectionId,\n      this.collectionConfigBuilder.query\n    )\n    const whereClause = this.getWhereClauseFromAlias(collectionAlias)\n\n    if (whereClause) {\n      // Convert WHERE clause to BasicExpression format for collection subscription\n      const whereExpression = convertToBasicExpression(\n        whereClause,\n        collectionAlias!\n      )\n\n      if (whereExpression) {\n        // Use index optimization for this collection\n        this.subscribeToChanges(whereExpression)\n      } else {\n        // This should not happen - if we have a whereClause but can't create whereExpression,\n        // it indicates a bug in our optimization logic\n        throw new Error(\n          `Failed to convert WHERE clause to collection filter for collection '${this.collectionId}'. ` +\n            `This indicates a bug in the query optimization logic.`\n        )\n      }\n    } else {\n      // No WHERE clause for this collection, use regular subscription\n      this.subscribeToChanges()\n    }\n  }\n\n  private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {\n    let unsubscribe: () => void\n    if (this.collectionConfigBuilder.lazyCollections.has(this.collectionId)) {\n      unsubscribe = this.subscribeToMatchingChanges(whereExpression)\n    } else if (\n      Object.hasOwn(\n        this.collectionConfigBuilder.optimizableOrderByCollections,\n        this.collectionId\n      )\n    ) {\n      unsubscribe = this.subscribeToOrderedChanges(whereExpression)\n    } else {\n      unsubscribe = this.subscribeToAllChanges(whereExpression)\n    }\n    this.syncState.unsubscribeCallbacks.add(unsubscribe)\n  }\n\n  private sendChangesToPipeline(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    callback?: () => boolean\n  ) {\n    const input = this.syncState.inputs[this.collectionId]!\n    const sentChanges = sendChangesToInput(\n      input,\n      changes,\n      this.collection.config.getKey\n    )\n\n    // Do not provide the callback that loads more data\n    // if there's no more data to load\n    // otherwise we end up in an infinite loop trying to load more data\n    const dataLoader = sentChanges > 0 ? callback : undefined\n\n    // We need to call `maybeRunGraph` even if there's no data to load\n    // because we need to mark the collection as ready if it's not already\n    // and that's only done in `maybeRunGraph`\n    this.collectionConfigBuilder.maybeRunGraph(\n      this.config,\n      this.syncState,\n      dataLoader\n    )\n  }\n\n  // Wraps the sendChangesToPipeline function\n  // in order to turn `update`s into `insert`s\n  // for keys that have not been sent to the pipeline yet\n  // and filter out deletes for keys that have not been sent\n  private sendVisibleChangesToPipeline = (\n    changes: Array<ChangeMessage<any, string | number>>,\n    loadedInitialState: boolean\n  ) => {\n    if (loadedInitialState) {\n      // There was no index for the join key\n      // so we loaded the initial state\n      // so we can safely assume that the pipeline has seen all keys\n      return this.sendChangesToPipeline(changes)\n    }\n\n    const newChanges = []\n    for (const change of changes) {\n      let newChange = change\n      if (!this.sentKeys.has(change.key)) {\n        if (change.type === `update`) {\n          newChange = { ...change, type: `insert` }\n        } else if (change.type === `delete`) {\n          // filter out deletes for keys that have not been sent\n          continue\n        }\n        this.sentKeys.add(change.key)\n      }\n      newChanges.push(newChange)\n    }\n\n    return this.sendChangesToPipeline(newChanges)\n  }\n\n  private loadKeys(\n    keys: Iterable<string | number>,\n    filterFn: (item: object) => boolean\n  ) {\n    const changes: Array<ChangeMessage<any, string | number>> = []\n    for (const key of keys) {\n      // Only load the key once\n      if (this.sentKeys.has(key)) continue\n\n      const value = this.collection.get(key)\n      if (value !== undefined && filterFn(value)) {\n        this.sentKeys.add(key)\n        changes.push({ type: `insert`, key, value })\n      }\n    }\n    if (changes.length > 0) {\n      this.sendChangesToPipeline(changes)\n    }\n  }\n\n  private subscribeToAllChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const sendChangesToPipeline = this.sendChangesToPipeline.bind(this)\n    const unsubscribe = this.collection.subscribeChanges(\n      sendChangesToPipeline,\n      {\n        includeInitialState: true,\n        ...(whereExpression ? { whereExpression } : undefined),\n      }\n    )\n    return unsubscribe\n  }\n\n  private subscribeToMatchingChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    // Flag to indicate we have send to whole initial state of the collection\n    // to the pipeline, this is set when there are no indexes that can be used\n    // to filter the changes and so the whole state was requested from the collection\n    let loadedInitialState = false\n\n    // Flag to indicate that we have started sending changes to the pipeline.\n    // This is set to true by either the first call to `loadKeys` or when the\n    // query requests the whole initial state in `loadInitialState`.\n    // Until that point we filter out all changes from subscription to the collection.\n    let sendChanges = false\n\n    const sendVisibleChanges = (\n      changes: Array<ChangeMessage<any, string | number>>\n    ) => {\n      // We filter out changes when sendChanges is false to ensure that we don't send\n      // any changes from the live subscription until the join operator requests either\n      // the initial state or its first key. This is needed otherwise it could receive\n      // changes which are then later subsumed by the initial state (and that would\n      // lead to weird bugs due to the data being received twice).\n      this.sendVisibleChangesToPipeline(\n        sendChanges ? changes : [],\n        loadedInitialState\n      )\n    }\n\n    const unsubscribe = this.collection.subscribeChanges(sendVisibleChanges, {\n      whereExpression,\n    })\n\n    // Create a function that loads keys from the collection\n    // into the query pipeline on demand\n    const filterFn = whereExpression\n      ? createFilterFunctionFromExpression(whereExpression)\n      : () => true\n    const loadKs = (keys: Set<string | number>) => {\n      sendChanges = true\n      return this.loadKeys(keys, filterFn)\n    }\n\n    // Store the functions to load keys and load initial state in the `lazyCollectionsCallbacks` map\n    // This is used by the join operator to dynamically load matching keys from the lazy collection\n    // or to get the full initial state of the collection if there's no index for the join key\n    this.collectionConfigBuilder.lazyCollectionsCallbacks[this.collectionId] = {\n      loadKeys: loadKs,\n      loadInitialState: () => {\n        // Make sure we only load the initial state once\n        if (loadedInitialState) return\n        loadedInitialState = true\n        sendChanges = true\n\n        const changes = this.collection.currentStateAsChanges({\n          whereExpression,\n        })\n        this.sendChangesToPipeline(changes)\n      },\n    }\n    return unsubscribe\n  }\n\n  private subscribeToOrderedChanges(\n    whereExpression: BasicExpression<boolean> | undefined\n  ) {\n    const { offset, limit, comparator, dataNeeded } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n\n    // Load the first `offset + limit` values from the index\n    // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[\n    this.loadNextItems(offset + limit)\n\n    const sendChangesInRange = (\n      changes: Iterable<ChangeMessage<any, string | number>>\n    ) => {\n      // Split live updates into a delete of the old value and an insert of the new value\n      // and filter out changes that are bigger than the biggest value we've sent so far\n      // because they can't affect the topK\n      const splittedChanges = splitUpdates(changes)\n      let filteredChanges = splittedChanges\n      if (dataNeeded!() === 0) {\n        // If the topK is full [..., maxSentValue] then we do not need to send changes > maxSentValue\n        // because they can never make it into the topK.\n        // However, if the topK isn't full yet, we need to also send changes > maxSentValue\n        // because they will make it into the topK\n        filteredChanges = filterChangesSmallerOrEqualToMax(\n          splittedChanges,\n          comparator,\n          this.biggest\n        )\n      }\n      this.sendChangesToPipeline(\n        filteredChanges,\n        this.loadMoreIfNeeded.bind(this)\n      )\n    }\n\n    // Subscribe to changes and only send changes that are smaller than the biggest value we've sent so far\n    // values that are bigger don't need to be sent because they can't affect the topK\n    const unsubscribe = this.collection.subscribeChanges(sendChangesInRange, {\n      whereExpression,\n    })\n\n    return unsubscribe\n  }\n\n  // This function is called by maybeRunGraph\n  // after each iteration of the query pipeline\n  // to ensure that the orderBy operator has enough data to work with\n  loadMoreIfNeeded() {\n    const orderByInfo =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]\n\n    if (!orderByInfo) {\n      // This query has no orderBy operator\n      // so there's no data to load, just return true\n      return true\n    }\n\n    const { dataNeeded } = orderByInfo\n\n    if (!dataNeeded) {\n      // This should never happen because the topK operator should always set the size callback\n      // which in turn should lead to the orderBy operator setting the dataNeeded callback\n      throw new Error(\n        `Missing dataNeeded callback for collection ${this.collectionId}`\n      )\n    }\n\n    // `dataNeeded` probes the orderBy operator to see if it needs more data\n    // if it needs more data, it returns the number of items it needs\n    const n = dataNeeded()\n    let noMoreNextItems = false\n    if (n > 0) {\n      const loadedItems = this.loadNextItems(n)\n      noMoreNextItems = loadedItems === 0\n    }\n\n    // Indicate that we're done loading data if we didn't need to load more data\n    // or there's no more data to load\n    return n === 0 || noMoreNextItems\n  }\n\n  private sendChangesToPipelineWithTracking(\n    changes: Iterable<ChangeMessage<any, string | number>>\n  ) {\n    const { comparator } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const trackedChanges = this.trackSentValues(changes, comparator)\n    this.sendChangesToPipeline(trackedChanges, this.loadMoreIfNeeded.bind(this))\n  }\n\n  // Loads the next `n` items from the collection\n  // starting from the biggest item it has sent\n  private loadNextItems(n: number) {\n    const { valueExtractorForRawRow, index } =\n      this.collectionConfigBuilder.optimizableOrderByCollections[\n        this.collectionId\n      ]!\n    const biggestSentRow = this.biggest\n    const biggestSentValue = biggestSentRow\n      ? valueExtractorForRawRow(biggestSentRow)\n      : biggestSentRow\n    // Take the `n` items after the biggest sent value\n    const nextOrderedKeys = index.take(n, biggestSentValue)\n    const nextInserts: Array<ChangeMessage<any, string | number>> =\n      nextOrderedKeys.map((key) => {\n        return { type: `insert`, key, value: this.collection.get(key) }\n      })\n    this.sendChangesToPipelineWithTracking(nextInserts)\n    return nextInserts.length\n  }\n\n  private getWhereClauseFromAlias(\n    collectionAlias: string | undefined\n  ): BasicExpression<boolean> | undefined {\n    const collectionWhereClausesCache =\n      this.collectionConfigBuilder.collectionWhereClausesCache\n    if (collectionAlias && collectionWhereClausesCache) {\n      return collectionWhereClausesCache.get(collectionAlias)\n    }\n    return undefined\n  }\n\n  private *trackSentValues(\n    changes: Iterable<ChangeMessage<any, string | number>>,\n    comparator: (a: any, b: any) => number\n  ) {\n    for (const change of changes) {\n      this.sentKeys.add(change.key)\n\n      if (!this.biggest) {\n        this.biggest = change.value\n      } else if (comparator(this.biggest, change.value) < 0) {\n        this.biggest = change.value\n      }\n\n      yield change\n    }\n  }\n}\n\n/**\n * Finds the alias for a collection ID in the query\n */\nfunction findCollectionAlias(\n  collectionId: string,\n  query: any\n): string | undefined {\n  // Check FROM clause\n  if (\n    query.from?.type === `collectionRef` &&\n    query.from.collection?.id === collectionId\n  ) {\n    return query.from.alias\n  }\n\n  // Check JOIN clauses\n  if (query.join) {\n    for (const joinClause of query.join) {\n      if (\n        joinClause.from?.type === `collectionRef` &&\n        joinClause.from.collection?.id === collectionId\n      ) {\n        return joinClause.from.alias\n      }\n    }\n  }\n\n  return undefined\n}\n\n/**\n * Helper function to send changes to a D2 input stream\n */\nfunction sendChangesToInput(\n  input: RootStreamBuilder<unknown>,\n  changes: Iterable<ChangeMessage>,\n  getKey: (item: ChangeMessage[`value`]) => any\n): number {\n  const multiSetArray: MultiSetArray<unknown> = []\n  for (const change of changes) {\n    const key = getKey(change.value)\n    if (change.type === `insert`) {\n      multiSetArray.push([[key, change.value], 1])\n    } else if (change.type === `update`) {\n      multiSetArray.push([[key, change.previousValue], -1])\n      multiSetArray.push([[key, change.value], 1])\n    } else {\n      // change.type === `delete`\n      multiSetArray.push([[key, change.value], -1])\n    }\n  }\n  input.sendData(new MultiSet(multiSetArray))\n  return multiSetArray.length\n}\n\n/** Splits updates into a delete of the old value and an insert of the new value */\nfunction* splitUpdates<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (change.type === `update`) {\n      yield { type: `delete`, key: change.key, value: change.previousValue! }\n      yield { type: `insert`, key: change.key, value: change.value }\n    } else {\n      yield change\n    }\n  }\n}\n\nfunction* filterChanges<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  f: (change: ChangeMessage<T, TKey>) => boolean\n): Generator<ChangeMessage<T, TKey>> {\n  for (const change of changes) {\n    if (f(change)) {\n      yield change\n    }\n  }\n}\n\n/**\n * Filters changes to only include those that are smaller than the provided max value\n * @param changes - Iterable of changes to filter\n * @param comparator - Comparator function to use for filtering\n * @param maxValue - Range to filter changes within (range boundaries are exclusive)\n * @returns Iterable of changes that fall within the range\n */\nfunction* filterChangesSmallerOrEqualToMax<\n  T extends object = Record<string, unknown>,\n  TKey extends string | number = string | number,\n>(\n  changes: Iterable<ChangeMessage<T, TKey>>,\n  comparator: (a: any, b: any) => number,\n  maxValue: any\n): Generator<ChangeMessage<T, TKey>> {\n  yield* filterChanges(changes, (change) => {\n    return !maxValue || comparator(change.value, maxValue) <= 0\n  })\n}\n"],"names":[],"mappings":";;;AAWO,MAAM,qBAGX;AAAA,EAOA,YACU,cACA,YACA,QACA,WACA,yBACR;AALQ,SAAA,eAAA;AACA,SAAA,aAAA;AACA,SAAA,SAAA;AACA,SAAA,YAAA;AACA,SAAA,0BAAA;AAVV,SAAQ,+BAAe,IAAA;AAGvB,SAAQ,UAAe;AAwFvB,SAAQ,+BAA+B,CACrC,SACA,uBACG;AACH,UAAI,oBAAoB;AAItB,eAAO,KAAK,sBAAsB,OAAO;AAAA,MAC3C;AAEA,YAAM,aAAa,CAAA;AACnB,iBAAW,UAAU,SAAS;AAC5B,YAAI,YAAY;AAChB,YAAI,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,GAAG;AAClC,cAAI,OAAO,SAAS,UAAU;AAC5B,wBAAY,EAAE,GAAG,QAAQ,MAAM,SAAA;AAAA,UACjC,WAAW,OAAO,SAAS,UAAU;AAEnC;AAAA,UACF;AACA,eAAK,SAAS,IAAI,OAAO,GAAG;AAAA,QAC9B;AACA,mBAAW,KAAK,SAAS;AAAA,MAC3B;AAEA,aAAO,KAAK,sBAAsB,UAAU;AAAA,IAC9C;AAAA,EA3GG;AAAA,EAEH,YAAY;AACV,UAAM,kBAAkB;AAAA,MACtB,KAAK;AAAA,MACL,KAAK,wBAAwB;AAAA,IAAA;AAE/B,UAAM,cAAc,KAAK,wBAAwB,eAAe;AAEhE,QAAI,aAAa;AAEf,YAAM,kBAAkB;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAGF,UAAI,iBAAiB;AAEnB,aAAK,mBAAmB,eAAe;AAAA,MACzC,OAAO;AAGL,cAAM,IAAI;AAAA,UACR,uEAAuE,KAAK,YAAY;AAAA,QAAA;AAAA,MAG5F;AAAA,IACF,OAAO;AAEL,WAAK,mBAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEQ,mBAAmB,iBAA4C;AACrE,QAAI;AACJ,QAAI,KAAK,wBAAwB,gBAAgB,IAAI,KAAK,YAAY,GAAG;AACvE,oBAAc,KAAK,2BAA2B,eAAe;AAAA,IAC/D,WACE,OAAO;AAAA,MACL,KAAK,wBAAwB;AAAA,MAC7B,KAAK;AAAA,IAAA,GAEP;AACA,oBAAc,KAAK,0BAA0B,eAAe;AAAA,IAC9D,OAAO;AACL,oBAAc,KAAK,sBAAsB,eAAe;AAAA,IAC1D;AACA,SAAK,UAAU,qBAAqB,IAAI,WAAW;AAAA,EACrD;AAAA,EAEQ,sBACN,SACA,UACA;AACA,UAAM,QAAQ,KAAK,UAAU,OAAO,KAAK,YAAY;AACrD,UAAM,cAAc;AAAA,MAClB;AAAA,MACA;AAAA,MACA,KAAK,WAAW,OAAO;AAAA,IAAA;AAMzB,UAAM,aAAa,cAAc,IAAI,WAAW;AAKhD,SAAK,wBAAwB;AAAA,MAC3B,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,IAAA;AAAA,EAEJ;AAAA,EAmCQ,SACN,MACA,UACA;AACA,UAAM,UAAsD,CAAA;AAC5D,eAAW,OAAO,MAAM;AAEtB,UAAI,KAAK,SAAS,IAAI,GAAG,EAAG;AAE5B,YAAM,QAAQ,KAAK,WAAW,IAAI,GAAG;AACrC,UAAI,UAAU,UAAa,SAAS,KAAK,GAAG;AAC1C,aAAK,SAAS,IAAI,GAAG;AACrB,gBAAQ,KAAK,EAAE,MAAM,UAAU,KAAK,OAAO;AAAA,MAC7C;AAAA,IACF;AACA,QAAI,QAAQ,SAAS,GAAG;AACtB,WAAK,sBAAsB,OAAO;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,sBACN,iBACA;AACA,UAAM,wBAAwB,KAAK,sBAAsB,KAAK,IAAI;AAClE,UAAM,cAAc,KAAK,WAAW;AAAA,MAClC;AAAA,MACA;AAAA,QACE,qBAAqB;AAAA,QACrB,GAAI,kBAAkB,EAAE,oBAAoB;AAAA,MAAA;AAAA,IAC9C;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,2BACN,iBACA;AAIA,QAAI,qBAAqB;AAMzB,QAAI,cAAc;AAElB,UAAM,qBAAqB,CACzB,YACG;AAMH,WAAK;AAAA,QACH,cAAc,UAAU,CAAA;AAAA,QACxB;AAAA,MAAA;AAAA,IAEJ;AAEA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAID,UAAM,WAAW,kBACb,mCAAmC,eAAe,IAClD,MAAM;AACV,UAAM,SAAS,CAAC,SAA+B;AAC7C,oBAAc;AACd,aAAO,KAAK,SAAS,MAAM,QAAQ;AAAA,IACrC;AAKA,SAAK,wBAAwB,yBAAyB,KAAK,YAAY,IAAI;AAAA,MACzE,UAAU;AAAA,MACV,kBAAkB,MAAM;AAEtB,YAAI,mBAAoB;AACxB,6BAAqB;AACrB,sBAAc;AAEd,cAAM,UAAU,KAAK,WAAW,sBAAsB;AAAA,UACpD;AAAA,QAAA,CACD;AACD,aAAK,sBAAsB,OAAO;AAAA,MACpC;AAAA,IAAA;AAEF,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,iBACA;AACA,UAAM,EAAE,QAAQ,OAAO,YAAY,WAAA,IACjC,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAIF,SAAK,cAAc,SAAS,KAAK;AAEjC,UAAM,qBAAqB,CACzB,YACG;AAIH,YAAM,kBAAkB,aAAa,OAAO;AAC5C,UAAI,kBAAkB;AACtB,UAAI,WAAA,MAAkB,GAAG;AAKvB,0BAAkB;AAAA,UAChB;AAAA,UACA;AAAA,UACA,KAAK;AAAA,QAAA;AAAA,MAET;AACA,WAAK;AAAA,QACH;AAAA,QACA,KAAK,iBAAiB,KAAK,IAAI;AAAA,MAAA;AAAA,IAEnC;AAIA,UAAM,cAAc,KAAK,WAAW,iBAAiB,oBAAoB;AAAA,MACvE;AAAA,IAAA,CACD;AAED,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,mBAAmB;AACjB,UAAM,cACJ,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AAEF,QAAI,CAAC,aAAa;AAGhB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,eAAe;AAEvB,QAAI,CAAC,YAAY;AAGf,YAAM,IAAI;AAAA,QACR,8CAA8C,KAAK,YAAY;AAAA,MAAA;AAAA,IAEnE;AAIA,UAAM,IAAI,WAAA;AACV,QAAI,kBAAkB;AACtB,QAAI,IAAI,GAAG;AACT,YAAM,cAAc,KAAK,cAAc,CAAC;AACxC,wBAAkB,gBAAgB;AAAA,IACpC;AAIA,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA,EAEQ,kCACN,SACA;AACA,UAAM,EAAE,WAAA,IACN,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK,gBAAgB,SAAS,UAAU;AAC/D,SAAK,sBAAsB,gBAAgB,KAAK,iBAAiB,KAAK,IAAI,CAAC;AAAA,EAC7E;AAAA;AAAA;AAAA,EAIQ,cAAc,GAAW;AAC/B,UAAM,EAAE,yBAAyB,UAC/B,KAAK,wBAAwB,8BAC3B,KAAK,YACP;AACF,UAAM,iBAAiB,KAAK;AAC5B,UAAM,mBAAmB,iBACrB,wBAAwB,cAAc,IACtC;AAEJ,UAAM,kBAAkB,MAAM,KAAK,GAAG,gBAAgB;AACtD,UAAM,cACJ,gBAAgB,IAAI,CAAC,QAAQ;AAC3B,aAAO,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,WAAW,IAAI,GAAG,EAAA;AAAA,IAC9D,CAAC;AACH,SAAK,kCAAkC,WAAW;AAClD,WAAO,YAAY;AAAA,EACrB;AAAA,EAEQ,wBACN,iBACsC;AACtC,UAAM,8BACJ,KAAK,wBAAwB;AAC/B,QAAI,mBAAmB,6BAA6B;AAClD,aAAO,4BAA4B,IAAI,eAAe;AAAA,IACxD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,CAAS,gBACP,SACA,YACA;AACA,eAAW,UAAU,SAAS;AAC5B,WAAK,SAAS,IAAI,OAAO,GAAG;AAE5B,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,UAAU,OAAO;AAAA,MACxB,WAAW,WAAW,KAAK,SAAS,OAAO,KAAK,IAAI,GAAG;AACrD,aAAK,UAAU,OAAO;AAAA,MACxB;AAEA,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAKA,SAAS,oBACP,cACA,OACoB;;AAEpB,QACE,WAAM,SAAN,mBAAY,UAAS,qBACrB,WAAM,KAAK,eAAX,mBAAuB,QAAO,cAC9B;AACA,WAAO,MAAM,KAAK;AAAA,EACpB;AAGA,MAAI,MAAM,MAAM;AACd,eAAW,cAAc,MAAM,MAAM;AACnC,YACE,gBAAW,SAAX,mBAAiB,UAAS,qBAC1B,gBAAW,KAAK,eAAhB,mBAA4B,QAAO,cACnC;AACA,eAAO,WAAW,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,mBACP,OACA,SACA,QACQ;AACR,QAAM,gBAAwC,CAAA;AAC9C,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,OAAO,OAAO,KAAK;AAC/B,QAAI,OAAO,SAAS,UAAU;AAC5B,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,WAAW,OAAO,SAAS,UAAU;AACnC,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,aAAa,GAAG,EAAE,CAAC;AACpD,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,CAAC,CAAC;AAAA,IAC7C,OAAO;AAEL,oBAAc,KAAK,CAAC,CAAC,KAAK,OAAO,KAAK,GAAG,EAAE,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,QAAM,SAAS,IAAI,SAAS,aAAa,CAAC;AAC1C,SAAO,cAAc;AACvB;AAGA,UAAU,aAIR,SACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,OAAO,SAAS,UAAU;AAC5B,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,cAAA;AACvD,YAAM,EAAE,MAAM,UAAU,KAAK,OAAO,KAAK,OAAO,OAAO,MAAA;AAAA,IACzD,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEA,UAAU,cAIR,SACA,GACmC;AACnC,aAAW,UAAU,SAAS;AAC5B,QAAI,EAAE,MAAM,GAAG;AACb,YAAM;AAAA,IACR;AAAA,EACF;AACF;AASA,UAAU,iCAIR,SACA,YACA,UACmC;AACnC,SAAO,cAAc,SAAS,CAAC,WAAW;AACxC,WAAO,CAAC,YAAY,WAAW,OAAO,OAAO,QAAQ,KAAK;AAAA,EAC5D,CAAC;AACH;"}

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@tanstack/db",
   "description": "A reactive client store for building super fast apps on sync",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "dependencies": {
     "@standard-schema/spec": "^1.0.0",
-    "@tanstack/db-ivm": "0.1.5"
+    "@tanstack/db-ivm": "0.1.6"
   },
   "devDependencies": {
     "@vitest/coverage-istanbul": "^3.2.4",

package/src/query/builder/types.ts

@@ -500,20 +500,20 @@ export type Ref<T = any> = {
   [K in keyof T]: IsNonExactOptional<T[K]> extends true
     ? IsNonExactNullable<T[K]> extends true
       ? // Both optional and nullable
-        NonNullable<T[K]> extends Record<string, any>
+        IsPlainObject<NonNullable<T[K]>> extends true
         ? Ref<NonNullable<T[K]>> | undefined
         : RefLeaf<NonNullable<T[K]>> | undefined
       : // Optional only
-        NonUndefined<T[K]> extends Record<string, any>
+        IsPlainObject<NonUndefined<T[K]>> extends true
         ? Ref<NonUndefined<T[K]>> | undefined
         : RefLeaf<NonUndefined<T[K]>> | undefined
     : IsNonExactNullable<T[K]> extends true
       ? // Nullable only
-        NonNull<T[K]> extends Record<string, any>
+        IsPlainObject<NonNull<T[K]>> extends true
         ? Ref<NonNull<T[K]>> | null
         : RefLeaf<NonNull<T[K]>> | null
       : // Required
-        T[K] extends Record<string, any>
+        IsPlainObject<T[K]> extends true
         ? Ref<T[K]>
         : RefLeaf<T[K]>
 } & RefLeaf<T>
@@ -825,3 +825,48 @@ export type WithResult<TContext extends Context, TResult> = Prettify<
 export type Prettify<T> = {
   [K in keyof T]: T[K]
 } & {}
+
+/**
+ * IsPlainObject - Utility type to check if T is a plain object
+ */
+type IsPlainObject<T> = T extends unknown
+  ? T extends object
+    ? T extends ReadonlyArray<any>
+      ? false
+      : T extends JsBuiltIns
+        ? false
+        : true
+    : false
+  : false
+
+/**
+ * JsBuiltIns - List of JavaScript built-ins
+ */
+type JsBuiltIns =
+  | ArrayBuffer
+  | ArrayBufferLike
+  | AsyncGenerator<any, any, any>
+  | BigInt64Array
+  | BigUint64Array
+  | DataView
+  | Date
+  | Error
+  | Float32Array
+  | Float64Array
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+  | Function
+  | Generator<any, any, any>
+  | Int16Array
+  | Int32Array
+  | Int8Array
+  | Map<any, any>
+  | Promise<any>
+  | RegExp
+  | Set<any>
+  | string
+  | Uint16Array
+  | Uint32Array
+  | Uint8Array
+  | Uint8ClampedArray
+  | WeakMap<any, any>
+  | WeakSet<any>

package/src/query/compiler/joins.ts

@@ -243,7 +243,7 @@ function processJoin(
       const activePipelineWithLoading: IStreamBuilder<
         [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]
       > = activePipeline.pipe(
-        tap(([joinKey, _]) => {
+        tap((data) => {
           if (deoptimized) {
             return
           }
@@ -270,10 +270,11 @@ function processJoin(
 
           const { loadKeys, loadInitialState } = collectionCallbacks
 
-          if (index && index.supports(`eq`)) {
+          if (index && index.supports(`in`)) {
             // Use the index to fetch the PKs of the rows in the lazy collection
             // that match this row from the active collection based on the value of the joinKey
-            const matchingKeys = index.lookup(`eq`, joinKey)
+            const joinKeys = data.getInner().map(([[joinKey]]) => joinKey)
+            const matchingKeys = index.lookup(`in`, joinKeys)
             // Inform the lazy collection that those keys need to be loaded
             loadKeys(matchingKeys)
           } else {

package/src/query/live/collection-subscriber.ts

@@ -138,6 +138,7 @@ export class CollectionSubscriber<
     keys: Iterable<string | number>,
     filterFn: (item: object) => boolean
   ) {
+    const changes: Array<ChangeMessage<any, string | number>> = []
     for (const key of keys) {
       // Only load the key once
       if (this.sentKeys.has(key)) continue
@@ -145,9 +146,12 @@ export class CollectionSubscriber<
       const value = this.collection.get(key)
       if (value !== undefined && filterFn(value)) {
         this.sentKeys.add(key)
-        this.sendChangesToPipeline([{ type: `insert`, key, value }])
+        changes.push({ type: `insert`, key, value })
       }
     }
+    if (changes.length > 0) {
+      this.sendChangesToPipeline(changes)
+    }
   }
 
   private subscribeToAllChanges(