@tanstack/db 0.1.11 → 0.2.0

package/src/query/compiler/evaluators.ts

@@ -337,6 +337,36 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any {
       }
     }
 
+    // Null/undefined checking functions
+    case `isUndefined`: {
+      const arg = compiledArgs[0]!
+      return (data) => {
+        const value = arg(data)
+        return value === undefined
+      }
+    }
+    case `isNotUndefined`: {
+      const arg = compiledArgs[0]!
+      return (data) => {
+        const value = arg(data)
+        return value !== undefined
+      }
+    }
+    case `isNull`: {
+      const arg = compiledArgs[0]!
+      return (data) => {
+        const value = arg(data)
+        return value === null
+      }
+    }
+    case `isNotNull`: {
+      const arg = compiledArgs[0]!
+      return (data) => {
+        const value = arg(data)
+        return value !== null
+      }
+    }
+
     default:
       throw new UnknownFunctionError(func.name)
   }

package/src/query/compiler/group-by.ts

@@ -349,12 +349,17 @@ function getAggregateFunction(aggExpr: Aggregate) {
     return typeof value === `number` ? value : value != null ? Number(value) : 0
   }
 
+  // Create a raw value extractor function for the expression to aggregate
+  const rawValueExtractor = ([, namespacedRow]: [string, NamespacedRow]) => {
+    return compiledExpr(namespacedRow)
+  }
+
   // Return the appropriate aggregate function
   switch (aggExpr.name.toLowerCase()) {
     case `sum`:
       return sum(valueExtractor)
     case `count`:
-      return count() // count() doesn't need a value extractor
+      return count(rawValueExtractor)
     case `avg`:
       return avg(valueExtractor)
     case `min`:

package/src/query/compiler/index.ts

@@ -7,12 +7,12 @@ import {
   LimitOffsetRequireOrderByError,
   UnsupportedFromTypeError,
 } from "../../errors.js"
-import { PropRef, getWhereExpression } from "../ir.js"
+import { PropRef, Value as ValClass, getWhereExpression } from "../ir.js"
 import { compileExpression } from "./evaluators.js"
 import { processJoins } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
 import { processOrderBy } from "./order-by.js"
-import { processSelectToResults } from "./select.js"
+import { processSelect } from "./select.js"
 import type { OrderByOptimizationInfo } from "./order-by.js"
 import type {
   BasicExpression,
@@ -173,7 +173,7 @@ export function compileQuery(
       })
     )
   } else if (query.select) {
-    pipeline = processSelectToResults(pipeline, query.select, allInputs)
+    pipeline = processSelect(pipeline, query.select, allInputs)
   } else {
     // If no SELECT clause, create __select_results with the main table data
     pipeline = pipeline.pipe(
@@ -270,7 +270,8 @@ export function compileQuery(
     const resultPipeline = orderedPipeline.pipe(
       map(([key, [row, orderByIndex]]) => {
         // Extract the final results from __select_results and include orderBy index
-        const finalResults = (row as any).__select_results
+        const raw = (row as any).__select_results
+        const finalResults = unwrapValue(raw)
         return [key, [finalResults, orderByIndex]] as [unknown, [any, string]]
       })
     )
@@ -294,7 +295,8 @@ export function compileQuery(
   const resultPipeline: ResultStream = pipeline.pipe(
     map(([key, row]) => {
       // Extract the final results from __select_results and return [key, [results, undefined]]
-      const finalResults = (row as any).__select_results
+      const raw = (row as any).__select_results
+      const finalResults = unwrapValue(raw)
       return [key, [finalResults, undefined]] as [
         unknown,
         [any, string | undefined],
@@ -359,7 +361,9 @@ function processFrom(
       const extractedInput = subQueryInput.pipe(
         map((data: any) => {
           const [key, [value, _orderByIndex]] = data
-          return [key, value] as [unknown, any]
+          // Unwrap Value expressions that might have leaked through as the entire row
+          const unwrapped = unwrapValue(value)
+          return [key, unwrapped] as [unknown, any]
         })
       )
 
@@ -374,6 +378,19 @@ function processFrom(
   }
 }
 
+// Helper to check if a value is a Value expression
+function isValue(raw: any): boolean {
+  return (
+    raw instanceof ValClass ||
+    (raw && typeof raw === `object` && `type` in raw && raw.type === `val`)
+  )
+}
+
+// Helper to unwrap a Value expression or return the value itself
+function unwrapValue(value: any): any {
+  return isValue(value) ? value.value : value
+}
+
 /**
  * Recursively maps optimized subqueries to their original queries for proper caching.
  * This ensures that when we encounter the same QueryRef object in different contexts,

package/src/query/compiler/joins.ts

@@ -7,9 +7,11 @@ import {
 } from "@tanstack/db-ivm"
 import {
   CollectionInputNotFoundError,
+  InvalidJoinCondition,
+  InvalidJoinConditionLeftTableError,
+  InvalidJoinConditionRightTableError,
   InvalidJoinConditionSameTableError,
   InvalidJoinConditionTableMismatchError,
-  InvalidJoinConditionWrongTablesError,
   JoinCollectionNotFoundError,
   UnsupportedJoinSourceTypeError,
   UnsupportedJoinTypeError,
@@ -139,10 +141,11 @@ function processJoin(
   )
 
   // Analyze which table each expression refers to and swap if necessary
+  const availableTableAliases = Object.keys(tables)
   const { mainExpr, joinedExpr } = analyzeJoinExpressions(
     joinClause.left,
     joinClause.right,
-    mainTableAlias,
+    availableTableAliases,
     joinedTableAlias
   )
 
@@ -187,93 +190,106 @@ function processJoin(
   }
 
   if (activeCollection) {
-    // This join can be optimized by having the active collection
-    // dynamically load keys into the lazy collection
-    // based on the value of the joinKey and by looking up
-    // matching rows in the index of the lazy collection
-
-    // Mark the lazy collection as lazy
-    // this Set is passed by the liveQueryCollection to the compiler
-    // such that the liveQueryCollection can check it after compilation
-    // to know which collections are lazy collections
-    lazyCollections.add(lazyCollection.id)
-
-    const activePipeline =
-      activeCollection === `main` ? mainPipeline : joinedPipeline
-
-    let index: BaseIndex<string | number> | undefined
-
-    const lazyCollectionJoinExpr =
-      activeCollection === `main`
-        ? (joinedExpr as PropRef)
-        : (mainExpr as PropRef)
-
-    const activeColl =
-      activeCollection === `main` ? collections[mainTableId]! : lazyCollection
-
-    const followRefResult = followRef(
-      rawQuery,
-      lazyCollectionJoinExpr,
-      activeColl
-    )!
-    const followRefCollection = followRefResult.collection
-
-    const fieldName = followRefResult.path[0]
-    if (fieldName) {
-      ensureIndexForField(fieldName, followRefResult.path, followRefCollection)
-    }
-
-    let deoptimized = false
-
-    const activePipelineWithLoading: IStreamBuilder<
-      [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]
-    > = activePipeline.pipe(
-      tap(([joinKey, _]) => {
-        if (deoptimized) {
-          return
-        }
-
-        // Find the index for the path we join on
-        // we need to find the index inside the map operator
-        // because the indexes are only available after the initial sync
-        // so we can't fetch it during compilation
-        index ??= findIndexForField(
-          followRefCollection.indexes,
-          followRefResult.path
+    // If the lazy collection comes from a subquery that has a limit and/or an offset clause
+    // then we need to deoptimize the join because we don't know which rows are in the result set
+    // since we simply lookup matching keys in the index but the index contains all rows
+    // (not just the ones that pass the limit and offset clauses)
+    const lazyFrom =
+      activeCollection === `main` ? joinClause.from : rawQuery.from
+    const limitedSubquery =
+      lazyFrom.type === `queryRef` &&
+      (lazyFrom.query.limit || lazyFrom.query.offset)
+
+    if (!limitedSubquery) {
+      // This join can be optimized by having the active collection
+      // dynamically load keys into the lazy collection
+      // based on the value of the joinKey and by looking up
+      // matching rows in the index of the lazy collection
+
+      // Mark the lazy collection as lazy
+      // this Set is passed by the liveQueryCollection to the compiler
+      // such that the liveQueryCollection can check it after compilation
+      // to know which collections are lazy collections
+      lazyCollections.add(lazyCollection.id)
+
+      const activePipeline =
+        activeCollection === `main` ? mainPipeline : joinedPipeline
+
+      let index: BaseIndex<string | number> | undefined
+
+      const lazyCollectionJoinExpr =
+        activeCollection === `main`
+          ? (joinedExpr as PropRef)
+          : (mainExpr as PropRef)
+
+      const followRefResult = followRef(
+        rawQuery,
+        lazyCollectionJoinExpr,
+        lazyCollection
+      )!
+      const followRefCollection = followRefResult.collection
+
+      const fieldName = followRefResult.path[0]
+      if (fieldName) {
+        ensureIndexForField(
+          fieldName,
+          followRefResult.path,
+          followRefCollection
         )
+      }
 
-        // The `callbacks` object is passed by the liveQueryCollection to the compiler.
-        // It contains a function to lazy load keys for each lazy collection
-        // as well as a function to switch back to a regular collection
-        // (useful when there's no index for available for lazily loading the collection)
-        const collectionCallbacks = callbacks[lazyCollection.id]
-        if (!collectionCallbacks) {
-          throw new Error(
-            `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`
+      let deoptimized = false
+
+      const activePipelineWithLoading: IStreamBuilder<
+        [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]]
+      > = activePipeline.pipe(
+        tap(([joinKey, _]) => {
+          if (deoptimized) {
+            return
+          }
+
+          // Find the index for the path we join on
+          // we need to find the index inside the map operator
+          // because the indexes are only available after the initial sync
+          // so we can't fetch it during compilation
+          index ??= findIndexForField(
+            followRefCollection.indexes,
+            followRefResult.path
           )
-        }
 
-        const { loadKeys, loadInitialState } = collectionCallbacks
-
-        if (index && index.supports(`eq`)) {
-          // Use the index to fetch the PKs of the rows in the lazy collection
-          // that match this row from the active collection based on the value of the joinKey
-          const matchingKeys = index.lookup(`eq`, joinKey)
-          // Inform the lazy collection that those keys need to be loaded
-          loadKeys(matchingKeys)
-        } else {
-          // We can't optimize the join because there is no index for the join key
-          // on the lazy collection, so we load the initial state
-          deoptimized = true
-          loadInitialState()
-        }
-      })
-    )
+          // The `callbacks` object is passed by the liveQueryCollection to the compiler.
+          // It contains a function to lazy load keys for each lazy collection
+          // as well as a function to switch back to a regular collection
+          // (useful when there's no index for available for lazily loading the collection)
+          const collectionCallbacks = callbacks[lazyCollection.id]
+          if (!collectionCallbacks) {
+            throw new Error(
+              `Internal error: callbacks for collection are missing in join pipeline. Make sure the live query collection sets them before running the pipeline.`
+            )
+          }
+
+          const { loadKeys, loadInitialState } = collectionCallbacks
+
+          if (index && index.supports(`eq`)) {
+            // Use the index to fetch the PKs of the rows in the lazy collection
+            // that match this row from the active collection based on the value of the joinKey
+            const matchingKeys = index.lookup(`eq`, joinKey)
+            // Inform the lazy collection that those keys need to be loaded
+            loadKeys(matchingKeys)
+          } else {
+            // We can't optimize the join because there is no index for the join key
+            // on the lazy collection, so we load the initial state
+            deoptimized = true
+            loadInitialState()
+          }
+        })
+      )
 
-    if (activeCollection === `main`) {
-      mainPipeline = activePipelineWithLoading
-    } else {
-      joinedPipeline = activePipelineWithLoading
+      if (activeCollection === `main`) {
+        mainPipeline = activePipelineWithLoading
+      } else {
+        joinedPipeline = activePipelineWithLoading
+      }
     }
   }
 
@@ -286,53 +302,65 @@ function processJoin(
 
 /**
  * Analyzes join expressions to determine which refers to which table
- * and returns them in the correct order (main table expression first, joined table expression second)
+ * and returns them in the correct order (available table expression first, joined table expression second)
  */
 function analyzeJoinExpressions(
   left: BasicExpression,
   right: BasicExpression,
-  mainTableAlias: string,
+  allAvailableTableAliases: Array<string>,
   joinedTableAlias: string
 ): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {
+  // Filter out the joined table alias from the available table aliases
+  const availableTableAliases = allAvailableTableAliases.filter(
+    (alias) => alias !== joinedTableAlias
+  )
+
   const leftTableAlias = getTableAliasFromExpression(left)
   const rightTableAlias = getTableAliasFromExpression(right)
 
-  // If left expression refers to main table and right refers to joined table, keep as is
+  // If left expression refers to an available table and right refers to joined table, keep as is
   if (
-    leftTableAlias === mainTableAlias &&
+    leftTableAlias &&
+    availableTableAliases.includes(leftTableAlias) &&
     rightTableAlias === joinedTableAlias
   ) {
     return { mainExpr: left, joinedExpr: right }
   }
 
-  // If left expression refers to joined table and right refers to main table, swap them
+  // If left expression refers to joined table and right refers to an available table, swap them
   if (
     leftTableAlias === joinedTableAlias &&
-    rightTableAlias === mainTableAlias
+    rightTableAlias &&
+    availableTableAliases.includes(rightTableAlias)
   ) {
     return { mainExpr: right, joinedExpr: left }
   }
 
+  // If one expression doesn't refer to any table, this is an invalid join
+  if (!leftTableAlias || !rightTableAlias) {
+    // For backward compatibility, use the first available table alias in error message
+    throw new InvalidJoinConditionTableMismatchError()
+  }
+
   // If both expressions refer to the same alias, this is an invalid join
   if (leftTableAlias === rightTableAlias) {
-    throw new InvalidJoinConditionSameTableError(leftTableAlias || `unknown`)
+    throw new InvalidJoinConditionSameTableError(leftTableAlias)
   }
 
-  // If one expression doesn't refer to either table, this is an invalid join
-  if (!leftTableAlias || !rightTableAlias) {
-    throw new InvalidJoinConditionTableMismatchError(
-      mainTableAlias,
-      joinedTableAlias
-    )
+  // Left side must refer to an available table
+  // This cannot happen with the query builder as there is no way to build a ref
+  // to an unavailable table, but just in case, but could happen with the IR
+  if (!availableTableAliases.includes(leftTableAlias)) {
+    throw new InvalidJoinConditionLeftTableError(leftTableAlias)
   }
 
-  // If expressions refer to tables not involved in this join, this is an invalid join
-  throw new InvalidJoinConditionWrongTablesError(
-    leftTableAlias,
-    rightTableAlias,
-    mainTableAlias,
-    joinedTableAlias
-  )
+  // Right side must refer to the joined table
+  if (rightTableAlias !== joinedTableAlias) {
+    throw new InvalidJoinConditionRightTableError(joinedTableAlias)
+  }
+
+  // This should not be reachable given the logic above, but just in case
+  throw new InvalidJoinCondition()
 }
 
 /**