@tanstack/db 0.0.27 → 0.0.30

package/src/query/compiler/group-by.ts

@@ -1,5 +1,11 @@
 import { filter, groupBy, groupByOperators, map } from "@electric-sql/d2mini"
 import { Func, PropRef } from "../ir.js"
+import {
+  AggregateFunctionNotInSelectError,
+  NonAggregateExpressionNotInGroupByError,
+  UnknownHavingExpressionTypeError,
+  UnsupportedAggregateFunctionError,
+} from "../../errors.js"
 import { compileExpression } from "./evaluators.js"
 import type {
   Aggregate,
@@ -48,9 +54,7 @@ function validateAndCreateMapping(
     )
 
     if (groupIndex === -1) {
-      throw new Error(
-        `Non-aggregate expression '${alias}' in SELECT must also appear in GROUP BY clause`
-      )
+      throw new NonAggregateExpressionNotInGroupByError(alias)
     }
 
     // Cache the mapping
@@ -354,7 +358,7 @@ function getAggregateFunction(aggExpr: Aggregate) {
     case `max`:
       return max(valueExtractor)
     default:
-      throw new Error(`Unsupported aggregate function: ${aggExpr.name}`)
+      throw new UnsupportedAggregateFunctionError(aggExpr.name)
   }
 }
 
@@ -376,9 +380,7 @@ function transformHavingClause(
         }
       }
       // If no matching aggregate found in SELECT, throw error
-      throw new Error(
-        `Aggregate function in HAVING clause must also be in SELECT clause: ${aggExpr.name}`
-      )
+      throw new AggregateFunctionNotInSelectError(aggExpr.name)
     }
 
     case `func`: {
@@ -410,9 +412,7 @@ function transformHavingClause(
       return havingExpr as BasicExpression
 
     default:
-      throw new Error(
-        `Unknown expression type in HAVING clause: ${(havingExpr as any).type}`
-      )
+      throw new UnknownHavingExpressionTypeError((havingExpr as any).type)
   }
 }
 

package/src/query/compiler/index.ts

@@ -1,11 +1,23 @@
 import { distinct, filter, map } from "@electric-sql/d2mini"
 import { optimizeQuery } from "../optimizer.js"
+import {
+  CollectionInputNotFoundError,
+  DistinctRequiresSelectError,
+  HavingRequiresGroupByError,
+  LimitOffsetRequireOrderByError,
+  UnsupportedFromTypeError,
+} from "../../errors.js"
 import { compileExpression } from "./evaluators.js"
 import { processJoins } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
 import { processOrderBy } from "./order-by.js"
 import { processSelectToResults } from "./select.js"
-import type { CollectionRef, QueryIR, QueryRef } from "../ir.js"
+import type {
+  BasicExpression,
+  CollectionRef,
+  QueryIR,
+  QueryRef,
+} from "../ir.js"
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,
@@ -13,20 +25,30 @@ import type {
 } from "../../types.js"
 import type { QueryCache, QueryMapping } from "./types.js"
 
+/**
+ * Result of query compilation including both the pipeline and collection-specific WHERE clauses
+ */
+export interface CompilationResult {
+  /** The compiled query pipeline */
+  pipeline: ResultStream
+  /** Map of collection aliases to their WHERE clauses for index optimization */
+  collectionWhereClauses: Map<string, BasicExpression<boolean>>
+}
+
 /**
  * Compiles a query2 IR into a D2 pipeline
  * @param rawQuery The query IR to compile
  * @param inputs Mapping of collection names to input streams
  * @param cache Optional cache for compiled subqueries (used internally for recursion)
  * @param queryMapping Optional mapping from optimized queries to original queries
- * @returns A stream builder representing the compiled query
+ * @returns A CompilationResult with the pipeline and collection WHERE clauses
  */
 export function compileQuery(
   rawQuery: QueryIR,
   inputs: Record<string, KeyedStream>,
   cache: QueryCache = new WeakMap(),
   queryMapping: QueryMapping = new WeakMap()
-): ResultStream {
+): CompilationResult {
   // Check if the original raw query has already been compiled
   const cachedResult = cache.get(rawQuery)
   if (cachedResult) {
@@ -34,7 +56,8 @@ export function compileQuery(
   }
 
   // Optimize the query before compilation
-  const query = optimizeQuery(rawQuery)
+  const { optimizedQuery: query, collectionWhereClauses } =
+    optimizeQuery(rawQuery)
 
   // Create mapping from optimized query to original for caching
   queryMapping.set(query, rawQuery)
@@ -82,11 +105,9 @@ export function compileQuery(
 
   // Process the WHERE clause if it exists
   if (query.where && query.where.length > 0) {
-    // Compile all WHERE expressions
-    const compiledWheres = query.where.map((where) => compileExpression(where))
-
     // Apply each WHERE condition as a filter (they are ANDed together)
-    for (const compiledWhere of compiledWheres) {
+    for (const where of query.where) {
+      const compiledWhere = compileExpression(where)
       pipeline = pipeline.pipe(
         filter(([_key, namespacedRow]) => {
           return compiledWhere(namespacedRow)
@@ -107,7 +128,7 @@ export function compileQuery(
   }
 
   if (query.distinct && !query.fnSelect && !query.select) {
-    throw new Error(`DISTINCT requires a SELECT clause.`)
+    throw new DistinctRequiresSelectError()
   }
 
   // Process the SELECT clause early - always create __select_results
@@ -182,7 +203,7 @@ export function compileQuery(
       : false
 
     if (!hasAggregates) {
-      throw new Error(`HAVING clause requires GROUP BY clause`)
+      throw new HavingRequiresGroupByError()
     }
   }
 
@@ -227,13 +248,16 @@ export function compileQuery(
 
     const result = resultPipeline
     // Cache the result before returning (use original query as key)
-    cache.set(rawQuery, result)
-    return result
+    const compilationResult = {
+      pipeline: result,
+      collectionWhereClauses,
+    }
+    cache.set(rawQuery, compilationResult)
+
+    return compilationResult
   } else if (query.limit !== undefined || query.offset !== undefined) {
     // If there's a limit or offset without orderBy, throw an error
-    throw new Error(
-      `LIMIT and OFFSET require an ORDER BY clause to ensure deterministic results`
-    )
+    throw new LimitOffsetRequireOrderByError()
   }
 
   // Final step: extract the __select_results and return tuple format (no orderBy)
@@ -250,8 +274,13 @@ export function compileQuery(
 
   const result = resultPipeline
   // Cache the result before returning (use original query as key)
-  cache.set(rawQuery, result)
-  return result
+  const compilationResult = {
+    pipeline: result,
+    collectionWhereClauses,
+  }
+  cache.set(rawQuery, compilationResult)
+
+  return compilationResult
 }
 
 /**
@@ -267,9 +296,7 @@ function processFrom(
     case `collectionRef`: {
       const input = allInputs[from.collection.id]
       if (!input) {
-        throw new Error(
-          `Input for collection "${from.collection.id}" not found in inputs map`
-        )
+        throw new CollectionInputNotFoundError(from.collection.id)
       }
       return { alias: from.alias, input }
     }
@@ -278,13 +305,16 @@ function processFrom(
       const originalQuery = queryMapping.get(from.query) || from.query
 
       // Recursively compile the sub-query with cache
-      const subQueryInput = compileQuery(
+      const subQueryResult = compileQuery(
         originalQuery,
         allInputs,
         cache,
         queryMapping
       )
 
+      // Extract the pipeline from the compilation result
+      const subQueryInput = subQueryResult.pipeline
+
       // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)
       // We need to extract just the value for use in parent queries
       const extractedInput = subQueryInput.pipe(
@@ -297,7 +327,7 @@ function processFrom(
       return { alias: from.alias, input: extractedInput }
     }
     default:
-      throw new Error(`Unsupported FROM type: ${(from as any).type}`)
+      throw new UnsupportedFromTypeError((from as any).type)
   }
 }
 

package/src/query/compiler/joins.ts

@@ -4,10 +4,23 @@ import {
   join as joinOperator,
   map,
 } from "@electric-sql/d2mini"
+import {
+  CollectionInputNotFoundError,
+  InvalidJoinConditionSameTableError,
+  InvalidJoinConditionTableMismatchError,
+  InvalidJoinConditionWrongTablesError,
+  UnsupportedJoinSourceTypeError,
+  UnsupportedJoinTypeError,
+} from "../../errors.js"
 import { compileExpression } from "./evaluators.js"
 import { compileQuery } from "./index.js"
 import type { IStreamBuilder, JoinType } from "@electric-sql/d2mini"
-import type { CollectionRef, JoinClause, QueryRef } from "../ir.js"
+import type {
+  BasicExpression,
+  CollectionRef,
+  JoinClause,
+  QueryRef,
+} from "../ir.js"
 import type {
   KeyedStream,
   NamespacedAndKeyedStream,
@@ -75,18 +88,26 @@ function processJoin(
         ? `full`
         : (joinClause.type as JoinType)
 
+  // Analyze which table each expression refers to and swap if necessary
+  const { mainExpr, joinedExpr } = analyzeJoinExpressions(
+    joinClause.left,
+    joinClause.right,
+    mainTableAlias,
+    joinedTableAlias
+  )
+
   // Pre-compile the join expressions
-  const compiledLeftExpr = compileExpression(joinClause.left)
-  const compiledRightExpr = compileExpression(joinClause.right)
+  const compiledMainExpr = compileExpression(mainExpr)
+  const compiledJoinedExpr = compileExpression(joinedExpr)
 
   // Prepare the main pipeline for joining
   const mainPipeline = pipeline.pipe(
     map(([currentKey, namespacedRow]) => {
-      // Extract the join key from the left side of the join condition
-      const leftKey = compiledLeftExpr(namespacedRow)
+      // Extract the join key from the main table expression
+      const mainKey = compiledMainExpr(namespacedRow)
 
       // Return [joinKey, [originalKey, namespacedRow]]
-      return [leftKey, [currentKey, namespacedRow]] as [
+      return [mainKey, [currentKey, namespacedRow]] as [
         unknown,
         [string, typeof namespacedRow],
       ]
@@ -99,11 +120,11 @@ function processJoin(
       // Wrap the row in a namespaced structure
       const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }
 
-      // Extract the join key from the right side of the join condition
-      const rightKey = compiledRightExpr(namespacedRow)
+      // Extract the join key from the joined table expression
+      const joinedKey = compiledJoinedExpr(namespacedRow)
 
       // Return [joinKey, [originalKey, namespacedRow]]
-      return [rightKey, [currentKey, namespacedRow]] as [
+      return [joinedKey, [currentKey, namespacedRow]] as [
         unknown,
         [string, typeof namespacedRow],
       ]
@@ -112,7 +133,7 @@ function processJoin(
 
   // Apply the join operation
   if (![`inner`, `left`, `right`, `full`].includes(joinType)) {
-    throw new Error(`Unsupported join type: ${joinClause.type}`)
+    throw new UnsupportedJoinTypeError(joinClause.type)
   }
   return mainPipeline.pipe(
     joinOperator(joinedPipeline, joinType),
@@ -121,6 +142,83 @@ function processJoin(
   )
 }
 
+/**
+ * Analyzes join expressions to determine which refers to which table
+ * and returns them in the correct order (main table expression first, joined table expression second)
+ */
+function analyzeJoinExpressions(
+  left: BasicExpression,
+  right: BasicExpression,
+  mainTableAlias: string,
+  joinedTableAlias: string
+): { mainExpr: BasicExpression; joinedExpr: BasicExpression } {
+  const leftTableAlias = getTableAliasFromExpression(left)
+  const rightTableAlias = getTableAliasFromExpression(right)
+
+  // If left expression refers to main table and right refers to joined table, keep as is
+  if (
+    leftTableAlias === mainTableAlias &&
+    rightTableAlias === joinedTableAlias
+  ) {
+    return { mainExpr: left, joinedExpr: right }
+  }
+
+  // If left expression refers to joined table and right refers to main table, swap them
+  if (
+    leftTableAlias === joinedTableAlias &&
+    rightTableAlias === mainTableAlias
+  ) {
+    return { mainExpr: right, joinedExpr: left }
+  }
+
+  // If both expressions refer to the same alias, this is an invalid join
+  if (leftTableAlias === rightTableAlias) {
+    throw new InvalidJoinConditionSameTableError(leftTableAlias || `unknown`)
+  }
+
+  // If one expression doesn't refer to either table, this is an invalid join
+  if (!leftTableAlias || !rightTableAlias) {
+    throw new InvalidJoinConditionTableMismatchError(
+      mainTableAlias,
+      joinedTableAlias
+    )
+  }
+
+  // If expressions refer to tables not involved in this join, this is an invalid join
+  throw new InvalidJoinConditionWrongTablesError(
+    leftTableAlias,
+    rightTableAlias,
+    mainTableAlias,
+    joinedTableAlias
+  )
+}
+
+/**
+ * Extracts the table alias from a join expression
+ */
+function getTableAliasFromExpression(expr: BasicExpression): string | null {
+  switch (expr.type) {
+    case `ref`:
+      // PropRef path has the table alias as the first element
+      return expr.path[0] || null
+    case `func`: {
+      // For function expressions, we need to check if all arguments refer to the same table
+      const tableAliases = new Set<string>()
+      for (const arg of expr.args) {
+        const alias = getTableAliasFromExpression(arg)
+        if (alias) {
+          tableAliases.add(alias)
+        }
+      }
+      // If all arguments refer to the same table, return that table alias
+      return tableAliases.size === 1 ? Array.from(tableAliases)[0]! : null
+    }
+    default:
+      // Values (type='val') don't reference any table
+      return null
+  }
+}
+
 /**
  * Processes the join source (collection or sub-query)
  */
@@ -134,9 +232,7 @@ function processJoinSource(
     case `collectionRef`: {
       const input = allInputs[from.collection.id]
       if (!input) {
-        throw new Error(
-          `Input for collection "${from.collection.id}" not found in inputs map`
-        )
+        throw new CollectionInputNotFoundError(from.collection.id)
       }
       return { alias: from.alias, input }
     }
@@ -145,13 +241,16 @@ function processJoinSource(
       const originalQuery = queryMapping.get(from.query) || from.query
 
       // Recursively compile the sub-query with cache
-      const subQueryInput = compileQuery(
+      const subQueryResult = compileQuery(
         originalQuery,
         allInputs,
         cache,
         queryMapping
       )
 
+      // Extract the pipeline from the compilation result
+      const subQueryInput = subQueryResult.pipeline
+
       // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)
       // We need to extract just the value for use in parent queries
       const extractedInput = subQueryInput.pipe(
@@ -164,7 +263,7 @@ function processJoinSource(
       return { alias: from.alias, input: extractedInput as KeyedStream }
     }
     default:
-      throw new Error(`Unsupported join source type: ${(from as any).type}`)
+      throw new UnsupportedJoinSourceTypeError((from as any).type)
   }
 }
 

package/src/query/compiler/order-by.ts

@@ -1,4 +1,5 @@
 import { orderByWithFractionalIndex } from "@electric-sql/d2mini"
+import { ascComparator, descComparator } from "../../utils/comparison.js"
 import { compileExpression } from "./evaluators.js"
 import type { OrderByClause } from "../ir.js"
 import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js"
@@ -51,51 +52,6 @@ export function processOrderBy(
     return null
   }
 
-  // Create comparator functions
-  const ascComparator = (a: any, b: any): number => {
-    // Handle null/undefined
-    if (a == null && b == null) return 0
-    if (a == null) return -1
-    if (b == null) return 1
-
-    // if a and b are both strings, compare them based on locale
-    if (typeof a === `string` && typeof b === `string`) {
-      return a.localeCompare(b)
-    }
-
-    // if a and b are both arrays, compare them element by element
-    if (Array.isArray(a) && Array.isArray(b)) {
-      for (let i = 0; i < Math.min(a.length, b.length); i++) {
-        const result = ascComparator(a[i], b[i])
-        if (result !== 0) {
-          return result
-        }
-      }
-      // All elements are equal up to the minimum length
-      return a.length - b.length
-    }
-
-    // If both are dates, compare them
-    if (a instanceof Date && b instanceof Date) {
-      return a.getTime() - b.getTime()
-    }
-
-    // If at least one of the values is an object, convert to strings
-    const bothObjects = typeof a === `object` && typeof b === `object`
-    const notNull = a !== null && b !== null
-    if (bothObjects && notNull) {
-      return a.toString().localeCompare(b.toString())
-    }
-
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
-  }
-
-  const descComparator = (a: unknown, b: unknown): number => {
-    return ascComparator(b, a)
-  }
-
   // Create a multi-property comparator that respects the order and direction of each property
   const makeComparator = () => {
     return (a: unknown, b: unknown) => {

package/src/query/compiler/types.ts

@@ -1,10 +1,10 @@
 import type { QueryIR } from "../ir.js"
-import type { ResultStream } from "../../types.js"
+import type { CompilationResult } from "./index.js"
 
 /**
  * Cache for compiled subqueries to avoid duplicate compilation
  */
-export type QueryCache = WeakMap<QueryIR, ResultStream>
+export type QueryCache = WeakMap<QueryIR, CompilationResult>
 
 /**
  * Mapping from optimized queries back to their original queries for caching

package/src/query/live-query-collection.ts

@@ -2,6 +2,7 @@ import { D2, MultiSet, output } from "@electric-sql/d2mini"
 import { createCollection } from "../collection.js"
 import { compileQuery } from "./compiler/index.js"
 import { buildQuery, getQueryIR } from "./builder/index.js"
+import { convertToBasicExpression } from "./compiler/expressions.js"
 import type { InitialQueryBuilder, QueryBuilder } from "./builder/index.js"
 import type { Collection } from "../collection.js"
 import type {
@@ -14,6 +15,7 @@ import type {
 } from "../types.js"
 import type { Context, GetResult } from "./builder/types.js"
 import type { MultiSetArray, RootStreamBuilder } from "@electric-sql/d2mini"
+import type { BasicExpression } from "./ir.js"
 
 // Global counter for auto-generated collection IDs
 let liveQueryCollectionCounter = 0
@@ -170,6 +172,9 @@ export function liveQueryCollectionOptions<
   let graphCache: D2 | undefined
   let inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined
   let pipelineCache: ResultStream | undefined
+  let collectionWhereClausesCache:
+    | Map<string, BasicExpression<boolean>>
+    | undefined
 
   const compileBasePipeline = () => {
     graphCache = new D2()
@@ -179,10 +184,12 @@ export function liveQueryCollectionOptions<
         graphCache!.newInput<any>(),
       ])
     )
-    pipelineCache = compileQuery(
-      query,
-      inputsCache as Record<string, KeyedStream>
-    )
+
+    // Compile the query and get both pipeline and collection WHERE clauses
+    ;({
+      pipeline: pipelineCache,
+      collectionWhereClauses: collectionWhereClausesCache,
+    } = compileQuery(query, inputsCache as Record<string, KeyedStream>))
   }
 
   const maybeCompileBasePipeline = () => {
@@ -303,19 +310,54 @@ export function liveQueryCollectionOptions<
       // Unsubscribe callbacks
       const unsubscribeCallbacks = new Set<() => void>()
 
-      // Set up data flow from input collections to the compiled query
+      // Subscribe to all collections, using WHERE clause optimization when available
       Object.entries(collections).forEach(([collectionId, collection]) => {
         const input = inputs[collectionId]!
-
-        // Subscribe to changes
-        const unsubscribe = collection.subscribeChanges(
-          (changes: Array<ChangeMessage>) => {
-            sendChangesToInput(input, changes, collection.config.getKey)
-            maybeRunGraph()
-          },
-          { includeInitialState: true }
-        )
-        unsubscribeCallbacks.add(unsubscribe)
+        const collectionAlias = findCollectionAlias(collectionId, query)
+        const whereClause =
+          collectionAlias && collectionWhereClausesCache
+            ? collectionWhereClausesCache.get(collectionAlias)
+            : undefined
+
+        if (whereClause) {
+          // Convert WHERE clause to BasicExpression format for collection subscription
+          const whereExpression = convertToBasicExpression(
+            whereClause,
+            collectionAlias!
+          )
+
+          if (whereExpression) {
+            // Use index optimization for this collection
+            const subscription = collection.subscribeChanges(
+              (changes) => {
+                sendChangesToInput(input, changes, collection.config.getKey)
+                maybeRunGraph()
+              },
+              {
+                includeInitialState: true,
+                whereExpression: whereExpression,
+              }
+            )
+            unsubscribeCallbacks.add(subscription)
+          } else {
+            // This should not happen - if we have a whereClause but can't create whereExpression,
+            // it indicates a bug in our optimization logic
+            throw new Error(
+              `Failed to convert WHERE clause to collection filter for collection '${collectionId}'. ` +
+                `This indicates a bug in the query optimization logic.`
+            )
+          }
+        } else {
+          // No WHERE clause for this collection, use regular subscription
+          const subscription = collection.subscribeChanges(
+            (changes) => {
+              sendChangesToInput(input, changes, collection.config.getKey)
+              maybeRunGraph()
+            },
+            { includeInitialState: true }
+          )
+          unsubscribeCallbacks.add(subscription)
+        }
       })
 
       // Initial run
@@ -513,3 +555,41 @@ function extractCollectionsFromQuery(
 
   return collections
 }
+
+/**
+ * Converts WHERE expressions from the query IR into a BasicExpression for subscribeChanges
+ *
+ * @param whereExpressions Array of WHERE expressions to convert
+ * @param tableAlias The table alias used in the expressions
+ * @returns A BasicExpression that can be used with the collection's index system
+ */
+
+/**
+ * Finds the alias for a collection ID in the query
+ */
+function findCollectionAlias(
+  collectionId: string,
+  query: any
+): string | undefined {
+  // Check FROM clause
+  if (
+    query.from?.type === `collectionRef` &&
+    query.from.collection?.id === collectionId
+  ) {
+    return query.from.alias
+  }
+
+  // Check JOIN clauses
+  if (query.join) {
+    for (const joinClause of query.join) {
+      if (
+        joinClause.from?.type === `collectionRef` &&
+        joinClause.from.collection?.id === collectionId
+      ) {
+        return joinClause.from.alias
+      }
+    }
+  }
+
+  return undefined
+}