@tanstack/db 0.0.13 → 0.0.15

package/src/query/compiler/joins.ts

@@ -0,0 +1,228 @@
+import {
+  consolidate,
+  filter,
+  join as joinOperator,
+  map,
+} from "@electric-sql/d2mini"
+import { compileExpression } from "./evaluators.js"
+import { compileQuery } from "./index.js"
+import type { IStreamBuilder, JoinType } from "@electric-sql/d2mini"
+import type { CollectionRef, JoinClause, QueryIR, QueryRef } from "../ir.js"
+import type {
+  KeyedStream,
+  NamespacedAndKeyedStream,
+  NamespacedRow,
+  ResultStream,
+} from "../../types.js"
+
+/**
+ * Cache for compiled subqueries to avoid duplicate compilation
+ */
+type QueryCache = WeakMap<QueryIR, ResultStream>
+
+/**
+ * Processes all join clauses in a query
+ */
+export function processJoins(
+  pipeline: NamespacedAndKeyedStream,
+  joinClauses: Array<JoinClause>,
+  tables: Record<string, KeyedStream>,
+  mainTableAlias: string,
+  allInputs: Record<string, KeyedStream>,
+  cache: QueryCache
+): NamespacedAndKeyedStream {
+  let resultPipeline = pipeline
+
+  for (const joinClause of joinClauses) {
+    resultPipeline = processJoin(
+      resultPipeline,
+      joinClause,
+      tables,
+      mainTableAlias,
+      allInputs,
+      cache
+    )
+  }
+
+  return resultPipeline
+}
+
+/**
+ * Processes a single join clause
+ */
+function processJoin(
+  pipeline: NamespacedAndKeyedStream,
+  joinClause: JoinClause,
+  tables: Record<string, KeyedStream>,
+  mainTableAlias: string,
+  allInputs: Record<string, KeyedStream>,
+  cache: QueryCache
+): NamespacedAndKeyedStream {
+  // Get the joined table alias and input stream
+  const { alias: joinedTableAlias, input: joinedInput } = processJoinSource(
+    joinClause.from,
+    allInputs,
+    cache
+  )
+
+  // Add the joined table to the tables map
+  tables[joinedTableAlias] = joinedInput
+
+  // Convert join type to D2 join type
+  const joinType: JoinType =
+    joinClause.type === `cross`
+      ? `inner`
+      : joinClause.type === `outer`
+        ? `full`
+        : (joinClause.type as JoinType)
+
+  // Pre-compile the join expressions
+  const compiledLeftExpr = compileExpression(joinClause.left)
+  const compiledRightExpr = compileExpression(joinClause.right)
+
+  // Prepare the main pipeline for joining
+  const mainPipeline = pipeline.pipe(
+    map(([currentKey, namespacedRow]) => {
+      // Extract the join key from the left side of the join condition
+      const leftKey = compiledLeftExpr(namespacedRow)
+
+      // Return [joinKey, [originalKey, namespacedRow]]
+      return [leftKey, [currentKey, namespacedRow]] as [
+        unknown,
+        [string, typeof namespacedRow],
+      ]
+    })
+  )
+
+  // Prepare the joined pipeline
+  const joinedPipeline = joinedInput.pipe(
+    map(([currentKey, row]) => {
+      // Wrap the row in a namespaced structure
+      const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }
+
+      // Extract the join key from the right side of the join condition
+      const rightKey = compiledRightExpr(namespacedRow)
+
+      // Return [joinKey, [originalKey, namespacedRow]]
+      return [rightKey, [currentKey, namespacedRow]] as [
+        unknown,
+        [string, typeof namespacedRow],
+      ]
+    })
+  )
+
+  // Apply the join operation
+  if (![`inner`, `left`, `right`, `full`].includes(joinType)) {
+    throw new Error(`Unsupported join type: ${joinClause.type}`)
+  }
+  return mainPipeline.pipe(
+    joinOperator(joinedPipeline, joinType),
+    consolidate(),
+    processJoinResults(joinClause.type)
+  )
+}
+
+/**
+ * Processes the join source (collection or sub-query)
+ */
+function processJoinSource(
+  from: CollectionRef | QueryRef,
+  allInputs: Record<string, KeyedStream>,
+  cache: QueryCache
+): { alias: string; input: KeyedStream } {
+  switch (from.type) {
+    case `collectionRef`: {
+      const input = allInputs[from.collection.id]
+      if (!input) {
+        throw new Error(
+          `Input for collection "${from.collection.id}" not found in inputs map`
+        )
+      }
+      return { alias: from.alias, input }
+    }
+    case `queryRef`: {
+      // Recursively compile the sub-query with cache
+      const subQueryInput = compileQuery(from.query, allInputs, cache)
+
+      // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY)
+      // We need to extract just the value for use in parent queries
+      const extractedInput = subQueryInput.pipe(
+        map((data: any) => {
+          const [key, [value, _orderByIndex]] = data
+          return [key, value] as [unknown, any]
+        })
+      )
+
+      return { alias: from.alias, input: extractedInput as KeyedStream }
+    }
+    default:
+      throw new Error(`Unsupported join source type: ${(from as any).type}`)
+  }
+}
+
+/**
+ * Processes the results of a join operation
+ */
+function processJoinResults(joinType: string) {
+  return function (
+    pipeline: IStreamBuilder<
+      [
+        key: string,
+        [
+          [string, NamespacedRow] | undefined,
+          [string, NamespacedRow] | undefined,
+        ],
+      ]
+    >
+  ): NamespacedAndKeyedStream {
+    return pipeline.pipe(
+      // Process the join result and handle nulls
+      filter((result) => {
+        const [_key, [main, joined]] = result
+        const mainNamespacedRow = main?.[1]
+        const joinedNamespacedRow = joined?.[1]
+
+        // Handle different join types
+        if (joinType === `inner`) {
+          return !!(mainNamespacedRow && joinedNamespacedRow)
+        }
+
+        if (joinType === `left`) {
+          return !!mainNamespacedRow
+        }
+
+        if (joinType === `right`) {
+          return !!joinedNamespacedRow
+        }
+
+        // For full joins, always include
+        return true
+      }),
+      map((result) => {
+        const [_key, [main, joined]] = result
+        const mainKey = main?.[0]
+        const mainNamespacedRow = main?.[1]
+        const joinedKey = joined?.[0]
+        const joinedNamespacedRow = joined?.[1]
+
+        // Merge the namespaced rows
+        const mergedNamespacedRow: NamespacedRow = {}
+
+        // Add main row data if it exists
+        if (mainNamespacedRow) {
+          Object.assign(mergedNamespacedRow, mainNamespacedRow)
+        }
+
+        // Add joined row data if it exists
+        if (joinedNamespacedRow) {
+          Object.assign(mergedNamespacedRow, joinedNamespacedRow)
+        }
+
+        // We create a composite key that combines the main and joined keys
+        const resultKey = `[${mainKey},${joinedKey}]`
+
+        return [resultKey, mergedNamespacedRow] as [string, NamespacedRow]
+      })
+    )
+  }
+}

package/src/query/compiler/order-by.ts

@@ -0,0 +1,139 @@
+import { orderByWithFractionalIndex } from "@electric-sql/d2mini"
+import { compileExpression } from "./evaluators.js"
+import type { OrderByClause } from "../ir.js"
+import type { NamespacedAndKeyedStream, NamespacedRow } from "../../types.js"
+import type { IStreamBuilder, KeyValue } from "@electric-sql/d2mini"
+
+/**
+ * Processes the ORDER BY clause
+ * Works with the new structure that has both namespaced row data and __select_results
+ * Always uses fractional indexing and adds the index as __ordering_index to the result
+ */
+export function processOrderBy(
+  pipeline: NamespacedAndKeyedStream,
+  orderByClause: Array<OrderByClause>,
+  limit?: number,
+  offset?: number
+): IStreamBuilder<KeyValue<unknown, [NamespacedRow, string]>> {
+  // Pre-compile all order by expressions
+  const compiledOrderBy = orderByClause.map((clause) => ({
+    compiledExpression: compileExpression(clause.expression),
+    direction: clause.direction,
+  }))
+
+  // Create a value extractor function for the orderBy operator
+  const valueExtractor = (row: NamespacedRow & { __select_results?: any }) => {
+    // For ORDER BY expressions, we need to provide access to both:
+    // 1. The original namespaced row data (for direct table column references)
+    // 2. The __select_results (for SELECT alias references)
+
+    // Create a merged context for expression evaluation
+    const orderByContext = { ...row }
+
+    // If there are select results, merge them at the top level for alias access
+    if (row.__select_results) {
+      // Add select results as top-level properties for alias access
+      Object.assign(orderByContext, row.__select_results)
+    }
+
+    if (orderByClause.length > 1) {
+      // For multiple orderBy columns, create a composite key
+      return compiledOrderBy.map((compiled) =>
+        compiled.compiledExpression(orderByContext)
+      )
+    } else if (orderByClause.length === 1) {
+      // For a single orderBy column, use the value directly
+      const compiled = compiledOrderBy[0]!
+      return compiled.compiledExpression(orderByContext)
+    }
+
+    // Default case - no ordering
+    return null
+  }
+
+  // Create comparator functions
+  const ascComparator = (a: any, b: any): number => {
+    // Handle null/undefined
+    if (a == null && b == null) return 0
+    if (a == null) return -1
+    if (b == null) return 1
+
+    // if a and b are both strings, compare them based on locale
+    if (typeof a === `string` && typeof b === `string`) {
+      return a.localeCompare(b)
+    }
+
+    // if a and b are both arrays, compare them element by element
+    if (Array.isArray(a) && Array.isArray(b)) {
+      for (let i = 0; i < Math.min(a.length, b.length); i++) {
+        const result = ascComparator(a[i], b[i])
+        if (result !== 0) {
+          return result
+        }
+      }
+      // All elements are equal up to the minimum length
+      return a.length - b.length
+    }
+
+    // If both are dates, compare them
+    if (a instanceof Date && b instanceof Date) {
+      return a.getTime() - b.getTime()
+    }
+
+    // If at least one of the values is an object, convert to strings
+    const bothObjects = typeof a === `object` && typeof b === `object`
+    const notNull = a !== null && b !== null
+    if (bothObjects && notNull) {
+      return a.toString().localeCompare(b.toString())
+    }
+
+    if (a < b) return -1
+    if (a > b) return 1
+    return 0
+  }
+
+  const descComparator = (a: unknown, b: unknown): number => {
+    return ascComparator(b, a)
+  }
+
+  // Create a multi-property comparator that respects the order and direction of each property
+  const makeComparator = () => {
+    return (a: unknown, b: unknown) => {
+      // If we're comparing arrays (multiple properties), compare each property in order
+      if (orderByClause.length > 1) {
+        const arrayA = a as Array<unknown>
+        const arrayB = b as Array<unknown>
+        for (let i = 0; i < orderByClause.length; i++) {
+          const direction = orderByClause[i]!.direction
+          const compareFn =
+            direction === `desc` ? descComparator : ascComparator
+          const result = compareFn(arrayA[i], arrayB[i])
+          if (result !== 0) {
+            return result
+          }
+        }
+        return arrayA.length - arrayB.length
+      }
+
+      // Single property comparison
+      if (orderByClause.length === 1) {
+        const direction = orderByClause[0]!.direction
+        return direction === `desc` ? descComparator(a, b) : ascComparator(a, b)
+      }
+
+      return ascComparator(a, b)
+    }
+  }
+
+  const comparator = makeComparator()
+
+  // Use fractional indexing and return the tuple [value, index]
+  return pipeline.pipe(
+    orderByWithFractionalIndex(valueExtractor, {
+      limit,
+      offset,
+      comparator,
+    })
+    // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format
+  )
+}

package/src/query/compiler/select.ts

@@ -0,0 +1,173 @@
+import { map } from "@electric-sql/d2mini"
+import { compileExpression } from "./evaluators.js"
+import type { Aggregate, BasicExpression, Select } from "../ir.js"
+import type {
+  KeyedStream,
+  NamespacedAndKeyedStream,
+  NamespacedRow,
+} from "../../types.js"
+
+/**
+ * Processes the SELECT clause and places results in __select_results
+ * while preserving the original namespaced row for ORDER BY access
+ */
+export function processSelectToResults(
+  pipeline: NamespacedAndKeyedStream,
+  select: Select,
+  _allInputs: Record<string, KeyedStream>
+): NamespacedAndKeyedStream {
+  // Pre-compile all select expressions
+  const compiledSelect: Array<{
+    alias: string
+    compiledExpression: (row: NamespacedRow) => any
+  }> = []
+  const spreadAliases: Array<string> = []
+
+  for (const [alias, expression] of Object.entries(select)) {
+    if (alias.startsWith(`__SPREAD_SENTINEL__`)) {
+      // Extract the table alias from the sentinel key
+      const tableAlias = alias.replace(`__SPREAD_SENTINEL__`, ``)
+      spreadAliases.push(tableAlias)
+    } else {
+      if (isAggregateExpression(expression)) {
+        // For aggregates, we'll store the expression info for GROUP BY processing
+        // but still compile a placeholder that will be replaced later
+        compiledSelect.push({
+          alias,
+          compiledExpression: () => null, // Placeholder - will be handled by GROUP BY
+        })
+      } else {
+        compiledSelect.push({
+          alias,
+          compiledExpression: compileExpression(expression as BasicExpression),
+        })
+      }
+    }
+  }
+
+  return pipeline.pipe(
+    map(([key, namespacedRow]) => {
+      const selectResults: Record<string, any> = {}
+
+      // First pass: spread table data for any spread sentinels
+      for (const tableAlias of spreadAliases) {
+        const tableData = namespacedRow[tableAlias]
+        if (tableData && typeof tableData === `object`) {
+          // Spread the table data into the result, but don't overwrite explicit fields
+          for (const [fieldName, fieldValue] of Object.entries(tableData)) {
+            if (!(fieldName in selectResults)) {
+              selectResults[fieldName] = fieldValue
+            }
+          }
+        }
+      }
+
+      // Second pass: evaluate all compiled select expressions (non-aggregates)
+      for (const { alias, compiledExpression } of compiledSelect) {
+        selectResults[alias] = compiledExpression(namespacedRow)
+      }
+
+      // Return the namespaced row with __select_results added
+      return [
+        key,
+        {
+          ...namespacedRow,
+          __select_results: selectResults,
+        },
+      ] as [
+        string,
+        typeof namespacedRow & { __select_results: typeof selectResults },
+      ]
+    })
+  )
+}
+
+/**
+ * Processes the SELECT clause (legacy function - kept for compatibility)
+ */
+export function processSelect(
+  pipeline: NamespacedAndKeyedStream,
+  select: Select,
+  _allInputs: Record<string, KeyedStream>
+): KeyedStream {
+  // Pre-compile all select expressions
+  const compiledSelect: Array<{
+    alias: string
+    compiledExpression: (row: NamespacedRow) => any
+  }> = []
+  const spreadAliases: Array<string> = []
+
+  for (const [alias, expression] of Object.entries(select)) {
+    if (alias.startsWith(`__SPREAD_SENTINEL__`)) {
+      // Extract the table alias from the sentinel key
+      const tableAlias = alias.replace(`__SPREAD_SENTINEL__`, ``)
+      spreadAliases.push(tableAlias)
+    } else {
+      if (isAggregateExpression(expression)) {
+        // Aggregates should be handled by GROUP BY processing, not here
+        throw new Error(
+          `Aggregate expressions in SELECT clause should be handled by GROUP BY processing`
+        )
+      }
+      compiledSelect.push({
+        alias,
+        compiledExpression: compileExpression(expression as BasicExpression),
+      })
+    }
+  }
+
+  return pipeline.pipe(
+    map(([key, namespacedRow]) => {
+      const result: Record<string, any> = {}
+
+      // First pass: spread table data for any spread sentinels
+      for (const tableAlias of spreadAliases) {
+        const tableData = namespacedRow[tableAlias]
+        if (tableData && typeof tableData === `object`) {
+          // Spread the table data into the result, but don't overwrite explicit fields
+          for (const [fieldName, fieldValue] of Object.entries(tableData)) {
+            if (!(fieldName in result)) {
+              result[fieldName] = fieldValue
+            }
+          }
+        }
+      }
+
+      // Second pass: evaluate all compiled select expressions
+      for (const { alias, compiledExpression } of compiledSelect) {
+        result[alias] = compiledExpression(namespacedRow)
+      }
+
+      return [key, result] as [string, typeof result]
+    })
+  )
+}
+
+/**
+ * Helper function to check if an expression is an aggregate
+ */
+function isAggregateExpression(
+  expr: BasicExpression | Aggregate
+): expr is Aggregate {
+  return expr.type === `agg`
+}
+
+/**
+ * Processes a single argument in a function context
+ */
+export function processArgument(
+  arg: BasicExpression | Aggregate,
+  namespacedRow: NamespacedRow
+): any {
+  if (isAggregateExpression(arg)) {
+    throw new Error(
+      `Aggregate expressions are not supported in this context. Use GROUP BY clause for aggregates.`
+    )
+  }
+
+  // Pre-compile the expression and evaluate immediately
+  const compiledExpression = compileExpression(arg)
+  const value = compiledExpression(namespacedRow)
+
+  return value
+}

package/src/query/index.ts

@@ -1,5 +1,64 @@
-export * from "./query-builder.js"
-export * from "./compiled-query.js"
-export * from "./pipeline-compiler.js"
-export * from "./schema.js"
-export * from "./types.js"
+// Main exports for the new query builder system
+
+// Query builder exports
+export {
+  BaseQueryBuilder,
+  Query,
+  type InitialQueryBuilder,
+  type QueryBuilder,
+  type Context,
+  type Source,
+  type GetResult,
+} from "./builder/index.js"
+
+// Expression functions exports
+export {
+  // Operators
+  eq,
+  gt,
+  gte,
+  lt,
+  lte,
+  and,
+  or,
+  not,
+  inArray,
+  like,
+  ilike,
+  // Functions
+  upper,
+  lower,
+  length,
+  concat,
+  coalesce,
+  add,
+  // Aggregates
+  count,
+  avg,
+  sum,
+  min,
+  max,
+} from "./builder/functions.js"
+
+// Ref proxy utilities
+export { val, toExpression, isRefProxy } from "./builder/ref-proxy.js"
+
+// IR types (for advanced usage)
+export type {
+  QueryIR,
+  BasicExpression as Expression,
+  Aggregate,
+  CollectionRef,
+  QueryRef,
+  JoinClause,
+} from "./ir.js"
+
+// Compiler
+export { compileQuery } from "./compiler/index.js"
+
+// Live query collection utilities
+export {
+  createLiveQueryCollection,
+  liveQueryCollectionOptions,
+  type LiveQueryCollectionConfig,
+} from "./live-query-collection.js"