@tanstack/db 0.0.14 → 0.0.16

package/src/query/joins.ts

@@ -1,260 +0,0 @@
-import {
-  consolidate,
-  filter,
-  join as joinOperator,
-  map,
-} from "@electric-sql/d2mini"
-import { evaluateConditionOnNamespacedRow } from "./evaluators.js"
-import { extractJoinKey } from "./extractors.js"
-import type { Query } from "./index.js"
-import type { IStreamBuilder, JoinType } from "@electric-sql/d2mini"
-import type {
-  KeyedStream,
-  NamespacedAndKeyedStream,
-  NamespacedRow,
-} from "../types.js"
-
-/**
- * Creates a processing pipeline for join clauses
- */
-export function processJoinClause(
-  pipeline: NamespacedAndKeyedStream,
-  query: Query,
-  tables: Record<string, KeyedStream>,
-  mainTableAlias: string,
-  allInputs: Record<string, KeyedStream>
-) {
-  if (!query.join) return pipeline
-  const input = allInputs[query.from]
-
-  for (const joinClause of query.join) {
-    // Create a stream for the joined table
-    const joinedTableAlias = joinClause.as || joinClause.from
-
-    // Get the right join type for the operator
-    const joinType: JoinType =
-      joinClause.type === `cross` ? `inner` : joinClause.type
-
-    // The `in` is formatted as ['@mainKeyRef', '=', '@joinedKeyRef']
-    // Destructure the main key reference and the joined key references
-    const [mainKeyRef, , joinedKeyRefs] = joinClause.on
-
-    // We need to prepare the main pipeline and the joined pipeline
-    // to have the correct key format for joining
-    const mainPipeline = pipeline.pipe(
-      map(([currentKey, namespacedRow]) => {
-        // Extract the key from the ON condition left side for the main table
-        const mainRow = namespacedRow[mainTableAlias]!
-
-        // Extract the join key from the main row
-        const key = extractJoinKey(mainRow, mainKeyRef, mainTableAlias)
-
-        // Return [key, namespacedRow] as a KeyValue type
-        return [key, [currentKey, namespacedRow]] as [
-          unknown,
-          [string, typeof namespacedRow],
-        ]
-      })
-    )
-
-    // Get the joined table input from the inputs map
-    let joinedTableInput: KeyedStream
-
-    if (allInputs[joinClause.from]) {
-      // Use the provided input if available
-      joinedTableInput = allInputs[joinClause.from]!
-    } else {
-      // Create a new input if not provided
-      joinedTableInput =
-        input!.graph.newInput<[string, Record<string, unknown>]>()
-    }
-
-    tables[joinedTableAlias] = joinedTableInput
-
-    // Create a pipeline for the joined table
-    const joinedPipeline = joinedTableInput.pipe(
-      map(([currentKey, row]) => {
-        // Wrap the row in an object with the table alias as the key
-        const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }
-
-        // Extract the key from the ON condition right side for the joined table
-        const key = extractJoinKey(row, joinedKeyRefs, joinedTableAlias)
-
-        // Return [key, namespacedRow] as a KeyValue type
-        return [key, [currentKey, namespacedRow]] as [
-          string,
-          [string, typeof namespacedRow],
-        ]
-      })
-    )
-
-    // Apply join with appropriate typings based on join type
-    switch (joinType) {
-      case `inner`:
-        pipeline = mainPipeline.pipe(
-          joinOperator(joinedPipeline, `inner`),
-          consolidate(),
-          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
-        )
-        break
-      case `left`:
-        pipeline = mainPipeline.pipe(
-          joinOperator(joinedPipeline, `left`),
-          consolidate(),
-          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
-        )
-        break
-      case `right`:
-        pipeline = mainPipeline.pipe(
-          joinOperator(joinedPipeline, `right`),
-          consolidate(),
-          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
-        )
-        break
-      case `full`:
-        pipeline = mainPipeline.pipe(
-          joinOperator(joinedPipeline, `full`),
-          consolidate(),
-          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
-        )
-        break
-      default:
-        pipeline = mainPipeline.pipe(
-          joinOperator(joinedPipeline, `inner`),
-          consolidate(),
-          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
-        )
-    }
-  }
-  return pipeline
-}
-
-/**
- * Creates a processing pipeline for join results
- */
-export function processJoinResults(
-  mainTableAlias: string,
-  joinedTableAlias: string,
-  joinClause: { on: any; type: string }
-) {
-  return function (
-    pipeline: IStreamBuilder<
-      [
-        key: string,
-        [
-          [string, NamespacedRow] | undefined,
-          [string, NamespacedRow] | undefined,
-        ],
-      ]
-    >
-  ): NamespacedAndKeyedStream {
-    return pipeline.pipe(
-      // Process the join result and handle nulls in the same step
-      map((result) => {
-        const [_key, [main, joined]] = result
-        const mainKey = main?.[0]
-        const mainNamespacedRow = main?.[1]
-        const joinedKey = joined?.[0]
-        const joinedNamespacedRow = joined?.[1]
-
-        // For inner joins, both sides should be non-null
-        if (joinClause.type === `inner` || joinClause.type === `cross`) {
-          if (!mainNamespacedRow || !joinedNamespacedRow) {
-            return undefined // Will be filtered out
-          }
-        }
-
-        // For left joins, the main row must be non-null
-        if (joinClause.type === `left` && !mainNamespacedRow) {
-          return undefined // Will be filtered out
-        }
-
-        // For right joins, the joined row must be non-null
-        if (joinClause.type === `right` && !joinedNamespacedRow) {
-          return undefined // Will be filtered out
-        }
-
-        // Merge the nested rows
-        const mergedNamespacedRow: NamespacedRow = {}
-
-        // Add main row data if it exists
-        if (mainNamespacedRow) {
-          Object.entries(mainNamespacedRow).forEach(
-            ([tableAlias, tableData]) => {
-              mergedNamespacedRow[tableAlias] = tableData
-            }
-          )
-        }
-
-        // If we have a joined row, add it to the merged result
-        if (joinedNamespacedRow) {
-          Object.entries(joinedNamespacedRow).forEach(
-            ([tableAlias, tableData]) => {
-              mergedNamespacedRow[tableAlias] = tableData
-            }
-          )
-        } else if (joinClause.type === `left` || joinClause.type === `full`) {
-          // For left or full joins, add the joined table with undefined data if missing
-          // mergedNamespacedRow[joinedTableAlias] = undefined
-        }
-
-        // For right or full joins, add the main table with undefined data if missing
-        if (
-          !mainNamespacedRow &&
-          (joinClause.type === `right` || joinClause.type === `full`)
-        ) {
-          // mergedNamespacedRow[mainTableAlias] = undefined
-        }
-
-        // New key
-        const newKey = `[${mainKey},${joinedKey}]`
-
-        return [newKey, mergedNamespacedRow] as [
-          string,
-          typeof mergedNamespacedRow,
-        ]
-      }),
-      // Filter out undefined results
-      filter((value) => value !== undefined),
-      // Process the ON condition
-      filter(([_key, namespacedRow]: [string, NamespacedRow]) => {
-        // If there's no ON condition, or it's a cross join, always return true
-        if (!joinClause.on || joinClause.type === `cross`) {
-          return true
-        }
-
-        // For LEFT JOIN, if the right side is null, we should include the row
-        if (
-          joinClause.type === `left` &&
-          namespacedRow[joinedTableAlias] === undefined
-        ) {
-          return true
-        }
-
-        // For RIGHT JOIN, if the left side is null, we should include the row
-        if (
-          joinClause.type === `right` &&
-          namespacedRow[mainTableAlias] === undefined
-        ) {
-          return true
-        }
-
-        // For FULL JOIN, if either side is null, we should include the row
-        if (
-          joinClause.type === `full` &&
-          (namespacedRow[mainTableAlias] === undefined ||
-            namespacedRow[joinedTableAlias] === undefined)
-        ) {
-          return true
-        }
-
-        return evaluateConditionOnNamespacedRow(
-          namespacedRow,
-          joinClause.on,
-          mainTableAlias,
-          joinedTableAlias
-        )
-      })
-    )
-  }
-}

package/src/query/order-by.ts

@@ -1,264 +0,0 @@
-import {
-  map,
-  orderBy,
-  orderByWithFractionalIndex,
-  orderByWithIndex,
-} from "@electric-sql/d2mini"
-import { evaluateOperandOnNamespacedRow } from "./extractors"
-import { isOrderIndexFunctionCall } from "./utils"
-import type { ConditionOperand, Query } from "./schema"
-import type {
-  KeyedNamespacedRow,
-  NamespacedAndKeyedStream,
-  NamespacedRow,
-} from "../types"
-
-type OrderByItem = {
-  operand: ConditionOperand
-  direction: `asc` | `desc`
-}
-
-type OrderByItems = Array<OrderByItem>
-
-export function processOrderBy(
-  resultPipeline: NamespacedAndKeyedStream,
-  query: Query,
-  mainTableAlias: string
-) {
-  // Check if any column in the SELECT clause is an ORDER_INDEX function call
-  let hasOrderIndexColumn = false
-  let orderIndexType: `numeric` | `fractional` = `numeric`
-  let orderIndexAlias = ``
-
-  // Scan the SELECT clause for ORDER_INDEX functions
-  // TODO: Select is going to be optional in future - we will automatically add an
-  // attribute for the index column
-  for (const item of query.select!) {
-    if (typeof item === `object`) {
-      for (const [alias, expr] of Object.entries(item)) {
-        if (typeof expr === `object` && isOrderIndexFunctionCall(expr)) {
-          hasOrderIndexColumn = true
-          orderIndexAlias = alias
-          orderIndexType = getOrderIndexType(expr)
-          break
-        }
-      }
-    }
-    if (hasOrderIndexColumn) break
-  }
-
-  // Normalize orderBy to an array of objects
-  const orderByItems: OrderByItems = []
-
-  if (typeof query.orderBy === `string`) {
-    // Handle string format: '@column'
-    orderByItems.push({
-      operand: query.orderBy,
-      direction: `asc`,
-    })
-  } else if (Array.isArray(query.orderBy)) {
-    // Handle array format: ['@column1', { '@column2': 'desc' }]
-    for (const item of query.orderBy) {
-      if (typeof item === `string`) {
-        orderByItems.push({
-          operand: item,
-          direction: `asc`,
-        })
-      } else if (typeof item === `object`) {
-        for (const [column, direction] of Object.entries(item)) {
-          orderByItems.push({
-            operand: column,
-            direction: direction as `asc` | `desc`,
-          })
-        }
-      }
-    }
-  } else if (typeof query.orderBy === `object`) {
-    // Handle object format: { '@column': 'desc' }
-    for (const [column, direction] of Object.entries(query.orderBy)) {
-      orderByItems.push({
-        operand: column,
-        direction: direction as `asc` | `desc`,
-      })
-    }
-  }
-
-  // Create a value extractor function for the orderBy operator
-  // const valueExtractor = ([key, namespacedRow]: [
-  const valueExtractor = (namespacedRow: NamespacedRow) => {
-    // For multiple orderBy columns, create a composite key
-    if (orderByItems.length > 1) {
-      return orderByItems.map((item) =>
-        evaluateOperandOnNamespacedRow(
-          namespacedRow,
-          item.operand,
-          mainTableAlias
-        )
-      )
-    } else if (orderByItems.length === 1) {
-      // For a single orderBy column, use the value directly
-      const item = orderByItems[0]
-      const val = evaluateOperandOnNamespacedRow(
-        namespacedRow,
-        item!.operand,
-        mainTableAlias
-      )
-      return val
-    }
-
-    // Default case - no ordering
-    return null
-  }
-
-  const ascComparator = (a: any, b: any): number => {
-    // if a and b are both strings, compare them based on locale
-    if (typeof a === `string` && typeof b === `string`) {
-      return a.localeCompare(b)
-    }
-
-    // if a and b are both arrays, compare them element by element
-    if (Array.isArray(a) && Array.isArray(b)) {
-      for (let i = 0; i < Math.min(a.length, b.length); i++) {
-        // Compare the values
-        const result = ascComparator(a[i], b[i])
-
-        if (result !== 0) {
-          return result
-        }
-      }
-      // All elements are equal up to the minimum length
-      return a.length - b.length
-    }
-
-    // If at least one of the values is an object then we don't really know how to meaningfully compare them
-    // therefore we turn them into strings and compare those
-    // There are 2 exceptions:
-    // 1) if both objects are dates then we can compare them
-    // 2) if either object is nullish then we can't call toString on it
-    const bothObjects = typeof a === `object` && typeof b === `object`
-    const bothDates = a instanceof Date && b instanceof Date
-    const notNull = a !== null && b !== null
-    if (bothObjects && !bothDates && notNull) {
-      // Every object should support `toString`
-      return a.toString().localeCompare(b.toString())
-    }
-
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
-  }
-
-  const descComparator = (a: unknown, b: unknown): number => {
-    return ascComparator(b, a)
-  }
-
-  // Create a multi-property comparator that respects the order and direction of each property
-  const makeComparator = (orderByProps: OrderByItems) => {
-    return (a: unknown, b: unknown) => {
-      // If we're comparing arrays (multiple properties), compare each property in order
-      if (orderByProps.length > 1) {
-        // `a` and `b` must be arrays since `orderByItems.length > 1`
-        // hence the extracted values must be arrays
-        const arrayA = a as Array<unknown>
-        const arrayB = b as Array<unknown>
-        for (let i = 0; i < orderByProps.length; i++) {
-          const direction = orderByProps[i]!.direction
-          const compareFn =
-            direction === `desc` ? descComparator : ascComparator
-          const result = compareFn(arrayA[i], arrayB[i])
-          if (result !== 0) {
-            return result
-          }
-        }
-        // should normally always be 0 because
-        // both values are extracted based on orderByItems
-        return arrayA.length - arrayB.length
-      }
-
-      // Single property comparison
-      if (orderByProps.length === 1) {
-        const direction = orderByProps[0]!.direction
-        return direction === `desc` ? descComparator(a, b) : ascComparator(a, b)
-      }
-
-      return ascComparator(a, b)
-    }
-  }
-  const comparator = makeComparator(orderByItems)
-
-  // Apply the appropriate orderBy operator based on whether an ORDER_INDEX column is requested
-  if (hasOrderIndexColumn) {
-    if (orderIndexType === `numeric`) {
-      // Use orderByWithIndex for numeric indices
-      resultPipeline = resultPipeline.pipe(
-        orderByWithIndex(valueExtractor, {
-          limit: query.limit,
-          offset: query.offset,
-          comparator,
-        }),
-        map(([key, [value, index]]) => {
-          // Add the index to the result
-          // We add this to the main table alias for now
-          // TODO: re are going to need to refactor the whole order by pipeline
-          const result = {
-            ...(value as Record<string, unknown>),
-            [mainTableAlias]: {
-              ...value[mainTableAlias],
-              [orderIndexAlias]: index,
-            },
-          }
-          return [key, result] as KeyedNamespacedRow
-        })
-      )
-    } else {
-      // Use orderByWithFractionalIndex for fractional indices
-      resultPipeline = resultPipeline.pipe(
-        orderByWithFractionalIndex(valueExtractor, {
-          limit: query.limit,
-          offset: query.offset,
-          comparator,
-        }),
-        map(([key, [value, index]]) => {
-          // Add the index to the result
-          // We add this to the main table alias for now
-          // TODO: re are going to need to refactor the whole order by pipeline
-          const result = {
-            ...(value as Record<string, unknown>),
-            [mainTableAlias]: {
-              ...value[mainTableAlias],
-              [orderIndexAlias]: index,
-            },
-          }
-          return [key, result] as KeyedNamespacedRow
-        })
-      )
-    }
-  } else {
-    // Use regular orderBy if no index column is requested
-    resultPipeline = resultPipeline.pipe(
-      orderBy(valueExtractor, {
-        limit: query.limit,
-        offset: query.offset,
-        comparator,
-      })
-    )
-  }
-
-  return resultPipeline
-}
-
-// Helper function to extract the ORDER_INDEX type from a function call
-function getOrderIndexType(obj: any): `numeric` | `fractional` {
-  if (!isOrderIndexFunctionCall(obj)) {
-    throw new Error(`Not an ORDER_INDEX function call`)
-  }
-
-  const arg = obj[`ORDER_INDEX`]
-  if (arg === `numeric` || arg === true || arg === `default`) {
-    return `numeric`
-  } else if (arg === `fractional`) {
-    return `fractional`
-  } else {
-    throw new Error(`Invalid ORDER_INDEX type: ` + arg)
-  }
-}

package/src/query/pipeline-compiler.ts

@@ -1,149 +0,0 @@
-import { filter, map } from "@electric-sql/d2mini"
-import { evaluateWhereOnNamespacedRow } from "./evaluators.js"
-import { processJoinClause } from "./joins.js"
-import { processGroupBy } from "./group-by.js"
-import { processOrderBy } from "./order-by.js"
-import { processSelect } from "./select.js"
-import type { Query } from "./schema.js"
-import type { IStreamBuilder } from "@electric-sql/d2mini"
-import type {
-  InputRow,
-  KeyedStream,
-  NamespacedAndKeyedStream,
-} from "../types.js"
-
-/**
- * Compiles a query into a D2 pipeline
- * @param query The query to compile
- * @param inputs Mapping of table names to input streams
- * @returns A stream builder representing the compiled query
- */
-export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
-  query: Query,
-  inputs: Record<string, KeyedStream>
-): T {
-  // Create a copy of the inputs map to avoid modifying the original
-  const allInputs = { ...inputs }
-
-  // Process WITH queries if they exist
-  if (query.with && query.with.length > 0) {
-    // Process each WITH query in order
-    for (const withQuery of query.with) {
-      // Ensure the WITH query has an alias
-      if (!withQuery.as) {
-        throw new Error(`WITH query must have an "as" property`)
-      }
-
-      // Check if this CTE name already exists in the inputs
-      if (allInputs[withQuery.as]) {
-        throw new Error(`CTE with name "${withQuery.as}" already exists`)
-      }
-
-      // Create a new query without the 'with' property to avoid circular references
-      const withQueryWithoutWith = { ...withQuery, with: undefined }
-
-      // Compile the WITH query using the current set of inputs
-      // (which includes previously compiled WITH queries)
-      const compiledWithQuery = compileQueryPipeline(
-        withQueryWithoutWith,
-        allInputs
-      )
-
-      // Add the compiled query to the inputs map using its alias
-      allInputs[withQuery.as] = compiledWithQuery as KeyedStream
-    }
-  }
-
-  // Create a map of table aliases to inputs
-  const tables: Record<string, KeyedStream> = {}
-
-  // The main table is the one in the FROM clause
-  const mainTableAlias = query.as || query.from
-
-  // Get the main input from the inputs map (now including CTEs)
-  const input = allInputs[query.from]
-  if (!input) {
-    throw new Error(`Input for table "${query.from}" not found in inputs map`)
-  }
-
-  tables[mainTableAlias] = input
-
-  // Prepare the initial pipeline with the main table wrapped in its alias
-  let pipeline: NamespacedAndKeyedStream = input.pipe(
-    map(([key, row]) => {
-      // Initialize the record with a nested structure
-      const ret = [key, { [mainTableAlias]: row }] as [
-        string,
-        Record<string, typeof row>,
-      ]
-      return ret
-    })
-  )
-
-  // Process JOIN clauses if they exist
-  if (query.join) {
-    pipeline = processJoinClause(
-      pipeline,
-      query,
-      tables,
-      mainTableAlias,
-      allInputs
-    )
-  }
-
-  // Process the WHERE clause if it exists
-  if (query.where) {
-    pipeline = pipeline.pipe(
-      filter(([_key, row]) => {
-        const result = evaluateWhereOnNamespacedRow(
-          row,
-          query.where!,
-          mainTableAlias
-        )
-        return result
-      })
-    )
-  }
-
-  // Process the GROUP BY clause if it exists
-  if (query.groupBy) {
-    pipeline = processGroupBy(pipeline, query, mainTableAlias)
-  }
-
-  // Process the HAVING clause if it exists
-  // This works similarly to WHERE but is applied after any aggregations
-  if (query.having) {
-    pipeline = pipeline.pipe(
-      filter(([_key, row]) => {
-        // For HAVING, we're working with the flattened row that contains both
-        // the group by keys and the aggregate results directly
-        const result = evaluateWhereOnNamespacedRow(
-          row,
-          query.having!,
-          mainTableAlias
-        )
-        return result
-      })
-    )
-  }
-
-  // Process orderBy parameter if it exists
-  if (query.orderBy) {
-    pipeline = processOrderBy(pipeline, query, mainTableAlias)
-  } else if (query.limit !== undefined || query.offset !== undefined) {
-    // If there's a limit or offset without orderBy, throw an error
-    throw new Error(
-      `LIMIT and OFFSET require an ORDER BY clause to ensure deterministic results`
-    )
-  }
-
-  // Process the SELECT clause - this is where we flatten the structure
-  const resultPipeline: KeyedStream | NamespacedAndKeyedStream = query.select
-    ? processSelect(pipeline, query, mainTableAlias, allInputs)
-    : !query.join && !query.groupBy
-      ? pipeline.pipe(
-          map(([key, row]) => [key, row[mainTableAlias]] as InputRow)
-        )
-      : pipeline
-  return resultPipeline as T
-}