@tanstack/db 0.0.1

package/src/query/functions.ts

@@ -0,0 +1,297 @@
+import type { AllowedFunctionName } from "./schema.js"
+
+/**
+ * Type for function implementations
+ */
+type FunctionImplementation = (arg: unknown) => unknown
+
+/**
+ * Converts a string to uppercase
+ */
+function upperFunction(arg: unknown): string {
+  if (typeof arg !== `string`) {
+    throw new Error(`UPPER function expects a string argument`)
+  }
+  return arg.toUpperCase()
+}
+
+/**
+ * Converts a string to lowercase
+ */
+function lowerFunction(arg: unknown): string {
+  if (typeof arg !== `string`) {
+    throw new Error(`LOWER function expects a string argument`)
+  }
+  return arg.toLowerCase()
+}
+
+/**
+ * Returns the length of a string or array
+ */
+function lengthFunction(arg: unknown): number {
+  if (typeof arg === `string` || Array.isArray(arg)) {
+    return arg.length
+  }
+
+  throw new Error(`LENGTH function expects a string or array argument`)
+}
+
+/**
+ * Concatenates multiple strings
+ */
+function concatFunction(arg: unknown): string {
+  if (!Array.isArray(arg)) {
+    throw new Error(`CONCAT function expects an array of string arguments`)
+  }
+
+  if (arg.length === 0) {
+    return ``
+  }
+
+  // Check that all arguments are strings
+  for (let i = 0; i < arg.length; i++) {
+    if (arg[i] !== null && arg[i] !== undefined && typeof arg[i] !== `string`) {
+      throw new Error(
+        `CONCAT function expects all arguments to be strings, but argument at position ${i} is ${typeof arg[i]}`
+      )
+    }
+  }
+
+  // Concatenate strings, treating null and undefined as empty strings
+  return arg
+    .map((str) => (str === null || str === undefined ? `` : str))
+    .join(``)
+}
+
+/**
+ * Returns the first non-null, non-undefined value from an array
+ */
+function coalesceFunction(arg: unknown): unknown {
+  if (!Array.isArray(arg)) {
+    throw new Error(`COALESCE function expects an array of arguments`)
+  }
+
+  if (arg.length === 0) {
+    return null
+  }
+
+  // Return the first non-null, non-undefined value
+  for (const value of arg) {
+    if (value !== null && value !== undefined) {
+      return value
+    }
+  }
+
+  // If all values were null or undefined, return null
+  return null
+}
+
+/**
+ * Creates or converts a value to a Date object
+ */
+function dateFunction(arg: unknown): Date | null {
+  // If the argument is already a Date, return it
+  if (arg instanceof Date) {
+    return arg
+  }
+
+  // If the argument is null or undefined, return null
+  if (arg === null || arg === undefined) {
+    return null
+  }
+
+  // Handle string and number conversions
+  if (typeof arg === `string` || typeof arg === `number`) {
+    const date = new Date(arg)
+
+    // Check if the date is valid
+    if (isNaN(date.getTime())) {
+      throw new Error(`DATE function could not parse "${arg}" as a valid date`)
+    }
+
+    return date
+  }
+
+  throw new Error(`DATE function expects a string, number, or Date argument`)
+}
+
+/**
+ * Extracts a value from a JSON string or object using a path.
+ * Similar to PostgreSQL's json_extract_path function.
+ *
+ * Usage: JSON_EXTRACT([jsonInput, 'path', 'to', 'property'])
+ * Example: JSON_EXTRACT(['{"user": {"name": "John"}}', 'user', 'name']) returns "John"
+ */
+function jsonExtractFunction(arg: unknown): unknown {
+  if (!Array.isArray(arg) || arg.length < 1) {
+    throw new Error(
+      `JSON_EXTRACT function expects an array with at least one element [jsonInput, ...pathElements]`
+    )
+  }
+
+  const [jsonInput, ...pathElements] = arg
+
+  // Handle null or undefined input
+  if (jsonInput === null || jsonInput === undefined) {
+    return null
+  }
+
+  // Parse JSON if it's a string
+  let jsonData: any
+
+  if (typeof jsonInput === `string`) {
+    try {
+      jsonData = JSON.parse(jsonInput)
+    } catch (error) {
+      throw new Error(
+        `JSON_EXTRACT function could not parse JSON string: ${error instanceof Error ? error.message : String(error)}`
+      )
+    }
+  } else if (typeof jsonInput === `object`) {
+    // If already an object, use it directly
+    jsonData = jsonInput
+  } else {
+    throw new Error(
+      `JSON_EXTRACT function expects a JSON string or object as the first argument`
+    )
+  }
+
+  // If no path elements, return the parsed JSON
+  if (pathElements.length === 0) {
+    return jsonData
+  }
+
+  // Navigate through the path elements
+  let current = jsonData
+
+  for (let i = 0; i < pathElements.length; i++) {
+    const pathElement = pathElements[i]
+
+    // Path elements should be strings
+    if (typeof pathElement !== `string`) {
+      throw new Error(
+        `JSON_EXTRACT function expects path elements to be strings, but element at position ${i + 1} is ${typeof pathElement}`
+      )
+    }
+
+    // If current node is null or undefined, or not an object, we can't navigate further
+    if (
+      current === null ||
+      current === undefined ||
+      typeof current !== `object`
+    ) {
+      return null
+    }
+
+    // Access property
+    current = current[pathElement]
+  }
+
+  // Return null instead of undefined for consistency
+  return current === undefined ? null : current
+}
+
+/**
+ * Placeholder function for ORDER_INDEX
+ * This function doesn't do anything when called directly, as the actual index
+ * is provided by the orderBy operator during query execution.
+ * The argument can be 'numeric', 'fractional', or any truthy value (defaults to 'numeric')
+ */
+function orderIndexFunction(arg: unknown): null {
+  // This is just a placeholder - the actual index is provided by the orderBy operator
+  // The function validates that the argument is one of the expected values
+  if (
+    arg !== `numeric` &&
+    arg !== `fractional` &&
+    arg !== true &&
+    arg !== `default`
+  ) {
+    throw new Error(
+      `ORDER_INDEX function expects "numeric", "fractional", "default", or true as argument`
+    )
+  }
+  return null
+}
+
+/**
+ * Map of function names to their implementations
+ */
+const functionImplementations: Record<
+  AllowedFunctionName,
+  FunctionImplementation
+> = {
+  // Map function names to their implementation functions
+  DATE: dateFunction,
+  JSON_EXTRACT: jsonExtractFunction,
+  JSON_EXTRACT_PATH: jsonExtractFunction, // Alias for JSON_EXTRACT
+  UPPER: upperFunction,
+  LOWER: lowerFunction,
+  COALESCE: coalesceFunction,
+  CONCAT: concatFunction,
+  LENGTH: lengthFunction,
+  ORDER_INDEX: orderIndexFunction,
+}
+
+/**
+ * Evaluates a function call with the given name and arguments
+ * @param functionName The name of the function to evaluate
+ * @param arg The arguments to pass to the function
+ * @returns The result of the function call
+ */
+export function evaluateFunction(
+  functionName: AllowedFunctionName,
+  arg: unknown
+): unknown {
+  const implementation = functionImplementations[functionName] as
+    | FunctionImplementation
+    | undefined // Double check that the implementation is defined
+
+  if (!implementation) {
+    throw new Error(`Unknown function: ${functionName}`)
+  }
+  return implementation(arg)
+}
+
+/**
+ * Determines if an object is a function call
+ * @param obj The object to check
+ * @returns True if the object is a function call, false otherwise
+ */
+export function isFunctionCall(obj: unknown): boolean {
+  if (!obj || typeof obj !== `object`) {
+    return false
+  }
+
+  const keys = Object.keys(obj)
+  if (keys.length !== 1) {
+    return false
+  }
+
+  const functionName = keys[0] as string
+
+  // Check if the key is one of the allowed function names
+  return Object.keys(functionImplementations).includes(functionName)
+}
+
+/**
+ * Extracts the function name and argument from a function call object.
+ */
+export function extractFunctionCall(obj: Record<string, unknown>): {
+  functionName: AllowedFunctionName
+  argument: unknown
+} {
+  const keys = Object.keys(obj)
+  if (keys.length !== 1) {
+    throw new Error(`Invalid function call: object must have exactly one key`)
+  }
+
+  const functionName = keys[0] as AllowedFunctionName
+  if (!Object.keys(functionImplementations).includes(functionName)) {
+    throw new Error(`Invalid function name: ${functionName}`)
+  }
+
+  return {
+    functionName,
+    argument: obj[functionName],
+  }
+}

package/src/query/group-by.ts

@@ -0,0 +1,137 @@
+import { groupBy, groupByOperators, map } from "@electric-sql/d2ts"
+import {
+  evaluateOperandOnNestedRow,
+  extractValueFromNestedRow,
+} from "./extractors"
+import { isAggregateFunctionCall } from "./utils"
+import type { ConditionOperand, FunctionCall, Query } from "./schema"
+import type { IStreamBuilder } from "@electric-sql/d2ts"
+
+const { sum, count, avg, min, max, median, mode } = groupByOperators
+
+/**
+ * Process the groupBy clause in a D2QL query
+ */
+export function processGroupBy(
+  pipeline: IStreamBuilder<Record<string, unknown>>,
+  query: Query,
+  mainTableAlias: string
+) {
+  // Normalize groupBy to an array of column references
+  const groupByColumns = Array.isArray(query.groupBy)
+    ? query.groupBy
+    : [query.groupBy]
+
+  // Create a key extractor function for the groupBy operator
+  const keyExtractor = (nestedRow: Record<string, unknown>) => {
+    const key: Record<string, unknown> = {}
+
+    // Extract each groupBy column value
+    for (const column of groupByColumns) {
+      if (typeof column === `string` && (column as string).startsWith(`@`)) {
+        const columnRef = (column as string).substring(1)
+        const columnName = columnRef.includes(`.`)
+          ? columnRef.split(`.`)[1]
+          : columnRef
+
+        key[columnName!] = extractValueFromNestedRow(
+          nestedRow,
+          columnRef,
+          mainTableAlias
+        )
+      }
+    }
+
+    return key
+  }
+
+  // Create aggregate functions for any aggregated columns in the SELECT clause
+  const aggregates: Record<string, any> = {}
+
+  // Scan the SELECT clause for aggregate functions
+  for (const item of query.select) {
+    if (typeof item === `object`) {
+      for (const [alias, expr] of Object.entries(item)) {
+        if (typeof expr === `object` && isAggregateFunctionCall(expr)) {
+          // Get the function name (the only key in the object)
+          const functionName = Object.keys(expr)[0]
+          // Get the column reference or expression to aggregate
+          const columnRef = (expr as FunctionCall)[
+            functionName as keyof FunctionCall
+          ]
+
+          // Add the aggregate function to our aggregates object
+          aggregates[alias] = getAggregateFunction(
+            functionName!,
+            columnRef,
+            mainTableAlias
+          )
+        }
+      }
+    }
+  }
+
+  // Apply the groupBy operator if we have any aggregates
+  if (Object.keys(aggregates).length > 0) {
+    pipeline = pipeline.pipe(
+      groupBy(keyExtractor, aggregates),
+      // Convert KeyValue<string, ResultType> to Record<string, unknown>
+      map(([_key, value]) => {
+        // After groupBy, the value already contains both the key fields and aggregate results
+        // We need to return it as is, not wrapped in a nested structure
+        return value as Record<string, unknown>
+      })
+    )
+  }
+
+  return pipeline
+}
+
+/**
+ * Helper function to get an aggregate function based on the function name
+ */
+export function getAggregateFunction(
+  functionName: string,
+  columnRef: string | ConditionOperand,
+  mainTableAlias: string
+) {
+  // Create a value extractor function for the column to aggregate
+  const valueExtractor = (nestedRow: Record<string, unknown>) => {
+    let value: unknown
+    if (typeof columnRef === `string` && columnRef.startsWith(`@`)) {
+      value = extractValueFromNestedRow(
+        nestedRow,
+        columnRef.substring(1),
+        mainTableAlias
+      )
+    } else {
+      value = evaluateOperandOnNestedRow(
+        nestedRow,
+        columnRef as ConditionOperand,
+        mainTableAlias
+      )
+    }
+    // Ensure we return a number for aggregate functions
+    return typeof value === `number` ? value : 0
+  }
+
+  // Return the appropriate aggregate function
+  switch (functionName.toUpperCase()) {
+    case `SUM`:
+      return sum(valueExtractor)
+    case `COUNT`:
+      return count() // count() doesn't need a value extractor
+    case `AVG`:
+      return avg(valueExtractor)
+    case `MIN`:
+      return min(valueExtractor)
+    case `MAX`:
+      return max(valueExtractor)
+    case `MEDIAN`:
+      return median(valueExtractor)
+    case `MODE`:
+      return mode(valueExtractor)
+    default:
+      throw new Error(`Unsupported aggregate function: ${functionName}`)
+  }
+}

package/src/query/index.ts

@@ -0,0 +1,5 @@
+export * from "./query-builder.js"
+export * from "./compiled-query.js"
+export * from "./pipeline-compiler.js"
+export * from "./schema.js"
+export * from "./types.js"

package/src/query/joins.ts

@@ -0,0 +1,247 @@
+import {
+  consolidate,
+  filter,
+  join as joinOperator,
+  map,
+} from "@electric-sql/d2ts"
+import { evaluateConditionOnNestedRow } from "./evaluators.js"
+import { extractJoinKey } from "./extractors.js"
+import type { Query } from "./index.js"
+import type { IStreamBuilder, JoinType } from "@electric-sql/d2ts"
+
+/**
+ * Creates a processing pipeline for join clauses
+ */
+export function processJoinClause(
+  pipeline: IStreamBuilder<Record<string, unknown>>,
+  query: Query,
+  tables: Record<string, IStreamBuilder<Record<string, unknown>>>,
+  mainTableAlias: string,
+  allInputs: Record<string, IStreamBuilder<Record<string, unknown>>>
+) {
+  if (!query.join) return pipeline
+  const input = allInputs[query.from]
+
+  for (const joinClause of query.join) {
+    // Create a stream for the joined table
+    const joinedTableAlias = joinClause.as || joinClause.from
+
+    // Get the right join type for the operator
+    const joinType: JoinType =
+      joinClause.type === `cross` ? `inner` : joinClause.type
+
+    // We need to prepare the main pipeline and the joined pipeline
+    // to have the correct key format for joining
+    const mainPipeline = pipeline.pipe(
+      map((nestedRow: Record<string, unknown>) => {
+        // Extract the key from the ON condition left side for the main table
+        const mainRow = nestedRow[mainTableAlias] as Record<string, unknown>
+
+        // Extract the join key from the main row
+        const keyValue = extractJoinKey(
+          mainRow,
+          joinClause.on[0],
+          mainTableAlias
+        )
+
+        // Return [key, nestedRow] as a KeyValue type
+        return [keyValue, nestedRow] as [unknown, Record<string, unknown>]
+      })
+    )
+
+    // Get the joined table input from the inputs map
+    let joinedTableInput: IStreamBuilder<Record<string, unknown>>
+
+    if (allInputs[joinClause.from]) {
+      // Use the provided input if available
+      joinedTableInput = allInputs[joinClause.from]!
+    } else {
+      // Create a new input if not provided
+      joinedTableInput = input!.graph.newInput<Record<string, unknown>>()
+    }
+
+    tables[joinedTableAlias] = joinedTableInput
+
+    // Create a pipeline for the joined table
+    const joinedPipeline = joinedTableInput.pipe(
+      map((row: Record<string, unknown>) => {
+        // Wrap the row in an object with the table alias as the key
+        const nestedRow = { [joinedTableAlias]: row }
+
+        // Extract the key from the ON condition right side for the joined table
+        const keyValue = extractJoinKey(row, joinClause.on[2], joinedTableAlias)
+
+        // Return [key, nestedRow] as a KeyValue type
+        return [keyValue, nestedRow] as [unknown, Record<string, unknown>]
+      })
+    )
+
+    // Apply join with appropriate typings based on join type
+    switch (joinType) {
+      case `inner`:
+        pipeline = mainPipeline.pipe(
+          joinOperator(joinedPipeline, `inner`),
+          consolidate(),
+          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
+        )
+        break
+      case `left`:
+        pipeline = mainPipeline.pipe(
+          joinOperator(joinedPipeline, `left`),
+          consolidate(),
+          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
+        )
+        break
+      case `right`:
+        pipeline = mainPipeline.pipe(
+          joinOperator(joinedPipeline, `right`),
+          consolidate(),
+          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
+        )
+        break
+      case `full`:
+        pipeline = mainPipeline.pipe(
+          joinOperator(joinedPipeline, `full`),
+          consolidate(),
+          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
+        )
+        break
+      default:
+        pipeline = mainPipeline.pipe(
+          joinOperator(joinedPipeline, `inner`),
+          consolidate(),
+          processJoinResults(mainTableAlias, joinedTableAlias, joinClause)
+        )
+    }
+  }
+  return pipeline
+}
+
+/**
+ * Creates a processing pipeline for join results
+ */
+export function processJoinResults(
+  mainTableAlias: string,
+  joinedTableAlias: string,
+  joinClause: { on: any; where?: any; type: string }
+) {
+  return function (
+    pipeline: IStreamBuilder<unknown>
+  ): IStreamBuilder<Record<string, unknown>> {
+    return pipeline.pipe(
+      // Process the join result and handle nulls in the same step
+      map((result: unknown) => {
+        const [_key, [mainNestedRow, joinedNestedRow]] = result as [
+          unknown,
+          [
+            Record<string, unknown> | undefined,
+            Record<string, unknown> | undefined,
+          ],
+        ]
+
+        // For inner joins, both sides should be non-null
+        if (joinClause.type === `inner` || joinClause.type === `cross`) {
+          if (!mainNestedRow || !joinedNestedRow) {
+            return undefined // Will be filtered out
+          }
+        }
+
+        // For left joins, the main row must be non-null
+        if (joinClause.type === `left` && !mainNestedRow) {
+          return undefined // Will be filtered out
+        }
+
+        // For right joins, the joined row must be non-null
+        if (joinClause.type === `right` && !joinedNestedRow) {
+          return undefined // Will be filtered out
+        }
+
+        // Merge the nested rows
+        const mergedNestedRow: Record<string, unknown> = {}
+
+        // Add main row data if it exists
+        if (mainNestedRow) {
+          Object.entries(mainNestedRow).forEach(([tableAlias, tableData]) => {
+            mergedNestedRow[tableAlias] = tableData
+          })
+        }
+
+        // If we have a joined row, add it to the merged result
+        if (joinedNestedRow) {
+          Object.entries(joinedNestedRow).forEach(([tableAlias, tableData]) => {
+            mergedNestedRow[tableAlias] = tableData
+          })
+        } else if (joinClause.type === `left` || joinClause.type === `full`) {
+          // For left or full joins, add the joined table with null data if missing
+          mergedNestedRow[joinedTableAlias] = null
+        }
+
+        // For right or full joins, add the main table with null data if missing
+        if (
+          !mainNestedRow &&
+          (joinClause.type === `right` || joinClause.type === `full`)
+        ) {
+          mergedNestedRow[mainTableAlias] = null
+        }
+
+        return mergedNestedRow
+      }),
+      // Filter out undefined results
+      filter(
+        (value: unknown): value is Record<string, unknown> =>
+          value !== undefined
+      ),
+      // Process the ON condition
+      filter((nestedRow: Record<string, unknown>) => {
+        // If there's no ON condition, or it's a cross join, always return true
+        if (!joinClause.on || joinClause.type === `cross`) {
+          return true
+        }
+
+        // For LEFT JOIN, if the right side is null, we should include the row
+        if (
+          joinClause.type === `left` &&
+          nestedRow[joinedTableAlias] === null
+        ) {
+          return true
+        }
+
+        // For RIGHT JOIN, if the left side is null, we should include the row
+        if (joinClause.type === `right` && nestedRow[mainTableAlias] === null) {
+          return true
+        }
+
+        // For FULL JOIN, if either side is null, we should include the row
+        if (
+          joinClause.type === `full` &&
+          (nestedRow[mainTableAlias] === null ||
+            nestedRow[joinedTableAlias] === null)
+        ) {
+          return true
+        }
+
+        const result = evaluateConditionOnNestedRow(
+          nestedRow,
+          joinClause.on,
+          mainTableAlias,
+          joinedTableAlias
+        )
+        return result
+      }),
+      // Process the WHERE clause for the join if it exists
+      filter((nestedRow: Record<string, unknown>) => {
+        if (!joinClause.where) {
+          return true
+        }
+
+        const result = evaluateConditionOnNestedRow(
+          nestedRow,
+          joinClause.where,
+          mainTableAlias,
+          joinedTableAlias
+        )
+        return result
+      })
+    )
+  }
+}