@tanstack/db 0.0.1

package/src/query/compiled-query.ts

@@ -0,0 +1,193 @@
+import { D2, MessageType, MultiSet, output } from "@electric-sql/d2ts"
+import { Effect, batch } from "@tanstack/store"
+import { Collection } from "../collection.js"
+import { compileQueryPipeline } from "./pipeline-compiler.js"
+import type { ChangeMessage, SyncConfig } from "../types.js"
+import type {
+  IStreamBuilder,
+  MultiSetArray,
+  RootStreamBuilder,
+} from "@electric-sql/d2ts"
+import type { QueryBuilder, ResultsFromContext } from "./query-builder.js"
+import type { Context, Schema } from "./types.js"
+
+export function compileQuery<TContext extends Context<Schema>>(
+  queryBuilder: QueryBuilder<TContext>
+) {
+  return new CompiledQuery<ResultsFromContext<TContext>>(queryBuilder)
+}
+
+export class CompiledQuery<TResults extends object = Record<string, unknown>> {
+  private graph: D2
+  private inputs: Record<string, RootStreamBuilder<any>>
+  private inputCollections: Record<string, Collection<any>>
+  private resultCollection: Collection<TResults>
+  public state: `compiled` | `running` | `stopped` = `compiled`
+  private version = 0
+  private unsubscribeEffect?: () => void
+
+  constructor(queryBuilder: QueryBuilder<Context<Schema>>) {
+    const query = queryBuilder._query
+    const collections = query.collections
+
+    if (!collections) {
+      throw new Error(`No collections provided`)
+    }
+
+    this.inputCollections = collections
+
+    const graph = new D2({ initialFrontier: this.version })
+    const inputs = Object.fromEntries(
+      Object.entries(collections).map(([key]) => [key, graph.newInput<any>()])
+    )
+
+    const sync: SyncConfig<TResults>[`sync`] = ({ begin, write, commit }) => {
+      compileQueryPipeline<IStreamBuilder<[unknown, TResults]>>(
+        query,
+        inputs
+      ).pipe(
+        output(({ type, data }) => {
+          if (type === MessageType.DATA) {
+            begin()
+            data.collection
+              .getInner()
+              .reduce((acc, [[key, value], multiplicity]) => {
+                const changes = acc.get(key) || {
+                  deletes: 0,
+                  inserts: 0,
+                  value,
+                }
+                if (multiplicity < 0) {
+                  changes.deletes += Math.abs(multiplicity)
+                } else if (multiplicity > 0) {
+                  changes.inserts += multiplicity
+                  changes.value = value
+                }
+                acc.set(key, changes)
+                return acc
+              }, new Map<unknown, { deletes: number; inserts: number; value: TResults }>())
+              .forEach((changes, rawKey) => {
+                const key = (rawKey as any).toString()
+                const { deletes, inserts, value } = changes
+                if (inserts && !deletes) {
+                  write({
+                    key,
+                    value: value,
+                    type: `insert`,
+                  })
+                } else if (inserts >= deletes) {
+                  write({
+                    key,
+                    value: value,
+                    type: `update`,
+                  })
+                } else if (deletes > 0) {
+                  write({
+                    key,
+                    value: value,
+                    type: `delete`,
+                  })
+                }
+              })
+            commit()
+          }
+        })
+      )
+      graph.finalize()
+    }
+
+    this.graph = graph
+    this.inputs = inputs
+    this.resultCollection = new Collection<TResults>({
+      id: crypto.randomUUID(), // TODO: remove when we don't require any more
+      sync: {
+        sync,
+      },
+    })
+  }
+
+  get results() {
+    return this.resultCollection
+  }
+
+  private sendChangesToInput(inputKey: string, changes: Array<ChangeMessage>) {
+    const input = this.inputs[inputKey]!
+    const multiSetArray: MultiSetArray<unknown> = []
+    for (const change of changes) {
+      if (change.type === `insert`) {
+        multiSetArray.push([change.value, 1])
+      } else if (change.type === `update`) {
+        multiSetArray.push([change.previousValue, -1])
+        multiSetArray.push([change.value, 1])
+      } else {
+        // change.type === `delete`
+        multiSetArray.push([change.value, -1])
+      }
+    }
+    input.sendData(this.version, new MultiSet(multiSetArray))
+  }
+
+  private sendFrontierToInput(inputKey: string) {
+    const input = this.inputs[inputKey]!
+    input.sendFrontier(this.version)
+  }
+
+  private sendFrontierToAllInputs() {
+    Object.entries(this.inputs).forEach(([key]) => {
+      this.sendFrontierToInput(key)
+    })
+  }
+
+  private incrementVersion() {
+    this.version++
+  }
+
+  private runGraph() {
+    this.graph.run()
+  }
+
+  start() {
+    if (this.state === `running`) {
+      throw new Error(`Query is already running`)
+    } else if (this.state === `stopped`) {
+      throw new Error(`Query is stopped`)
+    }
+
+    batch(() => {
+      Object.entries(this.inputCollections).forEach(([key, collection]) => {
+        this.sendChangesToInput(key, collection.currentStateAsChanges())
+      })
+      this.incrementVersion()
+      this.sendFrontierToAllInputs()
+      this.runGraph()
+    })
+
+    const changeEffect = new Effect({
+      fn: () => {
+        batch(() => {
+          Object.entries(this.inputCollections).forEach(([key, collection]) => {
+            this.sendChangesToInput(key, collection.derivedChanges.state)
+          })
+          this.incrementVersion()
+          this.sendFrontierToAllInputs()
+          this.runGraph()
+        })
+      },
+      deps: Object.values(this.inputCollections).map(
+        (collection) => collection.derivedChanges
+      ),
+    })
+    this.unsubscribeEffect = changeEffect.mount()
+
+    this.state = `running`
+    return () => {
+      this.stop()
+    }
+  }
+
+  stop() {
+    this.unsubscribeEffect?.()
+    this.unsubscribeEffect = undefined
+    this.state = `stopped`
+  }
+}

package/src/query/evaluators.ts

@@ -0,0 +1,222 @@
+import { evaluateOperandOnNestedRow } from "./extractors.js"
+import { compareValues, convertLikeToRegex, isValueInArray } from "./utils.js"
+import type {
+  Comparator,
+  Condition,
+  ConditionOperand,
+  LogicalOperator,
+  SimpleCondition,
+} from "./schema.js"
+
+/**
+ * Evaluates a condition against a nested row structure
+ */
+export function evaluateConditionOnNestedRow(
+  nestedRow: Record<string, unknown>,
+  condition: Condition,
+  mainTableAlias?: string,
+  joinedTableAlias?: string
+): boolean {
+  // Handle simple conditions with exactly 3 elements
+  if (condition.length === 3 && !Array.isArray(condition[0])) {
+    const [left, comparator, right] = condition as SimpleCondition
+    return evaluateSimpleConditionOnNestedRow(
+      nestedRow,
+      left,
+      comparator,
+      right,
+      mainTableAlias,
+      joinedTableAlias
+    )
+  }
+
+  // Handle flat composite conditions (multiple conditions in a single array)
+  if (
+    condition.length > 3 &&
+    !Array.isArray(condition[0]) &&
+    typeof condition[1] === `string` &&
+    ![`and`, `or`].includes(condition[1] as string)
+  ) {
+    // Start with the first condition (first 3 elements)
+    let result = evaluateSimpleConditionOnNestedRow(
+      nestedRow,
+      condition[0],
+      condition[1] as Comparator,
+      condition[2],
+      mainTableAlias,
+      joinedTableAlias
+    )
+
+    // Process the rest in groups: logical operator, then 3 elements for each condition
+    for (let i = 3; i < condition.length; i += 4) {
+      const logicalOp = condition[i] as LogicalOperator
+
+      // Make sure we have a complete condition to evaluate
+      if (i + 3 <= condition.length) {
+        const nextResult = evaluateSimpleConditionOnNestedRow(
+          nestedRow,
+          condition[i + 1],
+          condition[i + 2] as Comparator,
+          condition[i + 3],
+          mainTableAlias,
+          joinedTableAlias
+        )
+
+        // Apply the logical operator
+        if (logicalOp === `and`) {
+          result = result && nextResult
+        } else {
+          // logicalOp === `or`
+          result = result || nextResult
+        }
+      }
+    }
+
+    return result
+  }
+
+  // Handle nested composite conditions where the first element is an array
+  if (condition.length > 0 && Array.isArray(condition[0])) {
+    // Start with the first condition
+    let result = evaluateConditionOnNestedRow(
+      nestedRow,
+      condition[0] as Condition,
+      mainTableAlias,
+      joinedTableAlias
+    )
+
+    // Process the rest of the conditions and logical operators in pairs
+    for (let i = 1; i < condition.length; i += 2) {
+      if (i + 1 >= condition.length) break // Make sure we have a pair
+
+      const operator = condition[i] as LogicalOperator
+      const nextCondition = condition[i + 1] as Condition
+
+      // Apply the logical operator
+      if (operator === `and`) {
+        result =
+          result &&
+          evaluateConditionOnNestedRow(
+            nestedRow,
+            nextCondition,
+            mainTableAlias,
+            joinedTableAlias
+          )
+      } else {
+        // logicalOp === `or`
+        result =
+          result ||
+          evaluateConditionOnNestedRow(
+            nestedRow,
+            nextCondition,
+            mainTableAlias,
+            joinedTableAlias
+          )
+      }
+    }
+
+    return result
+  }
+
+  // Fallback - this should not happen with valid conditions
+  return true
+}
+
+/**
+ * Evaluates a simple condition against a nested row structure
+ */
+export function evaluateSimpleConditionOnNestedRow(
+  nestedRow: Record<string, unknown>,
+  left: ConditionOperand,
+  comparator: Comparator,
+  right: ConditionOperand,
+  mainTableAlias?: string,
+  joinedTableAlias?: string
+): boolean {
+  const leftValue = evaluateOperandOnNestedRow(
+    nestedRow,
+    left,
+    mainTableAlias,
+    joinedTableAlias
+  )
+
+  const rightValue = evaluateOperandOnNestedRow(
+    nestedRow,
+    right,
+    mainTableAlias,
+    joinedTableAlias
+  )
+
+  // The rest of the function remains the same as evaluateSimpleCondition
+  switch (comparator) {
+    case `=`:
+      return leftValue === rightValue
+    case `!=`:
+      return leftValue !== rightValue
+    case `<`:
+      return compareValues(leftValue, rightValue, `<`)
+    case `<=`:
+      return compareValues(leftValue, rightValue, `<=`)
+    case `>`:
+      return compareValues(leftValue, rightValue, `>`)
+    case `>=`:
+      return compareValues(leftValue, rightValue, `>=`)
+    case `like`:
+    case `not like`:
+      if (typeof leftValue === `string` && typeof rightValue === `string`) {
+        // Convert SQL LIKE pattern to proper regex pattern
+        const pattern = convertLikeToRegex(rightValue)
+        const matches = new RegExp(`^${pattern}$`, `i`).test(leftValue)
+        return comparator === `like` ? matches : !matches
+      }
+      return comparator === `like` ? false : true
+    case `in`:
+      // If right value is not an array, we can't do an IN operation
+      if (!Array.isArray(rightValue)) {
+        return false
+      }
+
+      // For empty arrays, nothing is contained in them
+      if (rightValue.length === 0) {
+        return false
+      }
+
+      // Handle array-to-array comparison (check if any element in leftValue exists in rightValue)
+      if (Array.isArray(leftValue)) {
+        return leftValue.some((item) => isValueInArray(item, rightValue))
+      }
+
+      // Handle single value comparison
+      return isValueInArray(leftValue, rightValue)
+
+    case `not in`:
+      // If right value is not an array, everything is "not in" it
+      if (!Array.isArray(rightValue)) {
+        return true
+      }
+
+      // For empty arrays, everything is "not in" them
+      if (rightValue.length === 0) {
+        return true
+      }
+
+      // Handle array-to-array comparison (check if no element in leftValue exists in rightValue)
+      if (Array.isArray(leftValue)) {
+        return !leftValue.some((item) => isValueInArray(item, rightValue))
+      }
+
+      // Handle single value comparison
+      return !isValueInArray(leftValue, rightValue)
+
+    case `is`:
+      return leftValue === rightValue
+    case `is not`:
+      // Properly handle null/undefined checks
+      if (rightValue === null) {
+        return leftValue !== null && leftValue !== undefined
+      }
+      return leftValue !== rightValue
+    default:
+      return false
+  }
+}

package/src/query/extractors.ts

@@ -0,0 +1,211 @@
+import { evaluateFunction, isFunctionCall } from "./functions.js"
+import type { AllowedFunctionName, ConditionOperand } from "./schema.js"
+
+/**
+ * Extracts a value from a nested row structure
+ * @param nestedRow The nested row structure
+ * @param columnRef The column reference (may include table.column format)
+ * @param mainTableAlias The main table alias to check first for columns without table reference
+ * @param joinedTableAlias The joined table alias to check second for columns without table reference
+ * @returns The extracted value or undefined if not found
+ */
+export function extractValueFromNestedRow(
+  nestedRow: Record<string, unknown>,
+  columnRef: string,
+  mainTableAlias?: string,
+  joinedTableAlias?: string
+): unknown {
+  // Check if it's a table.column reference
+  if (columnRef.includes(`.`)) {
+    const [tableAlias, colName] = columnRef.split(`.`) as [string, string]
+
+    // Get the table data
+    const tableData = nestedRow[tableAlias] as
+      | Record<string, unknown>
+      | null
+      | undefined
+
+    if (!tableData) {
+      return null
+    }
+
+    // Return the column value from that table
+    const value = tableData[colName]
+    return value
+  } else {
+    // If no table is specified, first try to find in the main table if provided
+    if (mainTableAlias && nestedRow[mainTableAlias]) {
+      const mainTableData = nestedRow[mainTableAlias] as Record<string, unknown>
+      if (typeof mainTableData === `object` && columnRef in mainTableData) {
+        return mainTableData[columnRef]
+      }
+    }
+
+    // Then try the joined table if provided
+    if (joinedTableAlias && nestedRow[joinedTableAlias]) {
+      const joinedTableData = nestedRow[joinedTableAlias] as Record<
+        string,
+        unknown
+      >
+      if (typeof joinedTableData === `object` && columnRef in joinedTableData) {
+        return joinedTableData[columnRef]
+      }
+    }
+
+    // If not found in main or joined table, try to find the column in any table
+    for (const [_tableAlias, tableData] of Object.entries(nestedRow)) {
+      if (
+        tableData &&
+        typeof tableData === `object` &&
+        columnRef in (tableData as Record<string, unknown>)
+      ) {
+        return (tableData as Record<string, unknown>)[columnRef]
+      }
+    }
+    return undefined
+  }
+}
+
+/**
+ * Evaluates an operand against a nested row structure
+ */
+export function evaluateOperandOnNestedRow(
+  nestedRow: Record<string, unknown>,
+  operand: ConditionOperand,
+  mainTableAlias?: string,
+  joinedTableAlias?: string
+): unknown {
+  // Handle column references
+  if (typeof operand === `string` && operand.startsWith(`@`)) {
+    const columnRef = operand.substring(1)
+    return extractValueFromNestedRow(
+      nestedRow,
+      columnRef,
+      mainTableAlias,
+      joinedTableAlias
+    )
+  }
+
+  // Handle explicit column references
+  if (operand && typeof operand === `object` && `col` in operand) {
+    const colRef = (operand as { col: unknown }).col
+
+    if (typeof colRef === `string`) {
+      // First try to extract from nested row structure
+      const nestedValue = extractValueFromNestedRow(
+        nestedRow,
+        colRef,
+        mainTableAlias,
+        joinedTableAlias
+      )
+
+      // If not found in nested structure, check if it's a direct property of the row
+      // This is important for HAVING clauses that reference aggregated values
+      if (nestedValue === undefined && colRef in nestedRow) {
+        return nestedRow[colRef]
+      }
+
+      return nestedValue
+    }
+
+    return undefined
+  }
+
+  // Handle function calls
+  if (operand && typeof operand === `object` && isFunctionCall(operand)) {
+    // Get the function name (the only key in the object)
+    const functionName = Object.keys(operand)[0] as AllowedFunctionName
+    // Get the arguments using type assertion with specific function name
+    const args = (operand as any)[functionName]
+
+    // If the arguments are a reference or another expression, evaluate them first
+    const evaluatedArgs = Array.isArray(args)
+      ? args.map((arg) =>
+          evaluateOperandOnNestedRow(
+            nestedRow,
+            arg as ConditionOperand,
+            mainTableAlias,
+            joinedTableAlias
+          )
+        )
+      : evaluateOperandOnNestedRow(
+          nestedRow,
+          args as ConditionOperand,
+          mainTableAlias,
+          joinedTableAlias
+        )
+
+    // Call the function with the evaluated arguments
+    return evaluateFunction(
+      functionName,
+      evaluatedArgs as ConditionOperand | Array<ConditionOperand>
+    )
+  }
+
+  // Handle explicit literals
+  if (operand && typeof operand === `object` && `value` in operand) {
+    return (operand as { value: unknown }).value
+  }
+
+  // Handle literal values
+  return operand
+}
+
+/**
+ * Extracts a join key value from a row based on the operand
+ * @param row The data row (not nested)
+ * @param operand The operand to extract the key from
+ * @param defaultTableAlias The default table alias
+ * @returns The extracted key value
+ */
+export function extractJoinKey<T extends Record<string, unknown>>(
+  row: T,
+  operand: ConditionOperand,
+  defaultTableAlias?: string
+): unknown {
+  let keyValue: unknown
+
+  // Handle column references (e.g., "@orders.id" or "@id")
+  if (typeof operand === `string` && operand.startsWith(`@`)) {
+    const columnRef = operand.substring(1)
+
+    // If it contains a dot, extract the table and column
+    if (columnRef.includes(`.`)) {
+      const [tableAlias, colName] = columnRef.split(`.`) as [string, string]
+      // If this is referencing the current table, extract from row directly
+      if (tableAlias === defaultTableAlias) {
+        keyValue = row[colName]
+      } else {
+        // This might be a column from another table, return undefined
+        keyValue = undefined
+      }
+    } else {
+      // No table specified, look directly in the row
+      keyValue = row[columnRef]
+    }
+  } else if (operand && typeof operand === `object` && `col` in operand) {
+    // Handle explicit column references like { col: "orders.id" } or { col: "id" }
+    const colRef = (operand as { col: unknown }).col
+
+    if (typeof colRef === `string`) {
+      if (colRef.includes(`.`)) {
+        const [tableAlias, colName] = colRef.split(`.`) as [string, string]
+        // If this is referencing the current table, extract from row directly
+        if (tableAlias === defaultTableAlias) {
+          keyValue = row[colName]
+        } else {
+          // This might be a column from another table, return undefined
+          keyValue = undefined
+        }
+      } else {
+        // No table specified, look directly in the row
+        keyValue = row[colRef]
+      }
+    }
+  } else {
+    // Handle literals or other types
+    keyValue = operand
+  }
+
+  return keyValue
+}