@tanstack/db 0.0.4 → 0.0.5

package/src/query/joins.ts

@@ -4,20 +4,25 @@ import {
   join as joinOperator,
   map,
 } from "@electric-sql/d2ts"
-import { evaluateConditionOnNestedRow } from "./evaluators.js"
+import { evaluateConditionOnNamespacedRow } from "./evaluators.js"
 import { extractJoinKey } from "./extractors.js"
 import type { Query } from "./index.js"
 import type { IStreamBuilder, JoinType } from "@electric-sql/d2ts"
+import type {
+  KeyedStream,
+  NamespacedAndKeyedStream,
+  NamespacedRow,
+} from "../types.js"
 
 /**
  * Creates a processing pipeline for join clauses
  */
 export function processJoinClause(
-  pipeline: IStreamBuilder<Record<string, unknown>>,
+  pipeline: NamespacedAndKeyedStream,
   query: Query,
-  tables: Record<string, IStreamBuilder<Record<string, unknown>>>,
+  tables: Record<string, KeyedStream>,
   mainTableAlias: string,
-  allInputs: Record<string, IStreamBuilder<Record<string, unknown>>>
+  allInputs: Record<string, KeyedStream>
 ) {
   if (!query.join) return pipeline
   const input = allInputs[query.from]
@@ -30,49 +35,56 @@ export function processJoinClause(
     const joinType: JoinType =
       joinClause.type === `cross` ? `inner` : joinClause.type
 
+    // The `in` is formatted as ['@mainKeyRef', '=', '@joinedKeyRef']
+    // Destructure the main key reference and the joined key references
+    const [mainKeyRef, , joinedKeyRefs] = joinClause.on
+
     // We need to prepare the main pipeline and the joined pipeline
     // to have the correct key format for joining
     const mainPipeline = pipeline.pipe(
-      map((nestedRow: Record<string, unknown>) => {
+      map(([currentKey, namespacedRow]) => {
         // Extract the key from the ON condition left side for the main table
-        const mainRow = nestedRow[mainTableAlias] as Record<string, unknown>
+        const mainRow = namespacedRow[mainTableAlias]!
 
         // Extract the join key from the main row
-        const keyValue = extractJoinKey(
-          mainRow,
-          joinClause.on[0],
-          mainTableAlias
-        )
+        const key = extractJoinKey(mainRow, mainKeyRef, mainTableAlias)
 
-        // Return [key, nestedRow] as a KeyValue type
-        return [keyValue, nestedRow] as [unknown, Record<string, unknown>]
+        // Return [key, namespacedRow] as a KeyValue type
+        return [key, [currentKey, namespacedRow]] as [
+          unknown,
+          [string, typeof namespacedRow],
+        ]
       })
     )
 
     // Get the joined table input from the inputs map
-    let joinedTableInput: IStreamBuilder<Record<string, unknown>>
+    let joinedTableInput: KeyedStream
 
     if (allInputs[joinClause.from]) {
       // Use the provided input if available
       joinedTableInput = allInputs[joinClause.from]!
     } else {
       // Create a new input if not provided
-      joinedTableInput = input!.graph.newInput<Record<string, unknown>>()
+      joinedTableInput =
+        input!.graph.newInput<[string, Record<string, unknown>]>()
     }
 
     tables[joinedTableAlias] = joinedTableInput
 
     // Create a pipeline for the joined table
     const joinedPipeline = joinedTableInput.pipe(
-      map((row: Record<string, unknown>) => {
+      map(([currentKey, row]) => {
         // Wrap the row in an object with the table alias as the key
-        const nestedRow = { [joinedTableAlias]: row }
+        const namespacedRow: NamespacedRow = { [joinedTableAlias]: row }
 
         // Extract the key from the ON condition right side for the joined table
-        const keyValue = extractJoinKey(row, joinClause.on[2], joinedTableAlias)
+        const key = extractJoinKey(row, joinedKeyRefs, joinedTableAlias)
 
-        // Return [key, nestedRow] as a KeyValue type
-        return [keyValue, nestedRow] as [unknown, Record<string, unknown>]
+        // Return [key, namespacedRow] as a KeyValue type
+        return [key, [currentKey, namespacedRow]] as [
+          string,
+          [string, typeof namespacedRow],
+        ]
       })
     )
 
@@ -123,76 +135,89 @@ export function processJoinClause(
 export function processJoinResults(
   mainTableAlias: string,
   joinedTableAlias: string,
-  joinClause: { on: any; where?: any; type: string }
+  joinClause: { on: any; type: string }
 ) {
   return function (
-    pipeline: IStreamBuilder<unknown>
-  ): IStreamBuilder<Record<string, unknown>> {
+    pipeline: IStreamBuilder<
+      [
+        key: string,
+        [
+          [string, NamespacedRow] | undefined,
+          [string, NamespacedRow] | undefined,
+        ],
+      ]
+    >
+  ): NamespacedAndKeyedStream {
     return pipeline.pipe(
       // Process the join result and handle nulls in the same step
-      map((result: unknown) => {
-        const [_key, [mainNestedRow, joinedNestedRow]] = result as [
-          unknown,
-          [
-            Record<string, unknown> | undefined,
-            Record<string, unknown> | undefined,
-          ],
-        ]
+      map((result) => {
+        const [_key, [main, joined]] = result
+        const mainKey = main?.[0]
+        const mainNamespacedRow = main?.[1]
+        const joinedKey = joined?.[0]
+        const joinedNamespacedRow = joined?.[1]
 
         // For inner joins, both sides should be non-null
         if (joinClause.type === `inner` || joinClause.type === `cross`) {
-          if (!mainNestedRow || !joinedNestedRow) {
+          if (!mainNamespacedRow || !joinedNamespacedRow) {
             return undefined // Will be filtered out
           }
         }
 
         // For left joins, the main row must be non-null
-        if (joinClause.type === `left` && !mainNestedRow) {
+        if (joinClause.type === `left` && !mainNamespacedRow) {
           return undefined // Will be filtered out
         }
 
         // For right joins, the joined row must be non-null
-        if (joinClause.type === `right` && !joinedNestedRow) {
+        if (joinClause.type === `right` && !joinedNamespacedRow) {
           return undefined // Will be filtered out
         }
 
         // Merge the nested rows
-        const mergedNestedRow: Record<string, unknown> = {}
+        const mergedNamespacedRow: NamespacedRow = {}
 
         // Add main row data if it exists
-        if (mainNestedRow) {
-          Object.entries(mainNestedRow).forEach(([tableAlias, tableData]) => {
-            mergedNestedRow[tableAlias] = tableData
-          })
+        if (mainNamespacedRow) {
+          Object.entries(mainNamespacedRow).forEach(
+            ([tableAlias, tableData]) => {
+              mergedNamespacedRow[tableAlias] = tableData
+            }
+          )
         }
 
         // If we have a joined row, add it to the merged result
-        if (joinedNestedRow) {
-          Object.entries(joinedNestedRow).forEach(([tableAlias, tableData]) => {
-            mergedNestedRow[tableAlias] = tableData
-          })
+        if (joinedNamespacedRow) {
+          Object.entries(joinedNamespacedRow).forEach(
+            ([tableAlias, tableData]) => {
+              mergedNamespacedRow[tableAlias] = tableData
+            }
+          )
         } else if (joinClause.type === `left` || joinClause.type === `full`) {
-          // For left or full joins, add the joined table with null data if missing
-          mergedNestedRow[joinedTableAlias] = null
+          // For left or full joins, add the joined table with undefined data if missing
+          // mergedNamespacedRow[joinedTableAlias] = undefined
         }
 
-        // For right or full joins, add the main table with null data if missing
+        // For right or full joins, add the main table with undefined data if missing
         if (
-          !mainNestedRow &&
+          !mainNamespacedRow &&
           (joinClause.type === `right` || joinClause.type === `full`)
         ) {
-          mergedNestedRow[mainTableAlias] = null
+          // mergedNamespacedRow[mainTableAlias] = undefined
         }
 
-        return mergedNestedRow
+        // New key
+        const newKey = `[${mainKey},${joinedKey}]`
+
+        return [newKey, mergedNamespacedRow] as [
+          string,
+          typeof mergedNamespacedRow,
+        ]
       }),
       // Filter out undefined results
-      filter(
-        (value: unknown): value is Record<string, unknown> =>
-          value !== undefined
-      ),
+      filter((value) => value !== undefined),
       // Process the ON condition
-      filter((nestedRow: Record<string, unknown>) => {
+      filter(([_key, namespacedRow]: [string, NamespacedRow]) => {
         // If there's no ON condition, or it's a cross join, always return true
         if (!joinClause.on || joinClause.type === `cross`) {
           return true
@@ -201,46 +226,34 @@ export function processJoinResults(
         // For LEFT JOIN, if the right side is null, we should include the row
         if (
           joinClause.type === `left` &&
-          nestedRow[joinedTableAlias] === null
+          namespacedRow[joinedTableAlias] === undefined
         ) {
           return true
         }
 
         // For RIGHT JOIN, if the left side is null, we should include the row
-        if (joinClause.type === `right` && nestedRow[mainTableAlias] === null) {
+        if (
+          joinClause.type === `right` &&
+          namespacedRow[mainTableAlias] === undefined
+        ) {
           return true
         }
 
         // For FULL JOIN, if either side is null, we should include the row
         if (
           joinClause.type === `full` &&
-          (nestedRow[mainTableAlias] === null ||
-            nestedRow[joinedTableAlias] === null)
+          (namespacedRow[mainTableAlias] === undefined ||
+            namespacedRow[joinedTableAlias] === undefined)
         ) {
           return true
         }
 
-        const result = evaluateConditionOnNestedRow(
-          nestedRow,
+        return evaluateConditionOnNamespacedRow(
+          namespacedRow,
           joinClause.on,
           mainTableAlias,
           joinedTableAlias
         )
-        return result
-      }),
-      // Process the WHERE clause for the join if it exists
-      filter((nestedRow: Record<string, unknown>) => {
-        if (!joinClause.where) {
-          return true
-        }
-
-        const result = evaluateConditionOnNestedRow(
-          nestedRow,
-          joinClause.where,
-          mainTableAlias,
-          joinedTableAlias
-        )
-        return result
       })
     )
   }

package/src/query/order-by.ts

@@ -3,19 +3,18 @@ import {
   orderBy,
   orderByWithFractionalIndex,
   orderByWithIndex,
-  topK,
-  topKWithFractionalIndex,
-  topKWithIndex,
 } from "@electric-sql/d2ts"
-import { evaluateOperandOnNestedRow } from "./extractors"
+import { evaluateOperandOnNamespacedRow } from "./extractors"
 import { isOrderIndexFunctionCall } from "./utils"
 import type { ConditionOperand, Query } from "./schema"
-import type { IStreamBuilder } from "@electric-sql/d2ts"
+import type {
+  KeyedNamespacedRow,
+  NamespacedAndKeyedStream,
+  NamespacedRow,
+} from "../types"
 
 export function processOrderBy(
-  resultPipeline: IStreamBuilder<
-    Record<string, unknown> | [string | number, Record<string, unknown>]
-  >,
+  resultPipeline: NamespacedAndKeyedStream,
   query: Query,
   mainTableAlias: string
 ) {
@@ -25,7 +24,9 @@ export function processOrderBy(
   let orderIndexAlias = ``
 
   // Scan the SELECT clause for ORDER_INDEX functions
-  for (const item of query.select) {
+  // TODO: Select is going to be optional in future - we will automatically add an
+  // attribute for the index column
+  for (const item of query.select!) {
     if (typeof item === `object`) {
       for (const [alias, expr] of Object.entries(item)) {
         if (typeof expr === `object` && isOrderIndexFunctionCall(expr)) {
@@ -79,17 +80,13 @@ export function processOrderBy(
   }
 
   // Create a value extractor function for the orderBy operator
-  const valueExtractor = (value: unknown) => {
-    const row = value as Record<string, unknown>
-
-    // Create a nested row structure for evaluateOperandOnNestedRow
-    const nestedRow: Record<string, unknown> = { [mainTableAlias]: row }
-
+  // const valueExtractor = ([key, namespacedRow]: [
+  const valueExtractor = (namespacedRow: NamespacedRow) => {
     // For multiple orderBy columns, create a composite key
     if (orderByItems.length > 1) {
       return orderByItems.map((item) => {
-        const val = evaluateOperandOnNestedRow(
-          nestedRow,
+        const val = evaluateOperandOnNamespacedRow(
+          namespacedRow,
           item.operand,
           mainTableAlias
         )
@@ -106,8 +103,8 @@ export function processOrderBy(
     } else if (orderByItems.length === 1) {
       // For a single orderBy column, use the value directly
       const item = orderByItems[0]
-      const val = evaluateOperandOnNestedRow(
-        nestedRow,
+      const val = evaluateOperandOnNamespacedRow(
+        namespacedRow,
         item!.operand,
         mainTableAlias
       )
@@ -187,109 +184,62 @@ export function processOrderBy(
     return (a as any).toString().localeCompare((b as any).toString())
   }
 
-  let topKComparator: (a: unknown, b: unknown) => number
-  if (!query.keyBy) {
-    topKComparator = (a, b) => {
-      const aValue = valueExtractor(a)
-      const bValue = valueExtractor(b)
-      return comparator(aValue, bValue)
-    }
-  }
-
   // Apply the appropriate orderBy operator based on whether an ORDER_INDEX column is requested
   if (hasOrderIndexColumn) {
     if (orderIndexType === `numeric`) {
-      if (query.keyBy) {
-        // Use orderByWithIndex for numeric indices
-        resultPipeline = resultPipeline.pipe(
-          orderByWithIndex(valueExtractor, {
-            limit: query.limit,
-            offset: query.offset,
-            comparator,
-          }),
-          map(([key, [value, index]]) => {
-            // Add the index to the result
-            const result = {
-              ...(value as Record<string, unknown>),
-              [orderIndexAlias]: index,
-            }
-            return [key, result]
-          })
-        )
-      } else {
-        // Use topKWithIndex for numeric indices
-        resultPipeline = resultPipeline.pipe(
-          map((value) => [null, value]),
-          topKWithIndex(topKComparator!, {
-            limit: query.limit,
-            offset: query.offset,
-          }),
-          map(([_, [value, index]]) => {
-            // Add the index to the result
-            return {
-              ...(value as Record<string, unknown>),
-              [orderIndexAlias]: index,
-            }
-          })
-        )
-      }
-    } else {
-      if (query.keyBy) {
-        // Use orderByWithFractionalIndex for fractional indices
-        resultPipeline = resultPipeline.pipe(
-          orderByWithFractionalIndex(valueExtractor, {
-            limit: query.limit,
-            offset: query.offset,
-            comparator,
-          }),
-          map(([key, [value, index]]) => {
-            // Add the index to the result
-            const result = {
-              ...(value as Record<string, unknown>),
-              [orderIndexAlias]: index,
-            }
-            return [key, result]
-          })
-        )
-      } else {
-        // Use topKWithFractionalIndex for fractional indices
-        resultPipeline = resultPipeline.pipe(
-          map((value) => [null, value]),
-          topKWithFractionalIndex(topKComparator!, {
-            limit: query.limit,
-            offset: query.offset,
-          }),
-          map(([_, [value, index]]) => {
-            // Add the index to the result
-            return {
-              ...(value as Record<string, unknown>),
-              [orderIndexAlias]: index,
-            }
-          })
-        )
-      }
-    }
-  } else {
-    if (query.keyBy) {
-      // Use regular orderBy if no index column is requested and but a keyBy is requested
+      // Use orderByWithIndex for numeric indices
       resultPipeline = resultPipeline.pipe(
-        orderBy(valueExtractor, {
+        orderByWithIndex(valueExtractor, {
           limit: query.limit,
           offset: query.offset,
           comparator,
+        }),
+        map(([key, [value, index]]) => {
+          // Add the index to the result
+          // We add this to the main table alias for now
+          // TODO: re are going to need to refactor the whole order by pipeline
+          const result = {
+            ...(value as Record<string, unknown>),
+            [mainTableAlias]: {
+              ...value[mainTableAlias],
+              [orderIndexAlias]: index,
+            },
+          }
+          return [key, result] as KeyedNamespacedRow
         })
       )
     } else {
-      // Use topK if no index column is requested and no keyBy is requested
+      // Use orderByWithFractionalIndex for fractional indices
       resultPipeline = resultPipeline.pipe(
-        map((value) => [null, value]),
-        topK(topKComparator!, {
+        orderByWithFractionalIndex(valueExtractor, {
           limit: query.limit,
           offset: query.offset,
+          comparator,
         }),
-        map(([_, value]) => value as Record<string, unknown>)
+        map(([key, [value, index]]) => {
+          // Add the index to the result
+          // We add this to the main table alias for now
+          // TODO: re are going to need to refactor the whole order by pipeline
+          const result = {
+            ...(value as Record<string, unknown>),
+            [mainTableAlias]: {
+              ...value[mainTableAlias],
+              [orderIndexAlias]: index,
+            },
+          }
+          return [key, result] as KeyedNamespacedRow
+        })
       )
     }
+  } else {
+    // Use regular orderBy if no index column is requested
+    resultPipeline = resultPipeline.pipe(
+      orderBy(valueExtractor, {
+        limit: query.limit,
+        offset: query.offset,
+        comparator,
+      })
+    )
   }
 
   return resultPipeline

package/src/query/pipeline-compiler.ts

@@ -1,12 +1,16 @@
 import { filter, map } from "@electric-sql/d2ts"
-import { evaluateConditionOnNestedRow } from "./evaluators.js"
+import { evaluateConditionOnNamespacedRow } from "./evaluators.js"
 import { processJoinClause } from "./joins.js"
 import { processGroupBy } from "./group-by.js"
 import { processOrderBy } from "./order-by.js"
-import { processKeyBy } from "./key-by.js"
 import { processSelect } from "./select.js"
-import type { Condition, Query } from "./schema.js"
+import type { Query } from "./schema.js"
 import type { IStreamBuilder } from "@electric-sql/d2ts"
+import type {
+  InputRow,
+  KeyedStream,
+  NamespacedAndKeyedStream,
+} from "../types.js"
 
 /**
  * Compiles a query into a D2 pipeline
@@ -16,7 +20,7 @@ import type { IStreamBuilder } from "@electric-sql/d2ts"
  */
 export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
   query: Query,
-  inputs: Record<string, IStreamBuilder<Record<string, unknown>>>
+  inputs: Record<string, KeyedStream>
 ): T {
   // Create a copy of the inputs map to avoid modifying the original
   const allInputs = { ...inputs }
@@ -30,11 +34,6 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
         throw new Error(`WITH query must have an "as" property`)
       }
 
-      // Ensure the WITH query is not keyed
-      if ((withQuery as Query).keyBy !== undefined) {
-        throw new Error(`WITH query cannot have a "keyBy" property`)
-      }
-
       // Check if this CTE name already exists in the inputs
       if (allInputs[withQuery.as]) {
         throw new Error(`CTE with name "${withQuery.as}" already exists`)
@@ -51,14 +50,12 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
       )
 
       // Add the compiled query to the inputs map using its alias
-      allInputs[withQuery.as] = compiledWithQuery as IStreamBuilder<
-        Record<string, unknown>
-      >
+      allInputs[withQuery.as] = compiledWithQuery as KeyedStream
     }
   }
 
   // Create a map of table aliases to inputs
-  const tables: Record<string, IStreamBuilder<Record<string, unknown>>> = {}
+  const tables: Record<string, KeyedStream> = {}
 
   // The main table is the one in the FROM clause
   const mainTableAlias = query.as || query.from
@@ -72,10 +69,14 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
   tables[mainTableAlias] = input
 
   // Prepare the initial pipeline with the main table wrapped in its alias
-  let pipeline = input.pipe(
-    map((row: unknown) => {
+  let pipeline: NamespacedAndKeyedStream = input.pipe(
+    map(([key, row]) => {
       // Initialize the record with a nested structure
-      return { [mainTableAlias]: row } as Record<string, unknown>
+      const ret = [key, { [mainTableAlias]: row }] as [
+        string,
+        Record<string, typeof row>,
+      ]
+      return ret
     })
   )
 
@@ -93,10 +94,10 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
   // Process the WHERE clause if it exists
   if (query.where) {
     pipeline = pipeline.pipe(
-      filter((nestedRow) => {
-        const result = evaluateConditionOnNestedRow(
-          nestedRow,
-          query.where as Condition,
+      filter(([_key, row]) => {
+        const result = evaluateConditionOnNamespacedRow(
+          row,
+          query.where!,
           mainTableAlias
         )
         return result
@@ -113,12 +114,12 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
   // This works similarly to WHERE but is applied after any aggregations
   if (query.having) {
     pipeline = pipeline.pipe(
-      filter((row) => {
+      filter(([_key, row]) => {
         // For HAVING, we're working with the flattened row that contains both
         // the group by keys and the aggregate results directly
-        const result = evaluateConditionOnNestedRow(
-          { [mainTableAlias]: row, ...row } as Record<string, unknown>,
-          query.having as Condition,
+        const result = evaluateConditionOnNamespacedRow(
+          row,
+          query.having!,
           mainTableAlias
         )
         return result
@@ -126,21 +127,9 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
     )
   }
 
-  // Process the SELECT clause - this is where we flatten the structure
-  pipeline = processSelect(pipeline, query, mainTableAlias, allInputs)
-
-  let resultPipeline: IStreamBuilder<
-    Record<string, unknown> | [string | number, Record<string, unknown>]
-  > = pipeline
-
-  // Process keyBy parameter if it exists
-  if (query.keyBy) {
-    resultPipeline = processKeyBy(resultPipeline, query)
-  }
-
   // Process orderBy parameter if it exists
   if (query.orderBy) {
-    resultPipeline = processOrderBy(resultPipeline, query, mainTableAlias)
+    pipeline = processOrderBy(pipeline, query, mainTableAlias)
   } else if (query.limit !== undefined || query.offset !== undefined) {
     // If there's a limit or offset without orderBy, throw an error
     throw new Error(
@@ -148,5 +137,13 @@ export function compileQueryPipeline<T extends IStreamBuilder<unknown>>(
     )
   }
 
+  // Process the SELECT clause - this is where we flatten the structure
+  const resultPipeline: KeyedStream | NamespacedAndKeyedStream = query.select
+    ? processSelect(pipeline, query, mainTableAlias, allInputs)
+    : !query.join && !query.groupBy
+      ? pipeline.pipe(
+          map(([key, row]) => [key, row[mainTableAlias]] as InputRow)
+        )
+      : pipeline
   return resultPipeline as T
 }