@tanstack/db 0.0.4 → 0.0.5

package/src/query/compiled-query.ts

@@ -14,7 +14,9 @@ import type { Context, Schema } from "./types.js"
 export function compileQuery<TContext extends Context<Schema>>(
   queryBuilder: QueryBuilder<TContext>
 ) {
-  return new CompiledQuery<ResultsFromContext<TContext>>(queryBuilder)
+  return new CompiledQuery<
+    ResultsFromContext<TContext> & { _key?: string | number }
+  >(queryBuilder)
 }
 
 export class CompiledQuery<TResults extends object = Record<string, unknown>> {
@@ -67,24 +69,21 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
                 return acc
               }, new Map<unknown, { deletes: number; inserts: number; value: TResults }>())
               .forEach((changes, rawKey) => {
-                const key = (rawKey as any).toString()
                 const { deletes, inserts, value } = changes
+                const valueWithKey = { ...value, _key: rawKey }
                 if (inserts && !deletes) {
                   write({
-                    key,
-                    value: value,
+                    value: valueWithKey,
                     type: `insert`,
                   })
                 } else if (inserts >= deletes) {
                   write({
-                    key,
-                    value: value,
+                    value: valueWithKey,
                     type: `update`,
                   })
                 } else if (deletes > 0) {
                   write({
-                    key,
-                    value: value,
+                    value: valueWithKey,
                     type: `delete`,
                   })
                 }
@@ -100,6 +99,9 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
     this.inputs = inputs
     this.resultCollection = new Collection<TResults>({
       id: crypto.randomUUID(), // TODO: remove when we don't require any more
+      getId: (val) => {
+        return (val as any)._key
+      },
       sync: {
         sync,
       },
@@ -110,18 +112,23 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
     return this.resultCollection
   }
 
-  private sendChangesToInput(inputKey: string, changes: Array<ChangeMessage>) {
+  private sendChangesToInput(
+    inputKey: string,
+    changes: Array<ChangeMessage>,
+    getId: (item: ChangeMessage[`value`]) => any
+  ) {
     const input = this.inputs[inputKey]!
     const multiSetArray: MultiSetArray<unknown> = []
     for (const change of changes) {
+      const key = getId(change.value)
       if (change.type === `insert`) {
-        multiSetArray.push([change.value, 1])
+        multiSetArray.push([[key, change.value], 1])
       } else if (change.type === `update`) {
-        multiSetArray.push([change.previousValue, -1])
-        multiSetArray.push([change.value, 1])
+        multiSetArray.push([[key, change.previousValue], -1])
+        multiSetArray.push([[key, change.value], 1])
       } else {
         // change.type === `delete`
-        multiSetArray.push([change.value, -1])
+        multiSetArray.push([[key, change.value], -1])
       }
     }
     input.sendData(this.version, new MultiSet(multiSetArray))
@@ -155,7 +162,11 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
 
     batch(() => {
       Object.entries(this.inputCollections).forEach(([key, collection]) => {
-        this.sendChangesToInput(key, collection.currentStateAsChanges())
+        this.sendChangesToInput(
+          key,
+          collection.currentStateAsChanges(),
+          collection.config.getId
+        )
       })
       this.incrementVersion()
       this.sendFrontierToAllInputs()
@@ -166,7 +177,11 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
       fn: () => {
         batch(() => {
           Object.entries(this.inputCollections).forEach(([key, collection]) => {
-            this.sendChangesToInput(key, collection.derivedChanges.state)
+            this.sendChangesToInput(
+              key,
+              collection.derivedChanges.state,
+              collection.config.getId
+            )
           })
           this.incrementVersion()
           this.sendFrontierToAllInputs()

package/src/query/evaluators.ts

@@ -1,4 +1,4 @@
-import { evaluateOperandOnNestedRow } from "./extractors.js"
+import { evaluateOperandOnNamespacedRow } from "./extractors.js"
 import { compareValues, convertLikeToRegex, isValueInArray } from "./utils.js"
 import type {
   Comparator,
@@ -7,12 +7,13 @@ import type {
   LogicalOperator,
   SimpleCondition,
 } from "./schema.js"
+import type { NamespacedRow } from "../types.js"
 
 /**
  * Evaluates a condition against a nested row structure
  */
-export function evaluateConditionOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateConditionOnNamespacedRow(
+  namespacedRow: NamespacedRow,
   condition: Condition,
   mainTableAlias?: string,
   joinedTableAlias?: string
@@ -20,8 +21,8 @@ export function evaluateConditionOnNestedRow(
   // Handle simple conditions with exactly 3 elements
   if (condition.length === 3 && !Array.isArray(condition[0])) {
     const [left, comparator, right] = condition as SimpleCondition
-    return evaluateSimpleConditionOnNestedRow(
-      nestedRow,
+    return evaluateSimpleConditionOnNamespacedRow(
+      namespacedRow,
       left,
       comparator,
       right,
@@ -38,8 +39,8 @@ export function evaluateConditionOnNestedRow(
     ![`and`, `or`].includes(condition[1] as string)
   ) {
     // Start with the first condition (first 3 elements)
-    let result = evaluateSimpleConditionOnNestedRow(
-      nestedRow,
+    let result = evaluateSimpleConditionOnNamespacedRow(
+      namespacedRow,
       condition[0],
       condition[1] as Comparator,
       condition[2],
@@ -53,8 +54,8 @@ export function evaluateConditionOnNestedRow(
 
       // Make sure we have a complete condition to evaluate
       if (i + 3 <= condition.length) {
-        const nextResult = evaluateSimpleConditionOnNestedRow(
-          nestedRow,
+        const nextResult = evaluateSimpleConditionOnNamespacedRow(
+          namespacedRow,
           condition[i + 1],
           condition[i + 2] as Comparator,
           condition[i + 3],
@@ -78,8 +79,8 @@ export function evaluateConditionOnNestedRow(
   // Handle nested composite conditions where the first element is an array
   if (condition.length > 0 && Array.isArray(condition[0])) {
     // Start with the first condition
-    let result = evaluateConditionOnNestedRow(
-      nestedRow,
+    let result = evaluateConditionOnNamespacedRow(
+      namespacedRow,
       condition[0] as Condition,
       mainTableAlias,
       joinedTableAlias
@@ -96,8 +97,8 @@ export function evaluateConditionOnNestedRow(
       if (operator === `and`) {
         result =
           result &&
-          evaluateConditionOnNestedRow(
-            nestedRow,
+          evaluateConditionOnNamespacedRow(
+            namespacedRow,
             nextCondition,
             mainTableAlias,
             joinedTableAlias
@@ -106,8 +107,8 @@ export function evaluateConditionOnNestedRow(
         // logicalOp === `or`
         result =
           result ||
-          evaluateConditionOnNestedRow(
-            nestedRow,
+          evaluateConditionOnNamespacedRow(
+            namespacedRow,
             nextCondition,
             mainTableAlias,
             joinedTableAlias
@@ -125,23 +126,23 @@ export function evaluateConditionOnNestedRow(
 /**
  * Evaluates a simple condition against a nested row structure
  */
-export function evaluateSimpleConditionOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateSimpleConditionOnNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   left: ConditionOperand,
   comparator: Comparator,
   right: ConditionOperand,
   mainTableAlias?: string,
   joinedTableAlias?: string
 ): boolean {
-  const leftValue = evaluateOperandOnNestedRow(
-    nestedRow,
+  const leftValue = evaluateOperandOnNamespacedRow(
+    namespacedRow,
     left,
     mainTableAlias,
     joinedTableAlias
   )
 
-  const rightValue = evaluateOperandOnNestedRow(
-    nestedRow,
+  const rightValue = evaluateOperandOnNamespacedRow(
+    namespacedRow,
     right,
     mainTableAlias,
     joinedTableAlias

package/src/query/extractors.ts

@@ -3,14 +3,14 @@ import type { AllowedFunctionName, ConditionOperand } from "./schema.js"
 
 /**
  * Extracts a value from a nested row structure
- * @param nestedRow The nested row structure
+ * @param namespacedRow The nested row structure
  * @param columnRef The column reference (may include table.column format)
  * @param mainTableAlias The main table alias to check first for columns without table reference
  * @param joinedTableAlias The joined table alias to check second for columns without table reference
  * @returns The extracted value or undefined if not found
  */
-export function extractValueFromNestedRow(
-  nestedRow: Record<string, unknown>,
+export function extractValueFromNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   columnRef: string,
   mainTableAlias?: string,
   joinedTableAlias?: string
@@ -20,7 +20,7 @@ export function extractValueFromNestedRow(
     const [tableAlias, colName] = columnRef.split(`.`) as [string, string]
 
     // Get the table data
-    const tableData = nestedRow[tableAlias] as
+    const tableData = namespacedRow[tableAlias] as
       | Record<string, unknown>
       | null
       | undefined
@@ -34,16 +34,19 @@ export function extractValueFromNestedRow(
     return value
   } else {
     // If no table is specified, first try to find in the main table if provided
-    if (mainTableAlias && nestedRow[mainTableAlias]) {
-      const mainTableData = nestedRow[mainTableAlias] as Record<string, unknown>
+    if (mainTableAlias && namespacedRow[mainTableAlias]) {
+      const mainTableData = namespacedRow[mainTableAlias] as Record<
+        string,
+        unknown
+      >
       if (typeof mainTableData === `object` && columnRef in mainTableData) {
         return mainTableData[columnRef]
       }
     }
 
     // Then try the joined table if provided
-    if (joinedTableAlias && nestedRow[joinedTableAlias]) {
-      const joinedTableData = nestedRow[joinedTableAlias] as Record<
+    if (joinedTableAlias && namespacedRow[joinedTableAlias]) {
+      const joinedTableData = namespacedRow[joinedTableAlias] as Record<
         string,
         unknown
       >
@@ -53,7 +56,7 @@ export function extractValueFromNestedRow(
     }
 
     // If not found in main or joined table, try to find the column in any table
-    for (const [_tableAlias, tableData] of Object.entries(nestedRow)) {
+    for (const [_tableAlias, tableData] of Object.entries(namespacedRow)) {
       if (
         tableData &&
         typeof tableData === `object` &&
@@ -69,8 +72,8 @@ export function extractValueFromNestedRow(
 /**
  * Evaluates an operand against a nested row structure
  */
-export function evaluateOperandOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateOperandOnNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   operand: ConditionOperand,
   mainTableAlias?: string,
   joinedTableAlias?: string
@@ -78,8 +81,8 @@ export function evaluateOperandOnNestedRow(
   // Handle column references
   if (typeof operand === `string` && operand.startsWith(`@`)) {
     const columnRef = operand.substring(1)
-    return extractValueFromNestedRow(
-      nestedRow,
+    return extractValueFromNamespacedRow(
+      namespacedRow,
       columnRef,
       mainTableAlias,
       joinedTableAlias
@@ -92,8 +95,8 @@ export function evaluateOperandOnNestedRow(
 
     if (typeof colRef === `string`) {
       // First try to extract from nested row structure
-      const nestedValue = extractValueFromNestedRow(
-        nestedRow,
+      const nestedValue = extractValueFromNamespacedRow(
+        namespacedRow,
         colRef,
         mainTableAlias,
         joinedTableAlias
@@ -101,8 +104,8 @@ export function evaluateOperandOnNestedRow(
 
       // If not found in nested structure, check if it's a direct property of the row
       // This is important for HAVING clauses that reference aggregated values
-      if (nestedValue === undefined && colRef in nestedRow) {
-        return nestedRow[colRef]
+      if (nestedValue === undefined && colRef in namespacedRow) {
+        return namespacedRow[colRef]
       }
 
       return nestedValue
@@ -121,15 +124,15 @@ export function evaluateOperandOnNestedRow(
     // If the arguments are a reference or another expression, evaluate them first
     const evaluatedArgs = Array.isArray(args)
       ? args.map((arg) =>
-          evaluateOperandOnNestedRow(
-            nestedRow,
+          evaluateOperandOnNamespacedRow(
+            namespacedRow,
             arg as ConditionOperand,
             mainTableAlias,
             joinedTableAlias
           )
         )
-      : evaluateOperandOnNestedRow(
-          nestedRow,
+      : evaluateOperandOnNamespacedRow(
+          namespacedRow,
           args as ConditionOperand,
           mainTableAlias,
           joinedTableAlias

package/src/query/group-by.ts

@@ -1,11 +1,11 @@
-import { groupBy, groupByOperators, map } from "@electric-sql/d2ts"
+import { groupBy, groupByOperators } from "@electric-sql/d2ts"
 import {
-  evaluateOperandOnNestedRow,
-  extractValueFromNestedRow,
+  evaluateOperandOnNamespacedRow,
+  extractValueFromNamespacedRow,
 } from "./extractors"
 import { isAggregateFunctionCall } from "./utils"
 import type { ConditionOperand, FunctionCall, Query } from "./schema"
-import type { IStreamBuilder } from "@electric-sql/d2ts"
+import type { NamespacedAndKeyedStream } from "../types.js"
 
 const { sum, count, avg, min, max, median, mode } = groupByOperators
 
@@ -13,7 +13,7 @@ const { sum, count, avg, min, max, median, mode } = groupByOperators
  * Process the groupBy clause in a D2QL query
  */
 export function processGroupBy(
-  pipeline: IStreamBuilder<Record<string, unknown>>,
+  pipeline: NamespacedAndKeyedStream,
   query: Query,
   mainTableAlias: string
 ) {
@@ -23,7 +23,10 @@ export function processGroupBy(
     : [query.groupBy]
 
   // Create a key extractor function for the groupBy operator
-  const keyExtractor = (nestedRow: Record<string, unknown>) => {
+  const keyExtractor = ([_oldKey, namespacedRow]: [
+    string,
+    Record<string, unknown>,
+  ]) => {
     const key: Record<string, unknown> = {}
 
     // Extract each groupBy column value
@@ -34,8 +37,8 @@ export function processGroupBy(
           ? columnRef.split(`.`)[1]
           : columnRef
 
-        key[columnName!] = extractValueFromNestedRow(
-          nestedRow,
+        key[columnName!] = extractValueFromNamespacedRow(
+          namespacedRow,
           columnRef,
           mainTableAlias
         )
@@ -48,6 +51,10 @@ export function processGroupBy(
   // Create aggregate functions for any aggregated columns in the SELECT clause
   const aggregates: Record<string, any> = {}
 
+  if (!query.select) {
+    throw new Error(`SELECT clause is required for GROUP BY`)
+  }
+
   // Scan the SELECT clause for aggregate functions
   for (const item of query.select) {
     if (typeof item === `object`) {
@@ -73,15 +80,7 @@ export function processGroupBy(
 
   // Apply the groupBy operator if we have any aggregates
   if (Object.keys(aggregates).length > 0) {
-    pipeline = pipeline.pipe(
-      groupBy(keyExtractor, aggregates),
-      // Convert KeyValue<string, ResultType> to Record<string, unknown>
-      map(([_key, value]) => {
-        // After groupBy, the value already contains both the key fields and aggregate results
-        // We need to return it as is, not wrapped in a nested structure
-        return value as Record<string, unknown>
-      })
-    )
+    pipeline = pipeline.pipe(groupBy(keyExtractor, aggregates))
   }
 
   return pipeline
@@ -96,17 +95,20 @@ export function getAggregateFunction(
   mainTableAlias: string
 ) {
   // Create a value extractor function for the column to aggregate
-  const valueExtractor = (nestedRow: Record<string, unknown>) => {
+  const valueExtractor = ([_oldKey, namespacedRow]: [
+    string,
+    Record<string, unknown>,
+  ]) => {
     let value: unknown
     if (typeof columnRef === `string` && columnRef.startsWith(`@`)) {
-      value = extractValueFromNestedRow(
-        nestedRow,
+      value = extractValueFromNamespacedRow(
+        namespacedRow,
         columnRef.substring(1),
         mainTableAlias
       )
     } else {
-      value = evaluateOperandOnNestedRow(
-        nestedRow,
+      value = evaluateOperandOnNamespacedRow(
+        namespacedRow,
         columnRef as ConditionOperand,
         mainTableAlias
       )