@tanstack/db 0.5.18 → 0.5.19

package/src/query/builder/types.ts

@@ -345,6 +345,26 @@ export type JoinOnCallback<TContext extends Context> = (
   refs: RefsForContext<TContext>,
 ) => any
 
+/**
+ * FunctionalHavingRow - Type for the row parameter in functional having callbacks
+ *
+ * Functional having callbacks receive a namespaced row that includes:
+ * - Table data from the schema (when available)
+ * - $selected: The SELECT result fields (when select() has been called)
+ *
+ * After `select()` is called, this type includes `$selected` which provides access
+ * to the SELECT result fields via `$selected.fieldName` syntax.
+ *
+ * Note: When used with GROUP BY, functional having receives `{ $selected: ... }` with the
+ * aggregated SELECT results. When used without GROUP BY, it receives the full namespaced row
+ * which includes both table data and `$selected`.
+ *
+ * Example: `({ $selected }) => $selected.sessionCount > 2`
+ * Example (no GROUP BY): `(row) => row.user.salary > 70000 && row.$selected.user_count > 2`
+ */
+export type FunctionalHavingRow<TContext extends Context> = TContext[`schema`] &
+  (TContext[`result`] extends object ? { $selected: TContext[`result`] } : {})
+
 /**
  * RefProxyForContext - Creates ref proxies for all tables/collections in a query context
  *
@@ -364,6 +384,9 @@ export type JoinOnCallback<TContext extends Context> = (
  *
  * The logic prioritizes optional chaining by always placing `undefined` outside when
  * a type is both optional and nullable (e.g., `string | null | undefined`).
+ *
+ * After `select()` is called, this type also includes `$selected` which provides access
+ * to the SELECT result fields via `$selected.fieldName` syntax.
  */
 export type RefsForContext<TContext extends Context> = {
   [K in keyof TContext[`schema`]]: IsNonExactOptional<
@@ -383,7 +406,9 @@ export type RefsForContext<TContext extends Context> = {
       : // T is exactly undefined, exactly null, or neither optional nor nullable
         // Wrap in RefProxy as-is (includes exact undefined, exact null, and normal types)
         Ref<TContext[`schema`][K]>
-}
+} & (TContext[`result`] extends object
+  ? { $selected: Ref<TContext[`result`]> }
+  : {})
 
 /**
  * Type Detection Helpers

package/src/query/compiler/evaluators.ts

@@ -95,12 +95,48 @@ function compileExpressionInternal(
  * Compiles a reference expression into an optimized evaluator
  */
 function compileRef(ref: PropRef): CompiledExpression {
-  const [tableAlias, ...propertyPath] = ref.path
+  const [namespace, ...propertyPath] = ref.path
 
-  if (!tableAlias) {
+  if (!namespace) {
     throw new EmptyReferencePathError()
   }
 
+  // Handle $selected namespace - references SELECT result fields
+  if (namespace === `$selected`) {
+    // Access $selected directly
+    if (propertyPath.length === 0) {
+      // Just $selected - return entire $selected object
+      return (namespacedRow) => (namespacedRow as any).$selected
+    } else if (propertyPath.length === 1) {
+      // Single property access - most common case
+      const prop = propertyPath[0]!
+      return (namespacedRow) => {
+        const selectResults = (namespacedRow as any).$selected
+        return selectResults?.[prop]
+      }
+    } else {
+      // Multiple property navigation (nested SELECT fields)
+      return (namespacedRow) => {
+        const selectResults = (namespacedRow as any).$selected
+        if (selectResults === undefined) {
+          return undefined
+        }
+
+        let value: any = selectResults
+        for (const prop of propertyPath) {
+          if (value == null) {
+            return value
+          }
+          value = value[prop]
+        }
+        return value
+      }
+    }
+  }
+
+  // Handle table alias namespace (existing logic)
+  const tableAlias = namespace
+
   // Pre-compile the property path navigation
   if (propertyPath.length === 0) {
     // Simple table reference

package/src/query/compiler/group-by.ts

@@ -1,4 +1,10 @@
-import { filter, groupBy, groupByOperators, map } from '@tanstack/db-ivm'
+import {
+  filter,
+  groupBy,
+  groupByOperators,
+  map,
+  serializeValue,
+} from '@tanstack/db-ivm'
 import { Func, PropRef, getHavingExpression } from '../ir.js'
 import {
   AggregateFunctionNotInSelectError,
@@ -66,7 +72,7 @@ function validateAndCreateMapping(
 
 /**
  * Processes the GROUP BY clause with optional HAVING and SELECT
- * Works with the new __select_results structure from early SELECT processing
+ * Works with the new $selected structure from early SELECT processing
  */
 export function processGroupBy(
   pipeline: NamespacedAndKeyedStream,
@@ -98,11 +104,11 @@ export function processGroupBy(
       groupBy(keyExtractor, aggregates),
     ) as NamespacedAndKeyedStream
 
-    // Update __select_results to include aggregate values
+    // Update $selected to include aggregate values
     pipeline = pipeline.pipe(
       map(([, aggregatedRow]) => {
-        // Start with the existing __select_results from early SELECT processing
-        const selectResults = (aggregatedRow as any).__select_results || {}
+        // Start with the existing $selected from early SELECT processing
+        const selectResults = (aggregatedRow as any).$selected || {}
         const finalResults: Record<string, any> = { ...selectResults }
 
         if (selectClause) {
@@ -115,12 +121,12 @@ export function processGroupBy(
           }
         }
 
-        // Use a single key for the result and update __select_results
+        // Use a single key for the result and update $selected
         return [
           `single_group`,
           {
             ...aggregatedRow,
-            __select_results: finalResults,
+            $selected: finalResults,
           },
         ] as [unknown, Record<string, any>]
       }),
@@ -133,13 +139,14 @@ export function processGroupBy(
         const transformedHavingClause = replaceAggregatesByRefs(
           havingExpression,
           selectClause || {},
+          `$selected`,
         )
         const compiledHaving = compileExpression(transformedHavingClause)
 
         pipeline = pipeline.pipe(
           filter(([, row]) => {
             // Create a namespaced row structure for HAVING evaluation
-            const namespacedRow = { result: (row as any).__select_results }
+            const namespacedRow = { $selected: (row as any).$selected }
             return toBooleanPredicate(compiledHaving(namespacedRow))
           }),
         )
@@ -152,7 +159,7 @@ export function processGroupBy(
         pipeline = pipeline.pipe(
           filter(([, row]) => {
             // Create a namespaced row structure for functional HAVING evaluation
-            const namespacedRow = { result: (row as any).__select_results }
+            const namespacedRow = { $selected: (row as any).$selected }
             return toBooleanPredicate(fnHaving(namespacedRow))
           }),
         )
@@ -174,11 +181,11 @@ export function processGroupBy(
   // Create a key extractor function using simple __key_X format
   const keyExtractor = ([, row]: [
     string,
-    NamespacedRow & { __select_results?: any },
+    NamespacedRow & { $selected?: any },
   ]) => {
-    // Use the original namespaced row for GROUP BY expressions, not __select_results
+    // Use the original namespaced row for GROUP BY expressions, not $selected
     const namespacedRow = { ...row }
-    delete (namespacedRow as any).__select_results
+    delete (namespacedRow as any).$selected
 
     const key: Record<string, unknown> = {}
 
@@ -208,11 +215,11 @@ export function processGroupBy(
   // Apply the groupBy operator
   pipeline = pipeline.pipe(groupBy(keyExtractor, aggregates))
 
-  // Update __select_results to handle GROUP BY results
+  // Update $selected to handle GROUP BY results
   pipeline = pipeline.pipe(
     map(([, aggregatedRow]) => {
-      // Start with the existing __select_results from early SELECT processing
-      const selectResults = (aggregatedRow as any).__select_results || {}
+      // Start with the existing $selected from early SELECT processing
+      const selectResults = (aggregatedRow as any).$selected || {}
       const finalResults: Record<string, any> = {}
 
       if (selectClause) {
@@ -248,14 +255,14 @@ export function processGroupBy(
         for (let i = 0; i < groupByClause.length; i++) {
           keyParts.push(aggregatedRow[`__key_${i}`])
         }
-        finalKey = JSON.stringify(keyParts)
+        finalKey = serializeValue(keyParts)
       }
 
       return [
         finalKey,
         {
           ...aggregatedRow,
-          __select_results: finalResults,
+          $selected: finalResults,
         },
       ] as [unknown, Record<string, any>]
     }),
@@ -274,7 +281,7 @@ export function processGroupBy(
       pipeline = pipeline.pipe(
         filter(([, row]) => {
           // Create a namespaced row structure for HAVING evaluation
-          const namespacedRow = { result: (row as any).__select_results }
+          const namespacedRow = { $selected: (row as any).$selected }
           return compiledHaving(namespacedRow)
         }),
       )
@@ -287,7 +294,7 @@ export function processGroupBy(
       pipeline = pipeline.pipe(
         filter(([, row]) => {
           // Create a namespaced row structure for functional HAVING evaluation
-          const namespacedRow = { result: (row as any).__select_results }
+          const namespacedRow = { $selected: (row as any).$selected }
           return toBooleanPredicate(fnHaving(namespacedRow))
         }),
       )
@@ -385,12 +392,28 @@ function getAggregateFunction(aggExpr: Aggregate) {
 }
 
 /**
- * Transforms basic expressions and aggregates to replace Agg expressions with references to computed values
+ * Transforms expressions to replace aggregate functions with references to computed values.
+ *
+ * This function is used in both ORDER BY and HAVING clauses to transform expressions that reference:
+ * 1. Aggregate functions (e.g., `max()`, `count()`) - replaces with references to computed aggregates in SELECT
+ * 2. SELECT field references via $selected namespace (e.g., `$selected.latestActivity`) - validates and passes through unchanged
+ *
+ * For aggregate expressions, it finds matching aggregates in the SELECT clause and replaces them with
+ * PropRef([resultAlias, alias]) to reference the computed aggregate value.
+ *
+ * For ref expressions using the $selected namespace, it validates that the field exists in the SELECT clause
+ * and passes them through unchanged (since $selected is already the correct namespace). All other ref expressions
+ * are passed through unchanged (treating them as table column references).
+ *
+ * @param havingExpr - The expression to transform (can be aggregate, ref, func, or val)
+ * @param selectClause - The SELECT clause containing aliases and aggregate definitions
+ * @param resultAlias - The namespace alias for SELECT results (default: '$selected', used for aggregate references)
+ * @returns A transformed BasicExpression that references computed values instead of raw expressions
  */
 export function replaceAggregatesByRefs(
   havingExpr: BasicExpression | Aggregate,
   selectClause: Select,
-  resultAlias: string = `result`,
+  resultAlias: string = `$selected`,
 ): BasicExpression {
   switch (havingExpr.type) {
     case `agg`: {
@@ -417,7 +440,38 @@ export function replaceAggregatesByRefs(
     }
 
     case `ref`: {
-      // Non-aggregate refs are passed through unchanged (they reference table columns)
+      const refExpr = havingExpr
+      const path = refExpr.path
+
+      if (path.length === 0) {
+        // Empty path - pass through
+        return havingExpr as BasicExpression
+      }
+
+      // Check if this is a $selected reference
+      if (path.length > 0 && path[0] === `$selected`) {
+        // Extract the field path after $selected
+        const fieldPath = path.slice(1)
+
+        if (fieldPath.length === 0) {
+          // Just $selected without a field - pass through unchanged
+          return havingExpr as BasicExpression
+        }
+
+        // Verify the field exists in SELECT clause
+        const alias = fieldPath.join(`.`)
+        if (alias in selectClause) {
+          // Pass through unchanged - $selected is already the correct namespace
+          return havingExpr as BasicExpression
+        }
+
+        // Field doesn't exist in SELECT - this is an error, but we'll pass through for now
+        // (Could throw an error here in the future)
+        return havingExpr as BasicExpression
+      }
+
+      // Not a $selected reference - this is a table column reference, pass through unchanged
+      // SELECT fields should only be accessed via $selected namespace
       return havingExpr as BasicExpression
     }
 

package/src/query/compiler/index.ts

@@ -216,7 +216,7 @@ export function compileQuery(
     throw new DistinctRequiresSelectError()
   }
 
-  // Process the SELECT clause early - always create __select_results
+  // Process the SELECT clause early - always create $selected
   // This eliminates duplication and allows for DISTINCT implementation
   if (query.fnSelect) {
     // Handle functional select - apply the function to transform the row
@@ -227,15 +227,15 @@ export function compileQuery(
           key,
           {
             ...namespacedRow,
-            __select_results: selectResults,
+            $selected: selectResults,
           },
-        ] as [string, typeof namespacedRow & { __select_results: any }]
+        ] as [string, typeof namespacedRow & { $selected: any }]
       }),
     )
   } else if (query.select) {
     pipeline = processSelect(pipeline, query.select, allInputs)
   } else {
-    // If no SELECT clause, create __select_results with the main table data
+    // If no SELECT clause, create $selected with the main table data
     pipeline = pipeline.pipe(
       map(([key, namespacedRow]) => {
         const selectResults =
@@ -247,9 +247,9 @@ export function compileQuery(
           key,
           {
             ...namespacedRow,
-            __select_results: selectResults,
+            $selected: selectResults,
           },
-        ] as [string, typeof namespacedRow & { __select_results: any }]
+        ] as [string, typeof namespacedRow & { $selected: any }]
       }),
     )
   }
@@ -310,7 +310,7 @@ export function compileQuery(
 
   // Process the DISTINCT clause if it exists
   if (query.distinct) {
-    pipeline = pipeline.pipe(distinct(([_key, row]) => row.__select_results))
+    pipeline = pipeline.pipe(distinct(([_key, row]) => row.$selected))
   }
 
   // Process orderBy parameter if it exists
@@ -327,11 +327,11 @@ export function compileQuery(
       query.offset,
     )
 
-    // Final step: extract the __select_results and include orderBy index
+    // Final step: extract the $selected and include orderBy index
     const resultPipeline = orderedPipeline.pipe(
       map(([key, [row, orderByIndex]]) => {
-        // Extract the final results from __select_results and include orderBy index
-        const raw = (row as any).__select_results
+        // Extract the final results from $selected and include orderBy index
+        const raw = (row as any).$selected
         const finalResults = unwrapValue(raw)
         return [key, [finalResults, orderByIndex]] as [unknown, [any, string]]
       }),
@@ -354,11 +354,11 @@ export function compileQuery(
     throw new LimitOffsetRequireOrderByError()
   }
 
-  // Final step: extract the __select_results and return tuple format (no orderBy)
+  // Final step: extract the $selected and return tuple format (no orderBy)
   const resultPipeline: ResultStream = pipeline.pipe(
     map(([key, row]) => {
-      // Extract the final results from __select_results and return [key, [results, undefined]]
-      const raw = (row as any).__select_results
+      // Extract the final results from $selected and return [key, [results, undefined]]
+      const raw = (row as any).$selected
       const finalResults = unwrapValue(raw)
       return [key, [finalResults, undefined]] as [
         unknown,

package/src/query/compiler/order-by.ts

@@ -38,7 +38,7 @@ export type OrderByOptimizationInfo = {
 
 /**
  * Processes the ORDER BY clause
- * Works with the new structure that has both namespaced row data and __select_results
+ * Works with the new structure that has both namespaced row data and $selected
  * Always uses fractional indexing and adds the index as __ordering_index to the result
  */
 export function processOrderBy(
@@ -57,7 +57,7 @@ export function processOrderBy(
     const clauseWithoutAggregates = replaceAggregatesByRefs(
       clause.expression,
       selectClause,
-      `__select_results`,
+      `$selected`,
     )
 
     return {
@@ -67,12 +67,13 @@ export function processOrderBy(
   })
 
   // Create a value extractor function for the orderBy operator
-  const valueExtractor = (row: NamespacedRow & { __select_results?: any }) => {
+  const valueExtractor = (row: NamespacedRow & { $selected?: any }) => {
     // The namespaced row contains:
     // 1. Table aliases as top-level properties (e.g., row["tableName"])
-    // 2. SELECT results in __select_results (e.g., row.__select_results["aggregateAlias"])
-    // The replaceAggregatesByRefs function has already transformed any aggregate expressions
-    // that match SELECT aggregates to use the __select_results namespace.
+    // 2. SELECT results in $selected (e.g., row.$selected["aggregateAlias"])
+    // The replaceAggregatesByRefs function has already transformed:
+    // - Aggregate expressions that match SELECT aggregates to use the $selected namespace
+    // - $selected ref expressions are passed through unchanged (already using the correct namespace)
     const orderByContext = row
 
     if (orderByClause.length > 1) {

package/src/query/compiler/select.ts

@@ -100,7 +100,7 @@ function processNonMergeOp(
 function processRow(
   [key, namespacedRow]: [unknown, NamespacedRow],
   ops: Array<SelectOp>,
-): [unknown, typeof namespacedRow & { __select_results: any }] {
+): [unknown, typeof namespacedRow & { $selected: any }] {
   const selectResults: Record<string, any> = {}
 
   for (const op of ops) {
@@ -111,21 +111,18 @@ function processRow(
     }
   }
 
-  // Return the namespaced row with __select_results added
+  // Return the namespaced row with $selected added
   return [
     key,
     {
       ...namespacedRow,
-      __select_results: selectResults,
+      $selected: selectResults,
     },
-  ] as [
-    unknown,
-    typeof namespacedRow & { __select_results: typeof selectResults },
-  ]
+  ] as [unknown, typeof namespacedRow & { $selected: typeof selectResults }]
 }
 
 /**
- * Processes the SELECT clause and places results in __select_results
+ * Processes the SELECT clause and places results in $selected
  * while preserving the original namespaced row for ORDER BY access
  */
 export function processSelect(

package/src/query/live/collection-config-builder.ts

@@ -337,6 +337,10 @@ export class CollectionConfigBuilder<
       if (syncState.subscribedToAllCollections) {
         while (syncState.graph.pendingWork()) {
           syncState.graph.run()
+          // Flush accumulated changes after each graph step to commit them as one transaction.
+          // This ensures intermediate join states (like null on one side) don't cause
+          // duplicate key errors when the full join result arrives in the same step.
+          syncState.flushPendingChanges?.()
           callback?.()
         }
 
@@ -345,10 +349,14 @@ export class CollectionConfigBuilder<
         if (syncState.messagesCount === 0) {
           begin()
           commit()
-          // After initial commit, check if we should mark ready
-          // (in case all sources were already ready before we subscribed)
-          this.updateLiveQueryStatus(this.currentSyncConfig)
         }
+
+        // After graph processing completes, check if we should mark ready.
+        // This is the canonical place to transition to ready state because:
+        // 1. All data has been processed through the graph
+        // 2. All source collections have had a chance to send their initial data
+        // This prevents marking ready before data is processed (fixes isReady=true with empty data)
+        this.updateLiveQueryStatus(this.currentSyncConfig)
       }
     } finally {
       this.isGraphRunning = false
@@ -687,22 +695,35 @@ export class CollectionConfigBuilder<
     const { begin, commit } = config
     const { graph, inputs, pipeline } = this.maybeCompileBasePipeline()
 
+    // Accumulator for changes across all output callbacks within a single graph run.
+    // This allows us to batch all changes from intermediate join states into a single
+    // transaction, avoiding duplicate key errors when joins produce multiple outputs
+    // for the same key (e.g., first output with null, then output with joined data).
+    let pendingChanges: Map<unknown, Changes<TResult>> = new Map()
+
     pipeline.pipe(
       output((data) => {
         const messages = data.getInner()
         syncState.messagesCount += messages.length
 
-        begin()
-        messages
-          .reduce(
-            accumulateChanges<TResult>,
-            new Map<unknown, Changes<TResult>>(),
-          )
-          .forEach(this.applyChanges.bind(this, config))
-        commit()
+        // Accumulate changes from this output callback into the pending changes map.
+        // Changes for the same key are merged (inserts/deletes are added together).
+        messages.reduce(accumulateChanges<TResult>, pendingChanges)
       }),
     )
 
+    // Flush pending changes and reset the accumulator.
+    // Called at the end of each graph run to commit all accumulated changes.
+    syncState.flushPendingChanges = () => {
+      if (pendingChanges.size === 0) {
+        return
+      }
+      begin()
+      pendingChanges.forEach(this.applyChanges.bind(this, config))
+      commit()
+      pendingChanges = new Map()
+    }
+
     graph.finalize()
 
     // Extend the sync state with the graph, inputs, and pipeline
@@ -808,11 +829,14 @@ export class CollectionConfigBuilder<
       return
     }
 
-    // Mark ready when all source collections are ready AND
-    // the live query collection is not loading subset data.
-    // This prevents marking the live query ready before its data is loaded
+    // Mark ready when:
+    // 1. All subscriptions are set up (subscribedToAllCollections)
+    // 2. All source collections are ready
+    // 3. The live query collection is not loading subset data
+    // This prevents marking the live query ready before its data is processed
     // (fixes issue where useLiveQuery returns isReady=true with empty data)
     if (
+      this.currentSyncState?.subscribedToAllCollections &&
       this.allCollectionsReady() &&
       !this.liveQueryCollection?.isLoadingSubset
     ) {
@@ -913,8 +937,10 @@ export class CollectionConfigBuilder<
     // (graph only runs when all collections are subscribed)
     syncState.subscribedToAllCollections = true
 
-    // Initial status check after all subscriptions are set up
-    this.updateLiveQueryStatus(config)
+    // Note: We intentionally don't call updateLiveQueryStatus() here.
+    // The graph hasn't run yet, so marking ready would be premature.
+    // The canonical place to mark ready is after the graph processes data
+    // in maybeRunGraph(), which ensures data has been processed first.
 
     return loadSubsetDataCallbacks
   }
@@ -1096,8 +1122,11 @@ function accumulateChanges<T>(
     changes.deletes += Math.abs(multiplicity)
   } else if (multiplicity > 0) {
     changes.inserts += multiplicity
+    // Update value to the latest version for this key
     changes.value = value
-    changes.orderByIndex = orderByIndex
+    if (orderByIndex !== undefined) {
+      changes.orderByIndex = orderByIndex
+    }
   }
   acc.set(key, changes)
   return acc

package/src/query/live/types.ts

@@ -22,9 +22,11 @@ export type SyncState = {
   graph?: D2
   inputs?: Record<string, RootStreamBuilder<unknown>>
   pipeline?: ResultStream
+  flushPendingChanges?: () => void
 }
 
-export type FullSyncState = Required<SyncState>
+export type FullSyncState = Required<Omit<SyncState, `flushPendingChanges`>> &
+  Pick<SyncState, `flushPendingChanges`>
 
 /**
  * Configuration interface for live query collection options