@tanstack/db 0.5.32 → 0.6.0

package/src/query/live/collection-config-builder.ts

@@ -1,6 +1,6 @@
-import { D2, output } from '@tanstack/db-ivm'
-import { compileQuery } from '../compiler/index.js'
-import { buildQuery, getQueryIR } from '../builder/index.js'
+import { D2, output, serializeValue } from '@tanstack/db-ivm'
+import { INCLUDES_ROUTING, compileQuery } from '../compiler/index.js'
+import { createCollection } from '../../collection/index.js'
 import {
   MissingAliasInputsError,
   SetWindowRequiresOrderByError,
@@ -10,14 +10,24 @@ import { getActiveTransaction } from '../../transactions.js'
 import { CollectionSubscriber } from './collection-subscriber.js'
 import { getCollectionBuilder } from './collection-registry.js'
 import { LIVE_QUERY_INTERNAL } from './internal.js'
+import {
+  buildQueryFromConfig,
+  extractCollectionAliases,
+  extractCollectionFromSource,
+  extractCollectionsFromQuery,
+} from './utils.js'
 import type { LiveQueryInternalUtils } from './internal.js'
-import type { WindowOptions } from '../compiler/index.js'
+import type {
+  IncludesCompilationResult,
+  WindowOptions,
+} from '../compiler/index.js'
 import type { SchedulerContextId } from '../../scheduler.js'
 import type { CollectionSubscription } from '../../collection/subscription.js'
 import type { RootStreamBuilder } from '@tanstack/db-ivm'
 import type { OrderByOptimizationInfo } from '../compiler/order-by.js'
 import type { Collection } from '../../collection/index.js'
 import type {
+  ChangeMessage,
   CollectionConfigSingleRowOption,
   KeyedStream,
   ResultStream,
@@ -26,7 +36,12 @@ import type {
   UtilsRecord,
 } from '../../types.js'
 import type { Context, GetResult } from '../builder/types.js'
-import type { BasicExpression, QueryIR } from '../ir.js'
+import type {
+  BasicExpression,
+  IncludesMaterialization,
+  PropRef,
+  QueryIR,
+} from '../ir.js'
 import type { LazyCollectionCallbacks } from '../compiler/joins.js'
 import type {
   Changes,
@@ -135,6 +150,7 @@ export class CollectionConfigBuilder<
   public sourceWhereClausesCache:
     | Map<string, BasicExpression<boolean>>
     | undefined
+  private includesCache: Array<IncludesCompilationResult> | undefined
 
   // Map of source alias to subscription
   readonly subscriptions: Record<string, CollectionSubscription> = {}
@@ -151,7 +167,10 @@ export class CollectionConfigBuilder<
     // Generate a unique ID if not provided
     this.id = config.id || `live-query-${++liveQueryCollectionCounter}`
 
-    this.query = buildQueryFromConfig(config)
+    this.query = buildQueryFromConfig({
+      query: config.query,
+      requireObjectResult: true,
+    })
     this.collections = extractCollectionsFromQuery(this.query)
     const collectionAliasesById = extractCollectionAliases(this.query)
 
@@ -627,6 +646,7 @@ export class CollectionConfigBuilder<
       this.inputsCache = undefined
       this.pipelineCache = undefined
       this.sourceWhereClausesCache = undefined
+      this.includesCache = undefined
 
       // Reset lazy source alias state
       this.lazySources.clear()
@@ -675,6 +695,7 @@ export class CollectionConfigBuilder<
     this.pipelineCache = compilation.pipeline
     this.sourceWhereClausesCache = compilation.sourceWhereClauses
     this.compiledAliasToCollectionId = compilation.aliasToCollectionId
+    this.includesCache = compilation.includes
 
     // Defensive check: verify all compiled aliases have corresponding inputs
     // This should never happen since all aliases come from user declarations,
@@ -722,10 +743,19 @@ export class CollectionConfigBuilder<
       }),
     )
 
+    // Set up includes output routing and child collection lifecycle
+    const includesState = this.setupIncludesOutput(
+      this.includesCache,
+      syncState,
+    )
+
     // Flush pending changes and reset the accumulator.
     // Called at the end of each graph run to commit all accumulated changes.
     syncState.flushPendingChanges = () => {
-      if (pendingChanges.size === 0) {
+      const hasParentChanges = pendingChanges.size > 0
+      const hasChildChanges = hasPendingIncludesChanges(includesState)
+
+      if (!hasParentChanges && !hasChildChanges) {
         return
       }
 
@@ -757,10 +787,22 @@ export class CollectionConfigBuilder<
         changesToApply = merged
       }
 
-      begin()
-      changesToApply.forEach(this.applyChanges.bind(this, config))
-      commit()
+      // 1. Flush parent changes
+      if (hasParentChanges) {
+        begin()
+        changesToApply.forEach(this.applyChanges.bind(this, config))
+        commit()
+      }
       pendingChanges = new Map()
+
+      // 2. Process includes: create/dispose child Collections, route child changes
+      flushIncludesState(
+        includesState,
+        config.collection,
+        this.id,
+        hasParentChanges ? changesToApply : null,
+        config,
+      )
     }
 
     graph.finalize()
@@ -773,6 +815,88 @@ export class CollectionConfigBuilder<
     return syncState as FullSyncState
   }
 
+  /**
+   * Sets up output callbacks for includes child pipelines.
+   * Each includes entry gets its own output callback that accumulates child changes,
+   * and a child registry that maps correlation key → child Collection.
+   */
+  private setupIncludesOutput(
+    includesEntries: Array<IncludesCompilationResult> | undefined,
+    syncState: SyncState,
+  ): Array<IncludesOutputState> {
+    if (!includesEntries || includesEntries.length === 0) {
+      return []
+    }
+
+    return includesEntries.map((entry) => {
+      const state: IncludesOutputState = {
+        fieldName: entry.fieldName,
+        childCorrelationField: entry.childCorrelationField,
+        hasOrderBy: entry.hasOrderBy,
+        materialization: entry.materialization,
+        scalarField: entry.scalarField,
+        childRegistry: new Map(),
+        pendingChildChanges: new Map(),
+        correlationToParentKeys: new Map(),
+      }
+
+      // Attach output callback on the child pipeline
+      entry.pipeline.pipe(
+        output((data) => {
+          const messages = data.getInner()
+          syncState.messagesCount += messages.length
+
+          for (const [[childKey, tupleData], multiplicity] of messages) {
+            const [childResult, _orderByIndex, correlationKey, parentContext] =
+              tupleData as unknown as [
+                any,
+                string | undefined,
+                unknown,
+                Record<string, any> | null,
+              ]
+
+            const routingKey = computeRoutingKey(correlationKey, parentContext)
+
+            // Accumulate by [routingKey, childKey]
+            let byChild = state.pendingChildChanges.get(routingKey)
+            if (!byChild) {
+              byChild = new Map()
+              state.pendingChildChanges.set(routingKey, byChild)
+            }
+
+            const existing = byChild.get(childKey) || {
+              deletes: 0,
+              inserts: 0,
+              value: childResult,
+              orderByIndex: _orderByIndex,
+            }
+
+            if (multiplicity < 0) {
+              existing.deletes += Math.abs(multiplicity)
+            } else if (multiplicity > 0) {
+              existing.inserts += multiplicity
+              existing.value = childResult
+            }
+
+            byChild.set(childKey, existing)
+          }
+        }),
+      )
+
+      // Set up shared buffers for nested includes (e.g., comments inside issues)
+      if (entry.childCompilationResult.includes) {
+        state.nestedSetups = setupNestedPipelines(
+          entry.childCompilationResult.includes,
+          syncState,
+        )
+        state.nestedRoutingIndex = new Map()
+        state.nestedRoutingReverseIndex = new Map()
+      }
+
+      return state
+    })
+  }
+
   private applyChanges(
     config: SyncMethods<TResult>,
     changes: {
@@ -984,16 +1108,6 @@ export class CollectionConfigBuilder<
   }
 }
 
-function buildQueryFromConfig<TContext extends Context>(
-  config: LiveQueryCollectionConfig<any, any>,
-) {
-  // Build the query using the provided query builder function or instance
-  if (typeof config.query === `function`) {
-    return buildQuery<TContext>(config.query)
-  }
-  return getQueryIR(config.query)
-}
-
 function createOrderByComparator<T extends object>(
   orderByIndices: WeakMap<object, string>,
 ) {
@@ -1019,124 +1133,684 @@ function createOrderByComparator<T extends object>(
 }
 
 /**
- * Helper function to extract collections from a compiled query
- * Traverses the query IR to find all collection references
- * Maps collections by their ID (not alias) as expected by the compiler
+ * Shared buffer setup for a single nested includes level.
+ * Pipeline output writes into the buffer; during flush the buffer is drained
+ * into per-entry states via the routing index.
+ */
+type NestedIncludesSetup = {
+  compilationResult: IncludesCompilationResult
+  /** Shared buffer: nestedCorrelationKey → Map<childKey, Changes> */
+  buffer: Map<unknown, Map<unknown, Changes<any>>>
+  /** For 3+ levels of nesting */
+  nestedSetups?: Array<NestedIncludesSetup>
+}
+
+/**
+ * State tracked per includes entry for output routing and child lifecycle
  */
-function extractCollectionsFromQuery(
-  query: any,
-): Record<string, Collection<any, any, any>> {
-  const collections: Record<string, any> = {}
-
-  // Helper function to recursively extract collections from a query or source
-  function extractFromSource(source: any) {
-    if (source.type === `collectionRef`) {
-      collections[source.collection.id] = source.collection
-    } else if (source.type === `queryRef`) {
-      // Recursively extract from subquery
-      extractFromQuery(source.query)
+type IncludesOutputState = {
+  fieldName: string
+  childCorrelationField: PropRef
+  /** Whether the child query has an ORDER BY clause */
+  hasOrderBy: boolean
+  /** How the child result is materialized on the parent row */
+  materialization: IncludesMaterialization
+  /** Internal field used to unwrap scalar child selects */
+  scalarField?: string
+  /** Maps correlation key value → child Collection entry */
+  childRegistry: Map<unknown, ChildCollectionEntry>
+  /** Pending child changes: correlationKey → Map<childKey, Changes> */
+  pendingChildChanges: Map<unknown, Map<unknown, Changes<any>>>
+  /** Reverse index: correlation key → Set of parent collection keys */
+  correlationToParentKeys: Map<unknown, Set<unknown>>
+  /** Shared nested pipeline setups (one per nested includes level) */
+  nestedSetups?: Array<NestedIncludesSetup>
+  /** nestedCorrelationKey → parentCorrelationKey */
+  nestedRoutingIndex?: Map<unknown, unknown>
+  /** parentCorrelationKey → Set<nestedCorrelationKeys> */
+  nestedRoutingReverseIndex?: Map<unknown, Set<unknown>>
+}
+
+type ChildCollectionEntry = {
+  collection: Collection<any, any, any>
+  syncMethods: SyncMethods<any> | null
+  resultKeys: WeakMap<object, unknown>
+  orderByIndices: WeakMap<object, string> | null
+  /** Per-entry nested includes states (one per nested includes level) */
+  includesStates?: Array<IncludesOutputState>
+}
+
+function materializesInline(state: IncludesOutputState): boolean {
+  return state.materialization !== `collection`
+}
+
+function materializeIncludedValue(
+  state: IncludesOutputState,
+  entry: ChildCollectionEntry | undefined,
+): unknown {
+  if (!entry) {
+    if (state.materialization === `array`) {
+      return []
     }
+    if (state.materialization === `concat`) {
+      return ``
+    }
+    return undefined
+  }
+
+  if (state.materialization === `collection`) {
+    return entry.collection
+  }
+
+  const rows = [...entry.collection.toArray]
+  const values = state.scalarField
+    ? rows.map((row) => row?.[state.scalarField!])
+    : rows
+
+  if (state.materialization === `array`) {
+    return values
   }
 
-  // Helper function to recursively extract collections from a query
-  function extractFromQuery(q: any) {
-    // Extract from FROM clause
-    if (q.from) {
-      extractFromSource(q.from)
+  return values.map((value) => String(value ?? ``)).join(``)
+}
+
+/**
+ * Sets up shared buffers for nested includes pipelines.
+ * Instead of writing directly into a single shared IncludesOutputState,
+ * each nested pipeline writes into a buffer that is later drained per-entry.
+ */
+function setupNestedPipelines(
+  includes: Array<IncludesCompilationResult>,
+  syncState: SyncState,
+): Array<NestedIncludesSetup> {
+  return includes.map((entry) => {
+    const buffer: Map<unknown, Map<unknown, Changes<any>>> = new Map()
+
+    // Attach output callback that writes into the shared buffer
+    entry.pipeline.pipe(
+      output((data) => {
+        const messages = data.getInner()
+        syncState.messagesCount += messages.length
+
+        for (const [[childKey, tupleData], multiplicity] of messages) {
+          const [childResult, _orderByIndex, correlationKey, parentContext] =
+            tupleData as unknown as [
+              any,
+              string | undefined,
+              unknown,
+              Record<string, any> | null,
+            ]
+
+          const routingKey = computeRoutingKey(correlationKey, parentContext)
+
+          let byChild = buffer.get(routingKey)
+          if (!byChild) {
+            byChild = new Map()
+            buffer.set(routingKey, byChild)
+          }
+
+          const existing = byChild.get(childKey) || {
+            deletes: 0,
+            inserts: 0,
+            value: childResult,
+            orderByIndex: _orderByIndex,
+          }
+
+          if (multiplicity < 0) {
+            existing.deletes += Math.abs(multiplicity)
+          } else if (multiplicity > 0) {
+            existing.inserts += multiplicity
+            existing.value = childResult
+          }
+
+          byChild.set(childKey, existing)
+        }
+      }),
+    )
+
+    const setup: NestedIncludesSetup = {
+      compilationResult: entry,
+      buffer,
+    }
+
+    // Recursively set up deeper levels
+    if (entry.childCompilationResult.includes) {
+      setup.nestedSetups = setupNestedPipelines(
+        entry.childCompilationResult.includes,
+        syncState,
+      )
+    }
+
+    return setup
+  })
+}
+
+/**
+ * Creates fresh per-entry IncludesOutputState array from NestedIncludesSetup array.
+ * Each entry gets its own isolated state for nested includes.
+ */
+function createPerEntryIncludesStates(
+  setups: Array<NestedIncludesSetup>,
+): Array<IncludesOutputState> {
+  return setups.map((setup) => {
+    const state: IncludesOutputState = {
+      fieldName: setup.compilationResult.fieldName,
+      childCorrelationField: setup.compilationResult.childCorrelationField,
+      hasOrderBy: setup.compilationResult.hasOrderBy,
+      materialization: setup.compilationResult.materialization,
+      scalarField: setup.compilationResult.scalarField,
+      childRegistry: new Map(),
+      pendingChildChanges: new Map(),
+      correlationToParentKeys: new Map(),
+    }
+
+    if (setup.nestedSetups) {
+      state.nestedSetups = setup.nestedSetups
+      state.nestedRoutingIndex = new Map()
+      state.nestedRoutingReverseIndex = new Map()
     }
 
-    // Extract from JOIN clauses
-    if (q.join && Array.isArray(q.join)) {
-      for (const joinClause of q.join) {
-        if (joinClause.from) {
-          extractFromSource(joinClause.from)
+    return state
+  })
+}
+
+/**
+ * Drains shared buffers into per-entry states using the routing index.
+ * Returns the set of parent correlation keys that had changes routed to them.
+ */
+function drainNestedBuffers(state: IncludesOutputState): Set<unknown> {
+  const dirtyCorrelationKeys = new Set<unknown>()
+
+  if (!state.nestedSetups) return dirtyCorrelationKeys
+
+  for (let i = 0; i < state.nestedSetups.length; i++) {
+    const setup = state.nestedSetups[i]!
+    const toDelete: Array<unknown> = []
+
+    for (const [nestedCorrelationKey, childChanges] of setup.buffer) {
+      const parentCorrelationKey =
+        state.nestedRoutingIndex!.get(nestedCorrelationKey)
+      if (parentCorrelationKey === undefined) {
+        // Unroutable — parent not yet seen; keep in buffer
+        continue
+      }
+
+      const entry = state.childRegistry.get(parentCorrelationKey)
+      if (!entry || !entry.includesStates) {
+        continue
+      }
+
+      // Route changes into this entry's per-entry state at position i
+      const entryState = entry.includesStates[i]!
+      for (const [childKey, changes] of childChanges) {
+        let byChild = entryState.pendingChildChanges.get(nestedCorrelationKey)
+        if (!byChild) {
+          byChild = new Map()
+          entryState.pendingChildChanges.set(nestedCorrelationKey, byChild)
+        }
+        const existing = byChild.get(childKey)
+        if (existing) {
+          existing.inserts += changes.inserts
+          existing.deletes += changes.deletes
+          if (changes.inserts > 0) {
+            existing.value = changes.value
+            if (changes.orderByIndex !== undefined) {
+              existing.orderByIndex = changes.orderByIndex
+            }
+          }
+        } else {
+          byChild.set(childKey, { ...changes })
         }
       }
+
+      dirtyCorrelationKeys.add(parentCorrelationKey)
+      toDelete.push(nestedCorrelationKey)
+    }
+
+    for (const key of toDelete) {
+      setup.buffer.delete(key)
     }
   }
 
-  // Start extraction from the root query
-  extractFromQuery(query)
+  return dirtyCorrelationKeys
+}
+
+/**
+ * Updates the routing index after processing child changes.
+ * Maps nested correlation keys to parent correlation keys so that
+ * grandchild changes can be routed to the correct per-entry state.
+ */
+function updateRoutingIndex(
+  state: IncludesOutputState,
+  correlationKey: unknown,
+  childChanges: Map<unknown, Changes<any>>,
+): void {
+  if (!state.nestedSetups) return
+
+  for (const setup of state.nestedSetups) {
+    for (const [, change] of childChanges) {
+      if (change.inserts > 0) {
+        // Read the nested routing key from the INCLUDES_ROUTING stamp.
+        // Must use the composite routing key (not raw correlationKey) to match
+        // how nested buffers are keyed by computeRoutingKey.
+        const nestedRouting =
+          change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName]
+        const nestedCorrelationKey = nestedRouting?.correlationKey
+        const nestedParentContext = nestedRouting?.parentContext ?? null
+        const nestedRoutingKey = computeRoutingKey(
+          nestedCorrelationKey,
+          nestedParentContext,
+        )
+
+        if (nestedCorrelationKey != null) {
+          state.nestedRoutingIndex!.set(nestedRoutingKey, correlationKey)
+          let reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey)
+          if (!reverseSet) {
+            reverseSet = new Set()
+            state.nestedRoutingReverseIndex!.set(correlationKey, reverseSet)
+          }
+          reverseSet.add(nestedRoutingKey)
+        }
+      } else if (change.deletes > 0 && change.inserts === 0) {
+        // Remove from routing index
+        const nestedRouting2 =
+          change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName]
+        const nestedCorrelationKey = nestedRouting2?.correlationKey
+        const nestedParentContext2 = nestedRouting2?.parentContext ?? null
+        const nestedRoutingKey = computeRoutingKey(
+          nestedCorrelationKey,
+          nestedParentContext2,
+        )
+
+        if (nestedCorrelationKey != null) {
+          state.nestedRoutingIndex!.delete(nestedRoutingKey)
+          const reverseSet =
+            state.nestedRoutingReverseIndex!.get(correlationKey)
+          if (reverseSet) {
+            reverseSet.delete(nestedRoutingKey)
+            if (reverseSet.size === 0) {
+              state.nestedRoutingReverseIndex!.delete(correlationKey)
+            }
+          }
+        }
+      }
+    }
+  }
+}
 
-  return collections
+/**
+ * Cleans routing index entries when a parent is deleted.
+ * Uses the reverse index to find and remove all nested routing entries.
+ */
+function cleanRoutingIndexOnDelete(
+  state: IncludesOutputState,
+  correlationKey: unknown,
+): void {
+  if (!state.nestedRoutingReverseIndex) return
+
+  const nestedKeys = state.nestedRoutingReverseIndex.get(correlationKey)
+  if (nestedKeys) {
+    for (const nestedKey of nestedKeys) {
+      state.nestedRoutingIndex!.delete(nestedKey)
+    }
+    state.nestedRoutingReverseIndex.delete(correlationKey)
+  }
 }
 
 /**
- * Helper function to extract the collection that is referenced in the query's FROM clause.
- * The FROM clause may refer directly to a collection or indirectly to a subquery.
+ * Recursively checks whether any nested buffer has pending changes.
  */
-function extractCollectionFromSource(query: any): Collection<any, any, any> {
-  const from = query.from
-
-  if (from.type === `collectionRef`) {
-    return from.collection
-  } else if (from.type === `queryRef`) {
-    // Recursively extract from subquery
-    return extractCollectionFromSource(from.query)
+function hasNestedBufferChanges(setups: Array<NestedIncludesSetup>): boolean {
+  for (const setup of setups) {
+    if (setup.buffer.size > 0) return true
+    if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups))
+      return true
   }
+  return false
+}
 
-  throw new Error(
-    `Failed to extract collection. Invalid FROM clause: ${JSON.stringify(query)}`,
-  )
+/**
+ * Computes a composite routing key from correlation key and parent context.
+ * When parentContext is null (no parent filters), returns the raw correlationKey
+ * for zero behavioral change on existing queries.
+ */
+function computeRoutingKey(
+  correlationKey: unknown,
+  parentContext: Record<string, any> | null,
+): unknown {
+  if (parentContext == null) return correlationKey
+  return JSON.stringify([correlationKey, parentContext])
 }
 
 /**
- * Extracts all aliases used for each collection across the entire query tree.
- *
- * Traverses the QueryIR recursively to build a map from collection ID to all aliases
- * that reference that collection. This is essential for self-join support, where the
- * same collection may be referenced multiple times with different aliases.
- *
- * For example, given a query like:
- * ```ts
- * q.from({ employee: employeesCollection })
- *   .join({ manager: employeesCollection }, ({ employee, manager }) =>
- *     eq(employee.managerId, manager.id)
- *   )
- * ```
- *
- * This function would return:
- * ```
- * Map { "employees" => Set { "employee", "manager" } }
- * ```
- *
- * @param query - The query IR to extract aliases from
- * @returns A map from collection ID to the set of all aliases referencing that collection
+ * Creates a child Collection entry for includes subqueries.
+ * The child Collection is a full-fledged Collection instance that starts syncing immediately.
  */
-function extractCollectionAliases(query: QueryIR): Map<string, Set<string>> {
-  const aliasesById = new Map<string, Set<string>>()
+function createChildCollectionEntry(
+  parentId: string,
+  fieldName: string,
+  correlationKey: unknown,
+  hasOrderBy: boolean,
+  nestedSetups?: Array<NestedIncludesSetup>,
+): ChildCollectionEntry {
+  const resultKeys = new WeakMap<object, unknown>()
+  const orderByIndices = hasOrderBy ? new WeakMap<object, string>() : null
+  let syncMethods: SyncMethods<any> | null = null
+
+  const compare = orderByIndices
+    ? createOrderByComparator(orderByIndices)
+    : undefined
+
+  const collection = createCollection<any, string | number>({
+    id: `__child-collection:${parentId}-${fieldName}-${serializeValue(correlationKey)}`,
+    getKey: (item: any) => resultKeys.get(item) as string | number,
+    compare,
+    sync: {
+      rowUpdateMode: `full`,
+      sync: (methods) => {
+        syncMethods = methods
+        return () => {
+          syncMethods = null
+        }
+      },
+    },
+    startSync: true,
+  })
 
-  function recordAlias(source: any) {
-    if (!source) return
+  const entry: ChildCollectionEntry = {
+    collection,
+    get syncMethods() {
+      return syncMethods
+    },
+    resultKeys,
+    orderByIndices,
+  }
 
-    if (source.type === `collectionRef`) {
-      const { id } = source.collection
-      const existing = aliasesById.get(id)
-      if (existing) {
-        existing.add(source.alias)
-      } else {
-        aliasesById.set(id, new Set([source.alias]))
+  if (nestedSetups) {
+    entry.includesStates = createPerEntryIncludesStates(nestedSetups)
+  }
+
+  return entry
+}
+
+/**
+ * Flushes includes state using a bottom-up per-entry approach.
+ * Five phases ensure correct ordering:
+ *   1. Parent INSERTs — create child entries with per-entry nested states
+ *   2. Child changes — apply to child Collections, update routing index
+ *   3. Drain nested buffers — route buffered grandchild changes to per-entry states
+ *   4. Flush per-entry states — recursively flush nested includes on each entry
+ *   5. Parent DELETEs — clean up child entries and routing index
+ */
+function flushIncludesState(
+  includesState: Array<IncludesOutputState>,
+  parentCollection: Collection<any, any, any>,
+  parentId: string,
+  parentChanges: Map<unknown, Changes<any>> | null,
+  parentSyncMethods: SyncMethods<any> | null,
+): void {
+  for (const state of includesState) {
+    // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent
+    if (parentChanges) {
+      for (const [parentKey, changes] of parentChanges) {
+        if (changes.inserts > 0) {
+          const parentResult = changes.value
+          // Extract routing info from INCLUDES_ROUTING symbol (set by compiler)
+          const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName]
+          const correlationKey = routing?.correlationKey
+          const parentContext = routing?.parentContext ?? null
+          const routingKey = computeRoutingKey(correlationKey, parentContext)
+
+          if (correlationKey != null) {
+            // Ensure child Collection exists for this routing key
+            if (!state.childRegistry.has(routingKey)) {
+              const entry = createChildCollectionEntry(
+                parentId,
+                state.fieldName,
+                routingKey,
+                state.hasOrderBy,
+                state.nestedSetups,
+              )
+              state.childRegistry.set(routingKey, entry)
+            }
+            // Update reverse index: routing key → parent keys
+            let parentKeys = state.correlationToParentKeys.get(routingKey)
+            if (!parentKeys) {
+              parentKeys = new Set()
+              state.correlationToParentKeys.set(routingKey, parentKeys)
+            }
+            parentKeys.add(parentKey)
+
+            const childValue = materializeIncludedValue(
+              state,
+              state.childRegistry.get(routingKey),
+            )
+            parentResult[state.fieldName] = childValue
+
+            // Parent rows may already be materialized in the live collection by the
+            // time includes state is flushed, so update the stored row as well.
+            const storedParent = parentCollection.get(parentKey as any)
+            if (storedParent && storedParent !== parentResult) {
+              storedParent[state.fieldName] = childValue
+            }
+          }
+        }
       }
-    } else if (source.type === `queryRef`) {
-      traverse(source.query)
     }
-  }
 
-  function traverse(q?: QueryIR) {
-    if (!q) return
+    // Track affected correlation keys for inline materializations before clearing child changes.
+    const affectedCorrelationKeys = materializesInline(state)
+      ? new Set<unknown>(state.pendingChildChanges.keys())
+      : null
+
+    // Phase 2: Child changes — apply to child Collections
+    // Track which entries had child changes and capture their childChanges maps
+    const entriesWithChildChanges = new Map<
+      unknown,
+      { entry: ChildCollectionEntry; childChanges: Map<unknown, Changes<any>> }
+    >()
+    if (state.pendingChildChanges.size > 0) {
+      for (const [correlationKey, childChanges] of state.pendingChildChanges) {
+        // Ensure child Collection exists for this correlation key
+        let entry = state.childRegistry.get(correlationKey)
+        if (!entry) {
+          entry = createChildCollectionEntry(
+            parentId,
+            state.fieldName,
+            correlationKey,
+            state.hasOrderBy,
+            state.nestedSetups,
+          )
+          state.childRegistry.set(correlationKey, entry)
+        }
+
+        if (state.materialization === `collection`) {
+          attachChildCollectionToParent(
+            parentCollection,
+            state.fieldName,
+            correlationKey,
+            state.correlationToParentKeys,
+            entry.collection,
+          )
+        }
+
+        // Apply child changes to the child Collection
+        if (entry.syncMethods) {
+          entry.syncMethods.begin()
+          for (const [childKey, change] of childChanges) {
+            entry.resultKeys.set(change.value, childKey)
+            if (entry.orderByIndices && change.orderByIndex !== undefined) {
+              entry.orderByIndices.set(change.value, change.orderByIndex)
+            }
+            if (change.inserts > 0 && change.deletes === 0) {
+              entry.syncMethods.write({ value: change.value, type: `insert` })
+            } else if (
+              change.inserts > change.deletes ||
+              (change.inserts === change.deletes &&
+                entry.syncMethods.collection.has(
+                  entry.syncMethods.collection.getKeyFromItem(change.value),
+                ))
+            ) {
+              entry.syncMethods.write({ value: change.value, type: `update` })
+            } else if (change.deletes > 0) {
+              entry.syncMethods.write({ value: change.value, type: `delete` })
+            }
+          }
+          entry.syncMethods.commit()
+        }
+
+        // Update routing index for nested includes
+        updateRoutingIndex(state, correlationKey, childChanges)
+
+        entriesWithChildChanges.set(correlationKey, { entry, childChanges })
+      }
+      state.pendingChildChanges.clear()
+    }
+
+    // Phase 3: Drain nested buffers — route buffered grandchild changes to per-entry states
+    const dirtyFromBuffers = drainNestedBuffers(state)
+
+    // Phase 4: Flush per-entry states
+    // First: entries that had child changes in Phase 2
+    for (const [, { entry, childChanges }] of entriesWithChildChanges) {
+      if (entry.includesStates) {
+        flushIncludesState(
+          entry.includesStates,
+          entry.collection,
+          entry.collection.id,
+          childChanges,
+          entry.syncMethods,
+        )
+      }
+    }
+    // Then: entries that only had buffer-routed changes (no child changes at this level)
+    for (const correlationKey of dirtyFromBuffers) {
+      if (entriesWithChildChanges.has(correlationKey)) continue
+      const entry = state.childRegistry.get(correlationKey)
+      if (entry?.includesStates) {
+        flushIncludesState(
+          entry.includesStates,
+          entry.collection,
+          entry.collection.id,
+          null,
+          entry.syncMethods,
+        )
+      }
+    }
 
-    recordAlias(q.from)
+    // For inline materializations: re-emit affected parents with updated snapshots.
+    // We mutate items in-place (so collection.get() reflects changes immediately)
+    // and emit UPDATE events directly. We bypass the sync methods because
+    // commitPendingTransactions compares previous vs new visible state using
+    // deepEquals, but in-place mutation means both sides reference the same
+    // object, so the comparison always returns true and suppresses the event.
+    const inlineReEmitKeys = materializesInline(state)
+      ? new Set([...(affectedCorrelationKeys || []), ...dirtyFromBuffers])
+      : null
+    if (parentSyncMethods && inlineReEmitKeys && inlineReEmitKeys.size > 0) {
+      const events: Array<ChangeMessage<any>> = []
+      for (const correlationKey of inlineReEmitKeys) {
+        const parentKeys = state.correlationToParentKeys.get(correlationKey)
+        if (!parentKeys) continue
+        const entry = state.childRegistry.get(correlationKey)
+        for (const parentKey of parentKeys) {
+          const item = parentCollection.get(parentKey as any)
+          if (item) {
+            const key = parentSyncMethods.collection.getKeyFromItem(item)
+            // Capture previous value before in-place mutation
+            const previousValue = { ...item }
+            item[state.fieldName] = materializeIncludedValue(state, entry)
+            events.push({
+              type: `update`,
+              key,
+              value: item,
+              previousValue,
+            })
+          }
+        }
+      }
+      if (events.length > 0) {
+        // Emit directly — the in-place mutation already updated the data in
+        // syncedData, so we only need to notify subscribers.
+        const changesManager = (parentCollection as any)._changes as {
+          emitEvents: (
+            changes: Array<ChangeMessage<any>>,
+            forceEmit?: boolean,
+          ) => void
+        }
+        changesManager.emitEvents(events, true)
+      }
+    }
 
-    if (q.join) {
-      for (const joinClause of q.join) {
-        recordAlias(joinClause.from)
+    // Phase 5: Parent DELETEs — dispose child Collections and clean up
+    if (parentChanges) {
+      for (const [parentKey, changes] of parentChanges) {
+        if (changes.deletes > 0 && changes.inserts === 0) {
+          const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName]
+          const correlationKey = routing?.correlationKey
+          const parentContext = routing?.parentContext ?? null
+          const routingKey = computeRoutingKey(correlationKey, parentContext)
+          if (correlationKey != null) {
+            // Clean up reverse index first, only delete child collection
+            // when the last parent referencing it is removed
+            const parentKeys = state.correlationToParentKeys.get(routingKey)
+            if (parentKeys) {
+              parentKeys.delete(parentKey)
+              if (parentKeys.size === 0) {
+                cleanRoutingIndexOnDelete(state, routingKey)
+                state.childRegistry.delete(routingKey)
+                state.correlationToParentKeys.delete(routingKey)
+              }
+            }
+          }
+        }
       }
     }
   }
 
-  traverse(query)
+  // Clean up the internal routing stamp from parent/child results
+  if (parentChanges) {
+    for (const [, changes] of parentChanges) {
+      delete changes.value[INCLUDES_ROUTING]
+    }
+  }
+}
 
-  return aliasesById
+/**
+ * Checks whether any includes state has pending changes that need to be flushed.
+ * Checks direct pending child changes and shared nested buffers.
+ */
+function hasPendingIncludesChanges(
+  states: Array<IncludesOutputState>,
+): boolean {
+  for (const state of states) {
+    if (state.pendingChildChanges.size > 0) return true
+    if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups))
+      return true
+  }
+  return false
+}
+
+/**
+ * Attaches a child Collection to parent rows that match a given correlation key.
+ * Uses the reverse index to look up parent keys directly instead of scanning.
+ */
+function attachChildCollectionToParent(
+  parentCollection: Collection<any, any, any>,
+  fieldName: string,
+  correlationKey: unknown,
+  correlationToParentKeys: Map<unknown, Set<unknown>>,
+  childCollection: Collection<any, any, any>,
+): void {
+  const parentKeys = correlationToParentKeys.get(correlationKey)
+  if (!parentKeys) return
+
+  for (const parentKey of parentKeys) {
+    const item = parentCollection.get(parentKey as any)
+    if (item) {
+      item[fieldName] = childCollection
+    }
+  }
 }
 
 function accumulateChanges<T>(