@tanstack/db 0.5.32 → 0.6.0

package/dist/esm/query/live/collection-config-builder.js

@@ -1,12 +1,13 @@
-import { D2, output } from "@tanstack/db-ivm";
-import { compileQuery } from "../compiler/index.js";
-import { buildQuery, getQueryIR } from "../builder/index.js";
+import { D2, output, serializeValue } from "@tanstack/db-ivm";
+import { compileQuery, INCLUDES_ROUTING } from "../compiler/index.js";
+import { createCollection } from "../../collection/index.js";
 import { SetWindowRequiresOrderByError, MissingAliasInputsError } from "../../errors.js";
 import { transactionScopedScheduler } from "../../scheduler.js";
 import { getActiveTransaction } from "../../transactions.js";
 import { CollectionSubscriber } from "./collection-subscriber.js";
 import { getCollectionBuilder } from "./collection-registry.js";
 import { LIVE_QUERY_INTERNAL } from "./internal.js";
+import { buildQueryFromConfig, extractCollectionsFromQuery, extractCollectionAliases, extractCollectionFromSource } from "./utils.js";
 let liveQueryCollectionCounter = 0;
 class CollectionConfigBuilder {
   constructor(config) {
@@ -25,7 +26,10 @@ class CollectionConfigBuilder {
     this.lazySources = /* @__PURE__ */ new Set();
     this.optimizableOrderByCollections = {};
     this.id = config.id || `live-query-${++liveQueryCollectionCounter}`;
-    this.query = buildQueryFromConfig(config);
+    this.query = buildQueryFromConfig({
+      query: config.query,
+      requireObjectResult: true
+    });
     this.collections = extractCollectionsFromQuery(this.query);
     const collectionAliasesById = extractCollectionAliases(this.query);
     this.collectionByAlias = {};
@@ -360,6 +364,7 @@ class CollectionConfigBuilder {
       this.inputsCache = void 0;
       this.pipelineCache = void 0;
       this.sourceWhereClausesCache = void 0;
+      this.includesCache = void 0;
       this.lazySources.clear();
       this.optimizableOrderByCollections = {};
       this.lazySourcesCallbacks = {};
@@ -397,6 +402,7 @@ class CollectionConfigBuilder {
     this.pipelineCache = compilation.pipeline;
     this.sourceWhereClausesCache = compilation.sourceWhereClauses;
     this.compiledAliasToCollectionId = compilation.aliasToCollectionId;
+    this.includesCache = compilation.includes;
     const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter(
       (alias) => !Object.hasOwn(this.inputsCache, alias)
     );
@@ -425,8 +431,14 @@ class CollectionConfigBuilder {
         messages.reduce(accumulateChanges, pendingChanges);
       })
     );
+    const includesState = this.setupIncludesOutput(
+      this.includesCache,
+      syncState
+    );
     syncState.flushPendingChanges = () => {
-      if (pendingChanges.size === 0) {
+      const hasParentChanges = pendingChanges.size > 0;
+      const hasChildChanges = hasPendingIncludesChanges(includesState);
+      if (!hasParentChanges && !hasChildChanges) {
         return;
       }
       let changesToApply = pendingChanges;
@@ -450,10 +462,19 @@ class CollectionConfigBuilder {
         }
         changesToApply = merged;
       }
-      begin();
-      changesToApply.forEach(this.applyChanges.bind(this, config));
-      commit();
+      if (hasParentChanges) {
+        begin();
+        changesToApply.forEach(this.applyChanges.bind(this, config));
+        commit();
+      }
       pendingChanges = /* @__PURE__ */ new Map();
+      flushIncludesState(
+        includesState,
+        config.collection,
+        this.id,
+        hasParentChanges ? changesToApply : null,
+        config
+      );
     };
     graph.finalize();
     syncState.graph = graph;
@@ -461,6 +482,65 @@ class CollectionConfigBuilder {
     syncState.pipeline = pipeline;
     return syncState;
   }
+  /**
+   * Sets up output callbacks for includes child pipelines.
+   * Each includes entry gets its own output callback that accumulates child changes,
+   * and a child registry that maps correlation key → child Collection.
+   */
+  setupIncludesOutput(includesEntries, syncState) {
+    if (!includesEntries || includesEntries.length === 0) {
+      return [];
+    }
+    return includesEntries.map((entry) => {
+      const state = {
+        fieldName: entry.fieldName,
+        childCorrelationField: entry.childCorrelationField,
+        hasOrderBy: entry.hasOrderBy,
+        materialization: entry.materialization,
+        scalarField: entry.scalarField,
+        childRegistry: /* @__PURE__ */ new Map(),
+        pendingChildChanges: /* @__PURE__ */ new Map(),
+        correlationToParentKeys: /* @__PURE__ */ new Map()
+      };
+      entry.pipeline.pipe(
+        output((data) => {
+          const messages = data.getInner();
+          syncState.messagesCount += messages.length;
+          for (const [[childKey, tupleData], multiplicity] of messages) {
+            const [childResult, _orderByIndex, correlationKey, parentContext] = tupleData;
+            const routingKey = computeRoutingKey(correlationKey, parentContext);
+            let byChild = state.pendingChildChanges.get(routingKey);
+            if (!byChild) {
+              byChild = /* @__PURE__ */ new Map();
+              state.pendingChildChanges.set(routingKey, byChild);
+            }
+            const existing = byChild.get(childKey) || {
+              deletes: 0,
+              inserts: 0,
+              value: childResult,
+              orderByIndex: _orderByIndex
+            };
+            if (multiplicity < 0) {
+              existing.deletes += Math.abs(multiplicity);
+            } else if (multiplicity > 0) {
+              existing.inserts += multiplicity;
+              existing.value = childResult;
+            }
+            byChild.set(childKey, existing);
+          }
+        })
+      );
+      if (entry.childCompilationResult.includes) {
+        state.nestedSetups = setupNestedPipelines(
+          entry.childCompilationResult.includes,
+          syncState
+        );
+        state.nestedRoutingIndex = /* @__PURE__ */ new Map();
+        state.nestedRoutingReverseIndex = /* @__PURE__ */ new Map();
+      }
+      return state;
+    });
+  }
   applyChanges(config, changes, key) {
     const { write, collection } = config;
     const { deletes, inserts, value, orderByIndex } = changes;
@@ -588,12 +668,6 @@ class CollectionConfigBuilder {
     return loadSubsetDataCallbacks;
   }
 }
-function buildQueryFromConfig(config) {
-  if (typeof config.query === `function`) {
-    return buildQuery(config.query);
-  }
-  return getQueryIR(config.query);
-}
 function createOrderByComparator(orderByIndices) {
   return (val1, val2) => {
     const index1 = orderByIndices.get(val1);
@@ -610,68 +684,421 @@ function createOrderByComparator(orderByIndices) {
     return 0;
   };
 }
-function extractCollectionsFromQuery(query) {
-  const collections = {};
-  function extractFromSource(source) {
-    if (source.type === `collectionRef`) {
-      collections[source.collection.id] = source.collection;
-    } else if (source.type === `queryRef`) {
-      extractFromQuery(source.query);
-    }
-  }
-  function extractFromQuery(q) {
-    if (q.from) {
-      extractFromSource(q.from);
-    }
-    if (q.join && Array.isArray(q.join)) {
-      for (const joinClause of q.join) {
-        if (joinClause.from) {
-          extractFromSource(joinClause.from);
+function materializesInline(state) {
+  return state.materialization !== `collection`;
+}
+function materializeIncludedValue(state, entry) {
+  if (!entry) {
+    if (state.materialization === `array`) {
+      return [];
+    }
+    if (state.materialization === `concat`) {
+      return ``;
+    }
+    return void 0;
+  }
+  if (state.materialization === `collection`) {
+    return entry.collection;
+  }
+  const rows = [...entry.collection.toArray];
+  const values = state.scalarField ? rows.map((row) => row?.[state.scalarField]) : rows;
+  if (state.materialization === `array`) {
+    return values;
+  }
+  return values.map((value) => String(value ?? ``)).join(``);
+}
+function setupNestedPipelines(includes, syncState) {
+  return includes.map((entry) => {
+    const buffer = /* @__PURE__ */ new Map();
+    entry.pipeline.pipe(
+      output((data) => {
+        const messages = data.getInner();
+        syncState.messagesCount += messages.length;
+        for (const [[childKey, tupleData], multiplicity] of messages) {
+          const [childResult, _orderByIndex, correlationKey, parentContext] = tupleData;
+          const routingKey = computeRoutingKey(correlationKey, parentContext);
+          let byChild = buffer.get(routingKey);
+          if (!byChild) {
+            byChild = /* @__PURE__ */ new Map();
+            buffer.set(routingKey, byChild);
+          }
+          const existing = byChild.get(childKey) || {
+            deletes: 0,
+            inserts: 0,
+            value: childResult,
+            orderByIndex: _orderByIndex
+          };
+          if (multiplicity < 0) {
+            existing.deletes += Math.abs(multiplicity);
+          } else if (multiplicity > 0) {
+            existing.inserts += multiplicity;
+            existing.value = childResult;
+          }
+          byChild.set(childKey, existing);
+        }
+      })
+    );
+    const setup = {
+      compilationResult: entry,
+      buffer
+    };
+    if (entry.childCompilationResult.includes) {
+      setup.nestedSetups = setupNestedPipelines(
+        entry.childCompilationResult.includes,
+        syncState
+      );
+    }
+    return setup;
+  });
+}
+function createPerEntryIncludesStates(setups) {
+  return setups.map((setup) => {
+    const state = {
+      fieldName: setup.compilationResult.fieldName,
+      childCorrelationField: setup.compilationResult.childCorrelationField,
+      hasOrderBy: setup.compilationResult.hasOrderBy,
+      materialization: setup.compilationResult.materialization,
+      scalarField: setup.compilationResult.scalarField,
+      childRegistry: /* @__PURE__ */ new Map(),
+      pendingChildChanges: /* @__PURE__ */ new Map(),
+      correlationToParentKeys: /* @__PURE__ */ new Map()
+    };
+    if (setup.nestedSetups) {
+      state.nestedSetups = setup.nestedSetups;
+      state.nestedRoutingIndex = /* @__PURE__ */ new Map();
+      state.nestedRoutingReverseIndex = /* @__PURE__ */ new Map();
+    }
+    return state;
+  });
+}
+function drainNestedBuffers(state) {
+  const dirtyCorrelationKeys = /* @__PURE__ */ new Set();
+  if (!state.nestedSetups) return dirtyCorrelationKeys;
+  for (let i = 0; i < state.nestedSetups.length; i++) {
+    const setup = state.nestedSetups[i];
+    const toDelete = [];
+    for (const [nestedCorrelationKey, childChanges] of setup.buffer) {
+      const parentCorrelationKey = state.nestedRoutingIndex.get(nestedCorrelationKey);
+      if (parentCorrelationKey === void 0) {
+        continue;
+      }
+      const entry = state.childRegistry.get(parentCorrelationKey);
+      if (!entry || !entry.includesStates) {
+        continue;
+      }
+      const entryState = entry.includesStates[i];
+      for (const [childKey, changes] of childChanges) {
+        let byChild = entryState.pendingChildChanges.get(nestedCorrelationKey);
+        if (!byChild) {
+          byChild = /* @__PURE__ */ new Map();
+          entryState.pendingChildChanges.set(nestedCorrelationKey, byChild);
+        }
+        const existing = byChild.get(childKey);
+        if (existing) {
+          existing.inserts += changes.inserts;
+          existing.deletes += changes.deletes;
+          if (changes.inserts > 0) {
+            existing.value = changes.value;
+            if (changes.orderByIndex !== void 0) {
+              existing.orderByIndex = changes.orderByIndex;
+            }
+          }
+        } else {
+          byChild.set(childKey, { ...changes });
         }
       }
+      dirtyCorrelationKeys.add(parentCorrelationKey);
+      toDelete.push(nestedCorrelationKey);
+    }
+    for (const key of toDelete) {
+      setup.buffer.delete(key);
     }
   }
-  extractFromQuery(query);
-  return collections;
-}
-function extractCollectionFromSource(query) {
-  const from = query.from;
-  if (from.type === `collectionRef`) {
-    return from.collection;
-  } else if (from.type === `queryRef`) {
-    return extractCollectionFromSource(from.query);
-  }
-  throw new Error(
-    `Failed to extract collection. Invalid FROM clause: ${JSON.stringify(query)}`
-  );
+  return dirtyCorrelationKeys;
 }
-function extractCollectionAliases(query) {
-  const aliasesById = /* @__PURE__ */ new Map();
-  function recordAlias(source) {
-    if (!source) return;
-    if (source.type === `collectionRef`) {
-      const { id } = source.collection;
-      const existing = aliasesById.get(id);
-      if (existing) {
-        existing.add(source.alias);
-      } else {
-        aliasesById.set(id, /* @__PURE__ */ new Set([source.alias]));
+function updateRoutingIndex(state, correlationKey, childChanges) {
+  if (!state.nestedSetups) return;
+  for (const setup of state.nestedSetups) {
+    for (const [, change] of childChanges) {
+      if (change.inserts > 0) {
+        const nestedRouting = change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName];
+        const nestedCorrelationKey = nestedRouting?.correlationKey;
+        const nestedParentContext = nestedRouting?.parentContext ?? null;
+        const nestedRoutingKey = computeRoutingKey(
+          nestedCorrelationKey,
+          nestedParentContext
+        );
+        if (nestedCorrelationKey != null) {
+          state.nestedRoutingIndex.set(nestedRoutingKey, correlationKey);
+          let reverseSet = state.nestedRoutingReverseIndex.get(correlationKey);
+          if (!reverseSet) {
+            reverseSet = /* @__PURE__ */ new Set();
+            state.nestedRoutingReverseIndex.set(correlationKey, reverseSet);
+          }
+          reverseSet.add(nestedRoutingKey);
+        }
+      } else if (change.deletes > 0 && change.inserts === 0) {
+        const nestedRouting2 = change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName];
+        const nestedCorrelationKey = nestedRouting2?.correlationKey;
+        const nestedParentContext2 = nestedRouting2?.parentContext ?? null;
+        const nestedRoutingKey = computeRoutingKey(
+          nestedCorrelationKey,
+          nestedParentContext2
+        );
+        if (nestedCorrelationKey != null) {
+          state.nestedRoutingIndex.delete(nestedRoutingKey);
+          const reverseSet = state.nestedRoutingReverseIndex.get(correlationKey);
+          if (reverseSet) {
+            reverseSet.delete(nestedRoutingKey);
+            if (reverseSet.size === 0) {
+              state.nestedRoutingReverseIndex.delete(correlationKey);
+            }
+          }
+        }
       }
-    } else if (source.type === `queryRef`) {
-      traverse(source.query);
     }
   }
-  function traverse(q) {
-    if (!q) return;
-    recordAlias(q.from);
-    if (q.join) {
-      for (const joinClause of q.join) {
-        recordAlias(joinClause.from);
+}
+function cleanRoutingIndexOnDelete(state, correlationKey) {
+  if (!state.nestedRoutingReverseIndex) return;
+  const nestedKeys = state.nestedRoutingReverseIndex.get(correlationKey);
+  if (nestedKeys) {
+    for (const nestedKey of nestedKeys) {
+      state.nestedRoutingIndex.delete(nestedKey);
+    }
+    state.nestedRoutingReverseIndex.delete(correlationKey);
+  }
+}
+function hasNestedBufferChanges(setups) {
+  for (const setup of setups) {
+    if (setup.buffer.size > 0) return true;
+    if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups))
+      return true;
+  }
+  return false;
+}
+function computeRoutingKey(correlationKey, parentContext) {
+  if (parentContext == null) return correlationKey;
+  return JSON.stringify([correlationKey, parentContext]);
+}
+function createChildCollectionEntry(parentId, fieldName, correlationKey, hasOrderBy, nestedSetups) {
+  const resultKeys = /* @__PURE__ */ new WeakMap();
+  const orderByIndices = hasOrderBy ? /* @__PURE__ */ new WeakMap() : null;
+  let syncMethods = null;
+  const compare = orderByIndices ? createOrderByComparator(orderByIndices) : void 0;
+  const collection = createCollection({
+    id: `__child-collection:${parentId}-${fieldName}-${serializeValue(correlationKey)}`,
+    getKey: (item) => resultKeys.get(item),
+    compare,
+    sync: {
+      rowUpdateMode: `full`,
+      sync: (methods) => {
+        syncMethods = methods;
+        return () => {
+          syncMethods = null;
+        };
       }
+    },
+    startSync: true
+  });
+  const entry = {
+    collection,
+    get syncMethods() {
+      return syncMethods;
+    },
+    resultKeys,
+    orderByIndices
+  };
+  if (nestedSetups) {
+    entry.includesStates = createPerEntryIncludesStates(nestedSetups);
+  }
+  return entry;
+}
+function flushIncludesState(includesState, parentCollection, parentId, parentChanges, parentSyncMethods) {
+  for (const state of includesState) {
+    if (parentChanges) {
+      for (const [parentKey, changes] of parentChanges) {
+        if (changes.inserts > 0) {
+          const parentResult = changes.value;
+          const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName];
+          const correlationKey = routing?.correlationKey;
+          const parentContext = routing?.parentContext ?? null;
+          const routingKey = computeRoutingKey(correlationKey, parentContext);
+          if (correlationKey != null) {
+            if (!state.childRegistry.has(routingKey)) {
+              const entry = createChildCollectionEntry(
+                parentId,
+                state.fieldName,
+                routingKey,
+                state.hasOrderBy,
+                state.nestedSetups
+              );
+              state.childRegistry.set(routingKey, entry);
+            }
+            let parentKeys = state.correlationToParentKeys.get(routingKey);
+            if (!parentKeys) {
+              parentKeys = /* @__PURE__ */ new Set();
+              state.correlationToParentKeys.set(routingKey, parentKeys);
+            }
+            parentKeys.add(parentKey);
+            const childValue = materializeIncludedValue(
+              state,
+              state.childRegistry.get(routingKey)
+            );
+            parentResult[state.fieldName] = childValue;
+            const storedParent = parentCollection.get(parentKey);
+            if (storedParent && storedParent !== parentResult) {
+              storedParent[state.fieldName] = childValue;
+            }
+          }
+        }
+      }
+    }
+    const affectedCorrelationKeys = materializesInline(state) ? new Set(state.pendingChildChanges.keys()) : null;
+    const entriesWithChildChanges = /* @__PURE__ */ new Map();
+    if (state.pendingChildChanges.size > 0) {
+      for (const [correlationKey, childChanges] of state.pendingChildChanges) {
+        let entry = state.childRegistry.get(correlationKey);
+        if (!entry) {
+          entry = createChildCollectionEntry(
+            parentId,
+            state.fieldName,
+            correlationKey,
+            state.hasOrderBy,
+            state.nestedSetups
+          );
+          state.childRegistry.set(correlationKey, entry);
+        }
+        if (state.materialization === `collection`) {
+          attachChildCollectionToParent(
+            parentCollection,
+            state.fieldName,
+            correlationKey,
+            state.correlationToParentKeys,
+            entry.collection
+          );
+        }
+        if (entry.syncMethods) {
+          entry.syncMethods.begin();
+          for (const [childKey, change] of childChanges) {
+            entry.resultKeys.set(change.value, childKey);
+            if (entry.orderByIndices && change.orderByIndex !== void 0) {
+              entry.orderByIndices.set(change.value, change.orderByIndex);
+            }
+            if (change.inserts > 0 && change.deletes === 0) {
+              entry.syncMethods.write({ value: change.value, type: `insert` });
+            } else if (change.inserts > change.deletes || change.inserts === change.deletes && entry.syncMethods.collection.has(
+              entry.syncMethods.collection.getKeyFromItem(change.value)
+            )) {
+              entry.syncMethods.write({ value: change.value, type: `update` });
+            } else if (change.deletes > 0) {
+              entry.syncMethods.write({ value: change.value, type: `delete` });
+            }
+          }
+          entry.syncMethods.commit();
+        }
+        updateRoutingIndex(state, correlationKey, childChanges);
+        entriesWithChildChanges.set(correlationKey, { entry, childChanges });
+      }
+      state.pendingChildChanges.clear();
+    }
+    const dirtyFromBuffers = drainNestedBuffers(state);
+    for (const [, { entry, childChanges }] of entriesWithChildChanges) {
+      if (entry.includesStates) {
+        flushIncludesState(
+          entry.includesStates,
+          entry.collection,
+          entry.collection.id,
+          childChanges,
+          entry.syncMethods
+        );
+      }
+    }
+    for (const correlationKey of dirtyFromBuffers) {
+      if (entriesWithChildChanges.has(correlationKey)) continue;
+      const entry = state.childRegistry.get(correlationKey);
+      if (entry?.includesStates) {
+        flushIncludesState(
+          entry.includesStates,
+          entry.collection,
+          entry.collection.id,
+          null,
+          entry.syncMethods
+        );
+      }
+    }
+    const inlineReEmitKeys = materializesInline(state) ? /* @__PURE__ */ new Set([...affectedCorrelationKeys || [], ...dirtyFromBuffers]) : null;
+    if (parentSyncMethods && inlineReEmitKeys && inlineReEmitKeys.size > 0) {
+      const events = [];
+      for (const correlationKey of inlineReEmitKeys) {
+        const parentKeys = state.correlationToParentKeys.get(correlationKey);
+        if (!parentKeys) continue;
+        const entry = state.childRegistry.get(correlationKey);
+        for (const parentKey of parentKeys) {
+          const item = parentCollection.get(parentKey);
+          if (item) {
+            const key = parentSyncMethods.collection.getKeyFromItem(item);
+            const previousValue = { ...item };
+            item[state.fieldName] = materializeIncludedValue(state, entry);
+            events.push({
+              type: `update`,
+              key,
+              value: item,
+              previousValue
+            });
+          }
+        }
+      }
+      if (events.length > 0) {
+        const changesManager = parentCollection._changes;
+        changesManager.emitEvents(events, true);
+      }
+    }
+    if (parentChanges) {
+      for (const [parentKey, changes] of parentChanges) {
+        if (changes.deletes > 0 && changes.inserts === 0) {
+          const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName];
+          const correlationKey = routing?.correlationKey;
+          const parentContext = routing?.parentContext ?? null;
+          const routingKey = computeRoutingKey(correlationKey, parentContext);
+          if (correlationKey != null) {
+            const parentKeys = state.correlationToParentKeys.get(routingKey);
+            if (parentKeys) {
+              parentKeys.delete(parentKey);
+              if (parentKeys.size === 0) {
+                cleanRoutingIndexOnDelete(state, routingKey);
+                state.childRegistry.delete(routingKey);
+                state.correlationToParentKeys.delete(routingKey);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  if (parentChanges) {
+    for (const [, changes] of parentChanges) {
+      delete changes.value[INCLUDES_ROUTING];
+    }
+  }
+}
+function hasPendingIncludesChanges(states) {
+  for (const state of states) {
+    if (state.pendingChildChanges.size > 0) return true;
+    if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups))
+      return true;
+  }
+  return false;
+}
+function attachChildCollectionToParent(parentCollection, fieldName, correlationKey, correlationToParentKeys, childCollection) {
+  const parentKeys = correlationToParentKeys.get(correlationKey);
+  if (!parentKeys) return;
+  for (const parentKey of parentKeys) {
+    const item = parentCollection.get(parentKey);
+    if (item) {
+      item[fieldName] = childCollection;
     }
   }
-  traverse(query);
-  return aliasesById;
 }
 function accumulateChanges(acc, [[key, tupleData], multiplicity]) {
   const [value, orderByIndex] = tupleData;