@tanstack/db 0.5.31 → 0.5.33

package/dist/esm/query/effect.js

@@ -0,0 +1,602 @@
+import { D2, output } from "@tanstack/db-ivm";
+import { transactionScopedScheduler } from "../scheduler.js";
+import { getActiveTransaction } from "../transactions.js";
+import { compileQuery } from "./compiler/index.js";
+import { normalizeExpressionPaths, normalizeOrderByPaths } from "./compiler/expressions.js";
+import { getCollectionBuilder } from "./live/collection-registry.js";
+import { buildQueryFromConfig, extractCollectionsFromQuery, extractCollectionAliases, splitUpdates, filterDuplicateInserts, sendChangesToInput, computeSubscriptionOrderByHints, computeOrderedLoadCursor, trackBiggestSentValue } from "./live/utils.js";
+let effectCounter = 0;
+function createEffect(config) {
+  const id = config.id ?? `live-query-effect-${++effectCounter}`;
+  const abortController = new AbortController();
+  const ctx = {
+    effectId: id,
+    signal: abortController.signal
+  };
+  const inFlightHandlers = /* @__PURE__ */ new Set();
+  let disposed = false;
+  const onBatchProcessed = (events) => {
+    if (disposed) return;
+    if (events.length === 0) return;
+    if (config.onBatch) {
+      try {
+        const result = config.onBatch(events, ctx);
+        if (result instanceof Promise) {
+          const tracked = result.catch((error) => {
+            reportError(error, events[0], config.onError);
+          });
+          trackPromise(tracked, inFlightHandlers);
+        }
+      } catch (error) {
+        reportError(error, events[0], config.onError);
+      }
+    }
+    for (const event of events) {
+      if (abortController.signal.aborted) break;
+      const handler = getHandlerForEvent(event, config);
+      if (!handler) continue;
+      try {
+        const result = handler(event, ctx);
+        if (result instanceof Promise) {
+          const tracked = result.catch((error) => {
+            reportError(error, event, config.onError);
+          });
+          trackPromise(tracked, inFlightHandlers);
+        }
+      } catch (error) {
+        reportError(error, event, config.onError);
+      }
+    }
+  };
+  const dispose = async () => {
+    if (disposed) return;
+    disposed = true;
+    abortController.abort();
+    runner.dispose();
+    if (inFlightHandlers.size > 0) {
+      await Promise.allSettled([...inFlightHandlers]);
+    }
+  };
+  const runner = new EffectPipelineRunner({
+    query: config.query,
+    skipInitial: config.skipInitial ?? false,
+    onBatchProcessed,
+    onSourceError: (error) => {
+      if (disposed) return;
+      if (config.onSourceError) {
+        try {
+          config.onSourceError(error);
+        } catch (callbackError) {
+          console.error(
+            `[Effect '${id}'] onSourceError callback threw:`,
+            callbackError
+          );
+        }
+      } else {
+        console.error(`[Effect '${id}'] ${error.message}. Disposing effect.`);
+      }
+      dispose();
+    }
+  });
+  runner.start();
+  return {
+    dispose,
+    get disposed() {
+      return disposed;
+    }
+  };
+}
+class EffectPipelineRunner {
+  constructor(config) {
+    this.compiledAliasToCollectionId = {};
+    this.subscriptions = {};
+    this.lazySourcesCallbacks = {};
+    this.lazySources = /* @__PURE__ */ new Set();
+    this.optimizableOrderByCollections = {};
+    this.biggestSentValue = /* @__PURE__ */ new Map();
+    this.lastLoadRequestKey = /* @__PURE__ */ new Map();
+    this.unsubscribeCallbacks = /* @__PURE__ */ new Set();
+    this.sentToD2KeysByAlias = /* @__PURE__ */ new Map();
+    this.pendingChanges = /* @__PURE__ */ new Map();
+    this.initialLoadComplete = false;
+    this.subscribedToAllCollections = false;
+    this.builderDependencies = /* @__PURE__ */ new Set();
+    this.aliasDependencies = {};
+    this.isGraphRunning = false;
+    this.disposed = false;
+    this.deferredCleanup = false;
+    this.skipInitial = config.skipInitial;
+    this.onBatchProcessed = config.onBatchProcessed;
+    this.onSourceError = config.onSourceError;
+    this.query = buildQueryFromConfig({ query: config.query });
+    this.collections = extractCollectionsFromQuery(this.query);
+    const aliasesById = extractCollectionAliases(this.query);
+    this.collectionByAlias = {};
+    for (const [collectionId, aliases] of aliasesById.entries()) {
+      const collection = this.collections[collectionId];
+      if (!collection) continue;
+      for (const alias of aliases) {
+        this.collectionByAlias[alias] = collection;
+      }
+    }
+    this.compilePipeline();
+  }
+  /** Compile the D2 graph and query pipeline */
+  compilePipeline() {
+    this.graph = new D2();
+    this.inputs = Object.fromEntries(
+      Object.keys(this.collectionByAlias).map((alias) => [
+        alias,
+        this.graph.newInput()
+      ])
+    );
+    const compilation = compileQuery(
+      this.query,
+      this.inputs,
+      this.collections,
+      // These mutable objects are captured by reference. The join compiler
+      // reads them later when the graph runs, so they must be populated
+      // (in start()) before the first graph run.
+      this.subscriptions,
+      this.lazySourcesCallbacks,
+      this.lazySources,
+      this.optimizableOrderByCollections,
+      () => {
+      }
+      // setWindowFn (no-op — effects don't paginate)
+    );
+    this.pipeline = compilation.pipeline;
+    this.sourceWhereClauses = compilation.sourceWhereClauses;
+    this.compiledAliasToCollectionId = compilation.aliasToCollectionId;
+    this.pipeline.pipe(
+      output((data) => {
+        const messages = data.getInner();
+        messages.reduce(accumulateEffectChanges, this.pendingChanges);
+      })
+    );
+    this.graph.finalize();
+  }
+  /** Subscribe to source collections and start processing */
+  start() {
+    const compiledAliases = Object.entries(this.compiledAliasToCollectionId);
+    if (compiledAliases.length === 0) {
+      return;
+    }
+    if (!this.skipInitial) {
+      this.initialLoadComplete = true;
+    }
+    const pendingBuffers = /* @__PURE__ */ new Map();
+    for (const [alias, collectionId] of compiledAliases) {
+      const collection = this.collectionByAlias[alias] ?? this.collections[collectionId];
+      this.sentToD2KeysByAlias.set(alias, /* @__PURE__ */ new Set());
+      const dependencyBuilder = getCollectionBuilder(collection);
+      if (dependencyBuilder) {
+        this.aliasDependencies[alias] = [dependencyBuilder];
+        this.builderDependencies.add(dependencyBuilder);
+      } else {
+        this.aliasDependencies[alias] = [];
+      }
+      const whereClause = this.sourceWhereClauses?.get(alias);
+      const whereExpression = whereClause ? normalizeExpressionPaths(whereClause, alias) : void 0;
+      const buffer = [];
+      pendingBuffers.set(alias, buffer);
+      const isLazy = this.lazySources.has(alias);
+      const orderByInfo = this.getOrderByInfoForAlias(alias);
+      const changeCallback = orderByInfo ? (changes) => {
+        if (pendingBuffers.has(alias)) {
+          pendingBuffers.get(alias).push(changes);
+        } else {
+          this.trackSentValues(alias, changes, orderByInfo.comparator);
+          const split = [...splitUpdates(changes)];
+          this.handleSourceChanges(alias, split);
+        }
+      } : (changes) => {
+        if (pendingBuffers.has(alias)) {
+          pendingBuffers.get(alias).push(changes);
+        } else {
+          this.handleSourceChanges(alias, changes);
+        }
+      };
+      const subscriptionOptions = this.buildSubscriptionOptions(
+        alias,
+        isLazy,
+        orderByInfo,
+        whereExpression
+      );
+      const subscription = collection.subscribeChanges(
+        changeCallback,
+        subscriptionOptions
+      );
+      this.subscriptions[alias] = subscription;
+      if (orderByInfo) {
+        this.requestInitialOrderedSnapshot(alias, orderByInfo, subscription);
+      }
+      this.unsubscribeCallbacks.add(() => {
+        subscription.unsubscribe();
+        delete this.subscriptions[alias];
+      });
+      const statusUnsubscribe = collection.on(`status:change`, (event) => {
+        if (this.disposed) return;
+        const { status } = event;
+        if (status === `error`) {
+          this.onSourceError(
+            new Error(
+              `Source collection '${collectionId}' entered error state`
+            )
+          );
+          return;
+        }
+        if (status === `cleaned-up`) {
+          this.onSourceError(
+            new Error(
+              `Source collection '${collectionId}' was cleaned up while effect depends on it`
+            )
+          );
+          return;
+        }
+        if (this.skipInitial && !this.initialLoadComplete && this.checkAllCollectionsReady()) {
+          this.initialLoadComplete = true;
+        }
+      });
+      this.unsubscribeCallbacks.add(statusUnsubscribe);
+    }
+    this.subscribedToAllCollections = true;
+    for (const [alias] of pendingBuffers) {
+      const buffer = pendingBuffers.get(alias);
+      pendingBuffers.delete(alias);
+      const orderByInfo = this.getOrderByInfoForAlias(alias);
+      for (const changes of buffer) {
+        if (orderByInfo) {
+          this.trackSentValues(alias, changes, orderByInfo.comparator);
+          const split = [...splitUpdates(changes)];
+          this.sendChangesToD2(alias, split);
+        } else {
+          this.sendChangesToD2(alias, changes);
+        }
+      }
+    }
+    this.runGraph();
+    if (this.skipInitial && !this.initialLoadComplete) {
+      if (this.checkAllCollectionsReady()) {
+        this.initialLoadComplete = true;
+      }
+    }
+  }
+  /** Handle incoming changes from a source collection */
+  handleSourceChanges(alias, changes) {
+    this.sendChangesToD2(alias, changes);
+    this.scheduleGraphRun(alias);
+  }
+  /**
+   * Schedule a graph run via the transaction-scoped scheduler.
+   *
+   * When called within a transaction, the run is deferred until the
+   * transaction flushes, coalescing multiple changes into a single graph
+   * execution. Without a transaction, the graph runs immediately.
+   *
+   * Dependencies are discovered from source collections that are themselves
+   * live query collections, ensuring parent queries run before effects.
+   */
+  scheduleGraphRun(alias) {
+    const contextId = getActiveTransaction()?.id;
+    const deps = new Set(this.builderDependencies);
+    if (alias) {
+      const aliasDeps = this.aliasDependencies[alias];
+      if (aliasDeps) {
+        for (const dep of aliasDeps) {
+          deps.add(dep);
+        }
+      }
+    }
+    if (contextId) {
+      for (const dep of deps) {
+        if (typeof dep === `object` && dep !== null && `scheduleGraphRun` in dep && typeof dep.scheduleGraphRun === `function`) {
+          dep.scheduleGraphRun(void 0, { contextId });
+        }
+      }
+    }
+    transactionScopedScheduler.schedule({
+      contextId,
+      jobId: this,
+      dependencies: deps,
+      run: () => this.executeScheduledGraphRun()
+    });
+  }
+  /**
+   * Called by the scheduler when dependencies are satisfied.
+   * Checks that the effect is still active before running.
+   */
+  executeScheduledGraphRun() {
+    if (this.disposed || !this.subscribedToAllCollections) return;
+    this.runGraph();
+  }
+  /**
+   * Send changes to the D2 input for the given alias.
+   * Returns the number of multiset entries sent.
+   */
+  sendChangesToD2(alias, changes) {
+    if (this.disposed || !this.inputs || !this.graph) return 0;
+    const input = this.inputs[alias];
+    if (!input) return 0;
+    const collection = this.collectionByAlias[alias];
+    if (!collection) return 0;
+    const sentKeys = this.sentToD2KeysByAlias.get(alias);
+    const filtered = filterDuplicateInserts(changes, sentKeys);
+    return sendChangesToInput(input, filtered, collection.config.getKey);
+  }
+  /**
+   * Run the D2 graph until quiescence, then emit accumulated events once.
+   *
+   * All output across the entire while-loop is accumulated into a single
+   * batch so that users see one `onBatchProcessed` invocation per scheduler
+   * run, even when ordered loading causes multiple graph steps.
+   */
+  runGraph() {
+    if (this.isGraphRunning || this.disposed || !this.graph) return;
+    this.isGraphRunning = true;
+    try {
+      while (this.graph.pendingWork()) {
+        this.graph.run();
+        if (this.disposed) break;
+        this.loadMoreIfNeeded();
+      }
+      this.flushPendingChanges();
+    } finally {
+      this.isGraphRunning = false;
+      if (this.deferredCleanup) {
+        this.deferredCleanup = false;
+        this.finalCleanup();
+      }
+    }
+  }
+  /** Classify accumulated changes into DeltaEvents and invoke the callback */
+  flushPendingChanges() {
+    if (this.pendingChanges.size === 0) return;
+    if (this.skipInitial && !this.initialLoadComplete) {
+      this.pendingChanges = /* @__PURE__ */ new Map();
+      return;
+    }
+    const events = [];
+    for (const [key, changes] of this.pendingChanges) {
+      const event = classifyDelta(key, changes);
+      if (event) {
+        events.push(event);
+      }
+    }
+    this.pendingChanges = /* @__PURE__ */ new Map();
+    if (events.length > 0) {
+      this.onBatchProcessed(events);
+    }
+  }
+  /** Check if all source collections are in the ready state */
+  checkAllCollectionsReady() {
+    return Object.values(this.collections).every(
+      (collection) => collection.isReady()
+    );
+  }
+  /**
+   * Build subscription options for an alias based on whether it uses ordered
+   * loading, is lazy, or should pass orderBy/limit hints.
+   */
+  buildSubscriptionOptions(alias, isLazy, orderByInfo, whereExpression) {
+    if (orderByInfo) {
+      return { includeInitialState: false, whereExpression };
+    }
+    const includeInitialState = !isLazy;
+    const hints = computeSubscriptionOrderByHints(this.query, alias);
+    return {
+      includeInitialState,
+      whereExpression,
+      ...hints.orderBy ? { orderBy: hints.orderBy } : {},
+      ...hints.limit !== void 0 ? { limit: hints.limit } : {}
+    };
+  }
+  /**
+   * Request the initial ordered snapshot for an alias.
+   * Uses requestLimitedSnapshot (index-based cursor) or requestSnapshot
+   * (full load with limit) depending on whether an index is available.
+   */
+  requestInitialOrderedSnapshot(alias, orderByInfo, subscription) {
+    const { orderBy, offset, limit, index } = orderByInfo;
+    const normalizedOrderBy = normalizeOrderByPaths(orderBy, alias);
+    if (index) {
+      subscription.setOrderByIndex(index);
+      subscription.requestLimitedSnapshot({
+        limit: offset + limit,
+        orderBy: normalizedOrderBy,
+        trackLoadSubsetPromise: false
+      });
+    } else {
+      subscription.requestSnapshot({
+        orderBy: normalizedOrderBy,
+        limit: offset + limit,
+        trackLoadSubsetPromise: false
+      });
+    }
+  }
+  /**
+   * Get orderBy optimization info for a given alias.
+   * Returns undefined if no optimization exists for this alias.
+   */
+  getOrderByInfoForAlias(alias) {
+    const collectionId = this.compiledAliasToCollectionId[alias];
+    if (!collectionId) return void 0;
+    const info = this.optimizableOrderByCollections[collectionId];
+    if (info && info.alias === alias) {
+      return info;
+    }
+    return void 0;
+  }
+  /**
+   * After each graph run step, check if any ordered query's topK operator
+   * needs more data. If so, load more rows via requestLimitedSnapshot.
+   */
+  loadMoreIfNeeded() {
+    for (const [, orderByInfo] of Object.entries(
+      this.optimizableOrderByCollections
+    )) {
+      if (!orderByInfo.dataNeeded) continue;
+      if (this.pendingOrderedLoadPromise) {
+        continue;
+      }
+      const n = orderByInfo.dataNeeded();
+      if (n > 0) {
+        this.loadNextItems(orderByInfo, n);
+      }
+    }
+  }
+  /**
+   * Load n more items from the source collection, starting from the cursor
+   * position (the biggest value sent so far).
+   */
+  loadNextItems(orderByInfo, n) {
+    const { alias } = orderByInfo;
+    const subscription = this.subscriptions[alias];
+    if (!subscription) return;
+    const cursor = computeOrderedLoadCursor(
+      orderByInfo,
+      this.biggestSentValue.get(alias),
+      this.lastLoadRequestKey.get(alias),
+      alias,
+      n
+    );
+    if (!cursor) return;
+    this.lastLoadRequestKey.set(alias, cursor.loadRequestKey);
+    subscription.requestLimitedSnapshot({
+      orderBy: cursor.normalizedOrderBy,
+      limit: n,
+      minValues: cursor.minValues,
+      trackLoadSubsetPromise: false,
+      onLoadSubsetResult: (loadResult) => {
+        if (loadResult instanceof Promise) {
+          this.pendingOrderedLoadPromise = loadResult;
+          loadResult.finally(() => {
+            if (this.pendingOrderedLoadPromise === loadResult) {
+              this.pendingOrderedLoadPromise = void 0;
+            }
+          });
+        }
+      }
+    });
+  }
+  /**
+   * Track the biggest value sent for a given ordered alias.
+   * Used for cursor-based pagination in loadNextItems.
+   */
+  trackSentValues(alias, changes, comparator) {
+    const sentKeys = this.sentToD2KeysByAlias.get(alias) ?? /* @__PURE__ */ new Set();
+    const result = trackBiggestSentValue(
+      changes,
+      this.biggestSentValue.get(alias),
+      sentKeys,
+      comparator
+    );
+    this.biggestSentValue.set(alias, result.biggest);
+    if (result.shouldResetLoadKey) {
+      this.lastLoadRequestKey.delete(alias);
+    }
+  }
+  /** Tear down subscriptions and clear state */
+  dispose() {
+    if (this.disposed) return;
+    this.disposed = true;
+    this.subscribedToAllCollections = false;
+    this.unsubscribeCallbacks.forEach((fn) => fn());
+    this.unsubscribeCallbacks.clear();
+    this.sentToD2KeysByAlias.clear();
+    this.pendingChanges.clear();
+    this.lazySources.clear();
+    this.builderDependencies.clear();
+    this.biggestSentValue.clear();
+    this.lastLoadRequestKey.clear();
+    this.pendingOrderedLoadPromise = void 0;
+    for (const key of Object.keys(this.lazySourcesCallbacks)) {
+      delete this.lazySourcesCallbacks[key];
+    }
+    for (const key of Object.keys(this.aliasDependencies)) {
+      delete this.aliasDependencies[key];
+    }
+    for (const key of Object.keys(this.optimizableOrderByCollections)) {
+      delete this.optimizableOrderByCollections[key];
+    }
+    if (this.isGraphRunning) {
+      this.deferredCleanup = true;
+    } else {
+      this.finalCleanup();
+    }
+  }
+  /** Clear graph references — called after graph run completes or immediately from dispose */
+  finalCleanup() {
+    this.graph = void 0;
+    this.inputs = void 0;
+    this.pipeline = void 0;
+    this.sourceWhereClauses = void 0;
+  }
+}
+function getHandlerForEvent(event, config) {
+  switch (event.type) {
+    case `enter`:
+      return config.onEnter;
+    case `exit`:
+      return config.onExit;
+    case `update`:
+      return config.onUpdate;
+  }
+}
+function accumulateEffectChanges(acc, [[key, tupleData], multiplicity]) {
+  const [value] = tupleData;
+  const changes = acc.get(key) || {
+    deletes: 0,
+    inserts: 0
+  };
+  if (multiplicity < 0) {
+    changes.deletes += Math.abs(multiplicity);
+    changes.deleteValue ??= value;
+  } else if (multiplicity > 0) {
+    changes.inserts += multiplicity;
+    changes.insertValue = value;
+  }
+  acc.set(key, changes);
+  return acc;
+}
+function classifyDelta(key, changes) {
+  const { inserts, deletes, insertValue, deleteValue } = changes;
+  if (inserts > 0 && deletes === 0) {
+    return { type: `enter`, key, value: insertValue };
+  }
+  if (deletes > 0 && inserts === 0) {
+    return { type: `exit`, key, value: deleteValue };
+  }
+  if (inserts > 0 && deletes > 0) {
+    return {
+      type: `update`,
+      key,
+      value: insertValue,
+      previousValue: deleteValue
+    };
+  }
+  return void 0;
+}
+function trackPromise(promise, inFlightHandlers) {
+  inFlightHandlers.add(promise);
+  promise.finally(() => {
+    inFlightHandlers.delete(promise);
+  });
+}
+function reportError(error, event, onError) {
+  const normalised = error instanceof Error ? error : new Error(String(error));
+  if (onError) {
+    try {
+      onError(normalised, event);
+    } catch (onErrorError) {
+      console.error(`[Effect] Error in onError handler:`, onErrorError);
+      console.error(`[Effect] Original error:`, normalised);
+    }
+  } else {
+    console.error(`[Effect] Unhandled error in handler:`, normalised);
+  }
+}
+export {
+  createEffect
+};
+//# sourceMappingURL=effect.js.map