@tanstack/db 0.4.7 → 0.4.9

package/dist/esm/query/live/collection-config-builder.d.ts

@@ -1,37 +1,128 @@
+import { SchedulerContextId } from '../../scheduler.js';
 import { CollectionSubscription } from '../../collection/subscription.js';
 import { OrderByOptimizationInfo } from '../compiler/order-by.js';
-import { CollectionConfigSingleRowOption, SyncConfig } from '../../types.js';
+import { CollectionConfigSingleRowOption, SyncConfig, UtilsRecord } from '../../types.js';
 import { Context, GetResult } from '../builder/types.js';
 import { BasicExpression, QueryIR } from '../ir.js';
 import { LazyCollectionCallbacks } from '../compiler/joins.js';
 import { FullSyncState, LiveQueryCollectionConfig } from './types.js';
+export type LiveQueryCollectionUtils = UtilsRecord & {
+    getRunCount: () => number;
+    getBuilder: () => CollectionConfigBuilder<any, any>;
+};
 export declare class CollectionConfigBuilder<TContext extends Context, TResult extends object = GetResult<TContext>> {
     private readonly config;
     private readonly id;
     readonly query: QueryIR;
     private readonly collections;
+    private readonly collectionByAlias;
+    private compiledAliasToCollectionId;
     private readonly resultKeys;
     private readonly orderByIndices;
     private readonly compare?;
     private isGraphRunning;
+    private runCount;
+    currentSyncConfig: Parameters<SyncConfig<TResult>[`sync`]>[0] | undefined;
+    currentSyncState: FullSyncState | undefined;
+    private isInErrorState;
+    private liveQueryCollection?;
+    private readonly aliasDependencies;
+    private readonly builderDependencies;
+    private readonly pendingGraphRuns;
+    private unsubscribeFromSchedulerClears?;
     private graphCache;
     private inputsCache;
     private pipelineCache;
-    collectionWhereClausesCache: Map<string, BasicExpression<boolean>> | undefined;
+    sourceWhereClausesCache: Map<string, BasicExpression<boolean>> | undefined;
     readonly subscriptions: Record<string, CollectionSubscription>;
-    lazyCollectionsCallbacks: Record<string, LazyCollectionCallbacks>;
-    readonly lazyCollections: Set<string>;
+    lazySourcesCallbacks: Record<string, LazyCollectionCallbacks>;
+    readonly lazySources: Set<string>;
     optimizableOrderByCollections: Record<string, OrderByOptimizationInfo>;
     constructor(config: LiveQueryCollectionConfig<TContext, TResult>);
-    getConfig(): CollectionConfigSingleRowOption<TResult>;
-    maybeRunGraph(config: Parameters<SyncConfig<TResult>[`sync`]>[0], syncState: FullSyncState, callback?: () => boolean): void;
+    getConfig(): CollectionConfigSingleRowOption<TResult> & {
+        utils: LiveQueryCollectionUtils;
+    };
+    /**
+     * Resolves a collection alias to its collection ID.
+     *
+     * Uses a two-tier lookup strategy:
+     * 1. First checks compiled aliases (includes subquery inner aliases)
+     * 2. Falls back to declared aliases from the query's from/join clauses
+     *
+     * @param alias - The alias to resolve (e.g., "employee", "manager")
+     * @returns The collection ID that the alias references
+     * @throws {Error} If the alias is not found in either lookup
+     */
+    getCollectionIdForAlias(alias: string): string;
+    isLazyAlias(alias: string): boolean;
+    maybeRunGraph(callback?: () => boolean): void;
+    /**
+     * Schedules a graph run with the transaction-scoped scheduler.
+     * Ensures each builder runs at most once per transaction, with automatic dependency tracking
+     * to run parent queries before child queries. Outside a transaction, runs immediately.
+     *
+     * Multiple calls during a transaction are coalesced into a single execution.
+     * Dependencies are auto-discovered from subscribed live queries, or can be overridden.
+     * Load callbacks are combined when entries merge.
+     *
+     * Uses the current sync session's config and syncState from instance properties.
+     *
+     * @param callback - Optional callback to load more data if needed (returns true when done)
+     * @param options - Optional scheduling configuration
+     * @param options.contextId - Transaction ID to group work; defaults to active transaction
+     * @param options.jobId - Unique identifier for this job; defaults to this builder instance
+     * @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies
+     * @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies
+     */
+    scheduleGraphRun(callback?: () => boolean, options?: {
+        contextId?: SchedulerContextId;
+        jobId?: unknown;
+        alias?: string;
+        dependencies?: Array<CollectionConfigBuilder<any, any>>;
+    }): void;
+    /**
+     * Clears pending graph run state for a specific context.
+     * Called when the scheduler clears a context (e.g., transaction rollback/abort).
+     */
+    clearPendingGraphRun(contextId: SchedulerContextId): void;
+    /**
+     * Executes a pending graph run. Called by the scheduler when dependencies are satisfied.
+     * Clears the pending state BEFORE execution so that any re-schedules during the run
+     * create fresh state and don't interfere with the current execution.
+     * Uses instance sync state - if sync has ended, gracefully returns without executing.
+     *
+     * @param contextId - Optional context ID to look up pending state
+     * @param pendingParam - For immediate execution (no context), pending state is passed directly
+     */
+    private executeGraphRun;
     private getSyncConfig;
+    incrementRunCount(): void;
+    getRunCount(): number;
     private syncFn;
+    /**
+     * Compiles the query pipeline with all declared aliases.
+     */
     private compileBasePipeline;
     private maybeCompileBasePipeline;
     private extendPipelineWithChangeProcessing;
     private applyChanges;
+    /**
+     * Handle status changes from source collections
+     */
+    private handleSourceStatusChange;
+    /**
+     * Update the live query status based on source collection statuses
+     */
+    private updateLiveQueryStatus;
+    /**
+     * Transition the live query to error state
+     */
+    private transitionToError;
     private allCollectionsReady;
-    private allCollectionsReadyOrInitialCommit;
+    /**
+     * Creates per-alias subscriptions enabling self-join support.
+     * Each alias gets its own subscription with independent filters, even for the same collection.
+     * Example: `{ employee: col, manager: col }` creates two separate subscriptions.
+     */
     private subscribeToAllCollections;
 }

package/dist/esm/query/live/collection-config-builder.js

@@ -1,21 +1,40 @@
 import { D2, output } from "@tanstack/db-ivm";
 import { compileQuery } from "../compiler/index.js";
 import { buildQuery, getQueryIR } from "../builder/index.js";
+import { MissingAliasInputsError } from "../../errors.js";
+import { transactionScopedScheduler } from "../../scheduler.js";
+import { getActiveTransaction } from "../../transactions.js";
 import { CollectionSubscriber } from "./collection-subscriber.js";
+import { getCollectionBuilder } from "./collection-registry.js";
 let liveQueryCollectionCounter = 0;
 class CollectionConfigBuilder {
   constructor(config) {
     this.config = config;
+    this.compiledAliasToCollectionId = {};
     this.resultKeys = /* @__PURE__ */ new WeakMap();
     this.orderByIndices = /* @__PURE__ */ new WeakMap();
     this.isGraphRunning = false;
+    this.runCount = 0;
+    this.isInErrorState = false;
+    this.aliasDependencies = {};
+    this.builderDependencies = /* @__PURE__ */ new Set();
+    this.pendingGraphRuns = /* @__PURE__ */ new Map();
     this.subscriptions = {};
-    this.lazyCollectionsCallbacks = {};
-    this.lazyCollections = /* @__PURE__ */ new Set();
+    this.lazySourcesCallbacks = {};
+    this.lazySources = /* @__PURE__ */ new Set();
     this.optimizableOrderByCollections = {};
     this.id = config.id || `live-query-${++liveQueryCollectionCounter}`;
     this.query = buildQueryFromConfig(config);
     this.collections = extractCollectionsFromQuery(this.query);
+    const collectionAliasesById = extractCollectionAliases(this.query);
+    this.collectionByAlias = {};
+    for (const [collectionId, aliases] of collectionAliasesById.entries()) {
+      const collection = this.collections[collectionId];
+      if (!collection) continue;
+      for (const alias of aliases) {
+        this.collectionByAlias[alias] = collection;
+      }
+    }
     if (this.query.orderBy && this.query.orderBy.length > 0) {
       this.compare = createOrderByComparator(this.orderByIndices);
     }
@@ -34,25 +53,63 @@ class CollectionConfigBuilder {
       onUpdate: this.config.onUpdate,
       onDelete: this.config.onDelete,
       startSync: this.config.startSync,
-      singleResult: this.query.singleResult
+      singleResult: this.query.singleResult,
+      utils: {
+        getRunCount: this.getRunCount.bind(this),
+        getBuilder: () => this
+      }
     };
   }
+  /**
+   * Resolves a collection alias to its collection ID.
+   *
+   * Uses a two-tier lookup strategy:
+   * 1. First checks compiled aliases (includes subquery inner aliases)
+   * 2. Falls back to declared aliases from the query's from/join clauses
+   *
+   * @param alias - The alias to resolve (e.g., "employee", "manager")
+   * @returns The collection ID that the alias references
+   * @throws {Error} If the alias is not found in either lookup
+   */
+  getCollectionIdForAlias(alias) {
+    const compiled = this.compiledAliasToCollectionId[alias];
+    if (compiled) {
+      return compiled;
+    }
+    const collection = this.collectionByAlias[alias];
+    if (collection) {
+      return collection.id;
+    }
+    throw new Error(`Unknown source alias "${alias}"`);
+  }
+  isLazyAlias(alias) {
+    return this.lazySources.has(alias);
+  }
   // The callback function is called after the graph has run.
   // This gives the callback a chance to load more data if needed,
   // that's used to optimize orderBy operators that set a limit,
   // in order to load some more data if we still don't have enough rows after the pipeline has run.
-  // That can happend because even though we load N rows, the pipeline might filter some of these rows out
+  // That can happen because even though we load N rows, the pipeline might filter some of these rows out
   // causing the orderBy operator to receive less than N rows or even no rows at all.
   // So this callback would notice that it doesn't have enough rows and load some more.
   // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.
-  maybeRunGraph(config, syncState, callback) {
+  maybeRunGraph(callback) {
     if (this.isGraphRunning) {
       return;
     }
+    if (!this.currentSyncConfig || !this.currentSyncState) {
+      throw new Error(
+        `maybeRunGraph called without active sync session. This should not happen.`
+      );
+    }
     this.isGraphRunning = true;
     try {
-      const { begin, commit, markReady } = config;
-      if (this.allCollectionsReadyOrInitialCommit() && syncState.subscribedToAllCollections) {
+      const { begin, commit } = this.currentSyncConfig;
+      const syncState = this.currentSyncState;
+      if (this.isInErrorState) {
+        return;
+      }
+      if (syncState.subscribedToAllCollections) {
         while (syncState.graph.pendingWork()) {
           syncState.graph.run();
           callback?.();
@@ -60,22 +117,136 @@ class CollectionConfigBuilder {
         if (syncState.messagesCount === 0) {
           begin();
           commit();
-        }
-        if (this.allCollectionsReady()) {
-          markReady();
+          this.updateLiveQueryStatus(this.currentSyncConfig);
         }
       }
     } finally {
       this.isGraphRunning = false;
     }
   }
+  /**
+   * Schedules a graph run with the transaction-scoped scheduler.
+   * Ensures each builder runs at most once per transaction, with automatic dependency tracking
+   * to run parent queries before child queries. Outside a transaction, runs immediately.
+   *
+   * Multiple calls during a transaction are coalesced into a single execution.
+   * Dependencies are auto-discovered from subscribed live queries, or can be overridden.
+   * Load callbacks are combined when entries merge.
+   *
+   * Uses the current sync session's config and syncState from instance properties.
+   *
+   * @param callback - Optional callback to load more data if needed (returns true when done)
+   * @param options - Optional scheduling configuration
+   * @param options.contextId - Transaction ID to group work; defaults to active transaction
+   * @param options.jobId - Unique identifier for this job; defaults to this builder instance
+   * @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies
+   * @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies
+   */
+  scheduleGraphRun(callback, options) {
+    const contextId = options?.contextId ?? getActiveTransaction()?.id;
+    const jobId = options?.jobId ?? this;
+    const dependentBuilders = (() => {
+      if (options?.dependencies) {
+        return options.dependencies;
+      }
+      const deps = new Set(this.builderDependencies);
+      if (options?.alias) {
+        const aliasDeps = this.aliasDependencies[options.alias];
+        if (aliasDeps) {
+          for (const dep of aliasDeps) {
+            deps.add(dep);
+          }
+        }
+      }
+      deps.delete(this);
+      return Array.from(deps);
+    })();
+    if (!this.currentSyncConfig || !this.currentSyncState) {
+      throw new Error(
+        `scheduleGraphRun called without active sync session. This should not happen.`
+      );
+    }
+    let pending = contextId ? this.pendingGraphRuns.get(contextId) : void 0;
+    if (!pending) {
+      pending = {
+        loadCallbacks: /* @__PURE__ */ new Set()
+      };
+      if (contextId) {
+        this.pendingGraphRuns.set(contextId, pending);
+      }
+    }
+    if (callback) {
+      pending.loadCallbacks.add(callback);
+    }
+    const pendingToPass = contextId ? void 0 : pending;
+    transactionScopedScheduler.schedule({
+      contextId,
+      jobId,
+      dependencies: dependentBuilders,
+      run: () => this.executeGraphRun(contextId, pendingToPass)
+    });
+  }
+  /**
+   * Clears pending graph run state for a specific context.
+   * Called when the scheduler clears a context (e.g., transaction rollback/abort).
+   */
+  clearPendingGraphRun(contextId) {
+    this.pendingGraphRuns.delete(contextId);
+  }
+  /**
+   * Executes a pending graph run. Called by the scheduler when dependencies are satisfied.
+   * Clears the pending state BEFORE execution so that any re-schedules during the run
+   * create fresh state and don't interfere with the current execution.
+   * Uses instance sync state - if sync has ended, gracefully returns without executing.
+   *
+   * @param contextId - Optional context ID to look up pending state
+   * @param pendingParam - For immediate execution (no context), pending state is passed directly
+   */
+  executeGraphRun(contextId, pendingParam) {
+    const pending = pendingParam ?? (contextId ? this.pendingGraphRuns.get(contextId) : void 0);
+    if (contextId) {
+      this.pendingGraphRuns.delete(contextId);
+    }
+    if (!pending) {
+      return;
+    }
+    if (!this.currentSyncConfig || !this.currentSyncState) {
+      return;
+    }
+    this.incrementRunCount();
+    const combinedLoader = () => {
+      let allDone = true;
+      let firstError;
+      pending.loadCallbacks.forEach((loader) => {
+        try {
+          allDone = loader() && allDone;
+        } catch (error) {
+          allDone = false;
+          firstError ??= error;
+        }
+      });
+      if (firstError) {
+        throw firstError;
+      }
+      return allDone;
+    };
+    this.maybeRunGraph(combinedLoader);
+  }
   getSyncConfig() {
     return {
       rowUpdateMode: `full`,
       sync: this.syncFn.bind(this)
     };
   }
+  incrementRunCount() {
+    this.runCount++;
+  }
+  getRunCount() {
+    return this.runCount;
+  }
   syncFn(config) {
+    this.liveQueryCollection = config.collection;
+    this.currentSyncConfig = config;
     const syncState = {
       messagesCount: 0,
       subscribedToAllCollections: false,
@@ -85,44 +256,66 @@ class CollectionConfigBuilder {
       config,
       syncState
     );
+    this.currentSyncState = fullSyncState;
+    this.unsubscribeFromSchedulerClears = transactionScopedScheduler.onClear(
+      (contextId) => {
+        this.clearPendingGraphRun(contextId);
+      }
+    );
     const loadMoreDataCallbacks = this.subscribeToAllCollections(
       config,
       fullSyncState
     );
-    this.maybeRunGraph(config, fullSyncState, loadMoreDataCallbacks);
+    this.scheduleGraphRun(loadMoreDataCallbacks);
     return () => {
       syncState.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe());
+      this.currentSyncConfig = void 0;
+      this.currentSyncState = void 0;
+      this.pendingGraphRuns.clear();
       this.graphCache = void 0;
       this.inputsCache = void 0;
       this.pipelineCache = void 0;
-      this.collectionWhereClausesCache = void 0;
-      this.lazyCollections.clear();
+      this.sourceWhereClausesCache = void 0;
+      this.lazySources.clear();
       this.optimizableOrderByCollections = {};
-      this.lazyCollectionsCallbacks = {};
+      this.lazySourcesCallbacks = {};
+      Object.keys(this.subscriptions).forEach(
+        (key) => delete this.subscriptions[key]
+      );
+      this.compiledAliasToCollectionId = {};
+      this.unsubscribeFromSchedulerClears?.();
+      this.unsubscribeFromSchedulerClears = void 0;
     };
   }
+  /**
+   * Compiles the query pipeline with all declared aliases.
+   */
   compileBasePipeline() {
     this.graphCache = new D2();
     this.inputsCache = Object.fromEntries(
-      Object.entries(this.collections).map(([key]) => [
-        key,
+      Object.keys(this.collectionByAlias).map((alias) => [
+        alias,
         this.graphCache.newInput()
       ])
     );
-    const {
-      pipeline: pipelineCache,
-      collectionWhereClauses: collectionWhereClausesCache
-    } = compileQuery(
+    const compilation = compileQuery(
       this.query,
       this.inputsCache,
       this.collections,
       this.subscriptions,
-      this.lazyCollectionsCallbacks,
-      this.lazyCollections,
+      this.lazySourcesCallbacks,
+      this.lazySources,
       this.optimizableOrderByCollections
     );
-    this.pipelineCache = pipelineCache;
-    this.collectionWhereClausesCache = collectionWhereClausesCache;
+    this.pipelineCache = compilation.pipeline;
+    this.sourceWhereClausesCache = compilation.sourceWhereClauses;
+    this.compiledAliasToCollectionId = compilation.aliasToCollectionId;
+    const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter(
+      (alias) => !Object.hasOwn(this.inputsCache, alias)
+    );
+    if (missingAliases.length > 0) {
+      throw new MissingAliasInputsError(missingAliases);
+    }
   }
   maybeCompileBasePipeline() {
     if (!this.graphCache || !this.inputsCache || !this.pipelineCache) {
@@ -188,40 +381,95 @@ class CollectionConfigBuilder {
       );
     }
   }
+  /**
+   * Handle status changes from source collections
+   */
+  handleSourceStatusChange(config, collectionId, event) {
+    const { status } = event;
+    if (status === `error`) {
+      this.transitionToError(
+        `Source collection '${collectionId}' entered error state`
+      );
+      return;
+    }
+    if (status === `cleaned-up`) {
+      this.transitionToError(
+        `Source collection '${collectionId}' was manually cleaned up while live query '${this.id}' depends on it. Live queries prevent automatic GC, so this was likely a manual cleanup() call.`
+      );
+      return;
+    }
+    this.updateLiveQueryStatus(config);
+  }
+  /**
+   * Update the live query status based on source collection statuses
+   */
+  updateLiveQueryStatus(config) {
+    const { markReady } = config;
+    if (this.isInErrorState) {
+      return;
+    }
+    if (this.allCollectionsReady()) {
+      markReady();
+    }
+  }
+  /**
+   * Transition the live query to error state
+   */
+  transitionToError(message) {
+    this.isInErrorState = true;
+    console.error(`[Live Query Error] ${message}`);
+    this.liveQueryCollection?._lifecycle.setStatus(`error`);
+  }
   allCollectionsReady() {
     return Object.values(this.collections).every(
       (collection) => collection.isReady()
     );
   }
-  allCollectionsReadyOrInitialCommit() {
-    return Object.values(this.collections).every(
-      (collection) => collection.status === `ready` || collection.status === `initialCommit`
-    );
-  }
+  /**
+   * Creates per-alias subscriptions enabling self-join support.
+   * Each alias gets its own subscription with independent filters, even for the same collection.
+   * Example: `{ employee: col, manager: col }` creates two separate subscriptions.
+   */
   subscribeToAllCollections(config, syncState) {
-    const loaders = Object.entries(this.collections).map(
-      ([collectionId, collection]) => {
-        const collectionSubscriber = new CollectionSubscriber(
-          collectionId,
-          collection,
-          config,
-          syncState,
-          this
-        );
-        const subscription = collectionSubscriber.subscribe();
-        this.subscriptions[collectionId] = subscription;
-        const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
-          collectionSubscriber,
-          subscription
-        );
-        return loadMore;
+    const compiledAliases = Object.entries(this.compiledAliasToCollectionId);
+    if (compiledAliases.length === 0) {
+      throw new Error(
+        `Compiler returned no alias metadata for query '${this.id}'. This should not happen; please report.`
+      );
+    }
+    const loaders = compiledAliases.map(([alias, collectionId]) => {
+      const collection = this.collectionByAlias[alias] ?? this.collections[collectionId];
+      const dependencyBuilder = getCollectionBuilder(collection);
+      if (dependencyBuilder && dependencyBuilder !== this) {
+        this.aliasDependencies[alias] = [dependencyBuilder];
+        this.builderDependencies.add(dependencyBuilder);
+      } else {
+        this.aliasDependencies[alias] = [];
       }
-    );
+      const collectionSubscriber = new CollectionSubscriber(
+        alias,
+        collectionId,
+        collection,
+        this
+      );
+      const statusUnsubscribe = collection.on(`status:change`, (event) => {
+        this.handleSourceStatusChange(config, collectionId, event);
+      });
+      syncState.unsubscribeCallbacks.add(statusUnsubscribe);
+      const subscription = collectionSubscriber.subscribe();
+      this.subscriptions[alias] = subscription;
+      const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(
+        collectionSubscriber,
+        subscription
+      );
+      return loadMore;
+    });
     const loadMoreDataCallback = () => {
       loaders.map((loader) => loader());
       return true;
     };
     syncState.subscribedToAllCollections = true;
+    this.updateLiveQueryStatus(config);
     return loadMoreDataCallback;
   }
 }
@@ -271,6 +519,34 @@ function extractCollectionsFromQuery(query) {
   extractFromQuery(query);
   return collections;
 }
+function extractCollectionAliases(query) {
+  const aliasesById = /* @__PURE__ */ new Map();
+  function recordAlias(source) {
+    if (!source) return;
+    if (source.type === `collectionRef`) {
+      const { id } = source.collection;
+      const existing = aliasesById.get(id);
+      if (existing) {
+        existing.add(source.alias);
+      } else {
+        aliasesById.set(id, /* @__PURE__ */ new Set([source.alias]));
+      }
+    } else if (source.type === `queryRef`) {
+      traverse(source.query);
+    }
+  }
+  function traverse(q) {
+    if (!q) return;
+    recordAlias(q.from);
+    if (q.join) {
+      for (const joinClause of q.join) {
+        recordAlias(joinClause.from);
+      }
+    }
+  }
+  traverse(query);
+  return aliasesById;
+}
 function accumulateChanges(acc, [[key, tupleData], multiplicity]) {
   const [value, orderByIndex] = tupleData;
   const changes = acc.get(key) || {