@tanstack/db 0.4.8 → 0.4.10

package/src/query/optimizer.ts

@@ -162,8 +162,8 @@ export interface GroupedWhereClauses {
 export interface OptimizationResult {
   /** The optimized query with WHERE clauses potentially moved to subqueries */
   optimizedQuery: QueryIR
-  /** Map of collection aliases to their extracted WHERE clauses for index optimization */
-  collectionWhereClauses: Map<string, BasicExpression<boolean>>
+  /** Map of source aliases to their extracted WHERE clauses for index optimization */
+  sourceWhereClauses: Map<string, BasicExpression<boolean>>
 }
 
 /**
@@ -184,14 +184,14 @@ export interface OptimizationResult {
  *   where: [eq(u.dept_id, 1), gt(p.views, 100)]
  * }
  *
- * const { optimizedQuery, collectionWhereClauses } = optimizeQuery(originalQuery)
+ * const { optimizedQuery, sourceWhereClauses } = optimizeQuery(originalQuery)
  * // Result: Single-source clauses moved to deepest possible subqueries
- * // collectionWhereClauses: Map { 'u' => eq(u.dept_id, 1), 'p' => gt(p.views, 100) }
+ * // sourceWhereClauses: Map { 'u' => eq(u.dept_id, 1), 'p' => gt(p.views, 100) }
  * ```
  */
 export function optimizeQuery(query: QueryIR): OptimizationResult {
-  // First, extract collection WHERE clauses before optimization
-  const collectionWhereClauses = extractCollectionWhereClauses(query)
+  // First, extract source WHERE clauses before optimization
+  const sourceWhereClauses = extractSourceWhereClauses(query)
 
   // Apply multi-level predicate pushdown with iterative convergence
   let optimized = query
@@ -214,7 +214,7 @@ export function optimizeQuery(query: QueryIR): OptimizationResult {
 
   return {
     optimizedQuery: cleaned,
-    collectionWhereClauses,
+    sourceWhereClauses,
   }
 }
 
@@ -224,16 +224,16 @@ export function optimizeQuery(query: QueryIR): OptimizationResult {
  * to specific collections, but only for simple queries without joins.
  *
  * @param query - The original QueryIR to analyze
- * @returns Map of collection aliases to their WHERE clauses
+ * @returns Map of source aliases to their WHERE clauses
  */
-function extractCollectionWhereClauses(
+function extractSourceWhereClauses(
   query: QueryIR
 ): Map<string, BasicExpression<boolean>> {
-  const collectionWhereClauses = new Map<string, BasicExpression<boolean>>()
+  const sourceWhereClauses = new Map<string, BasicExpression<boolean>>()
 
   // Only analyze queries that have WHERE clauses
   if (!query.where || query.where.length === 0) {
-    return collectionWhereClauses
+    return sourceWhereClauses
   }
 
   // Split all AND clauses at the root level for granular analysis
@@ -254,12 +254,12 @@ function extractCollectionWhereClauses(
     if (isCollectionReference(query, sourceAlias)) {
       // Check if the WHERE clause can be converted to collection-compatible format
       if (isConvertibleToCollectionFilter(whereClause)) {
-        collectionWhereClauses.set(sourceAlias, whereClause)
+        sourceWhereClauses.set(sourceAlias, whereClause)
       }
     }
   }
 
-  return collectionWhereClauses
+  return sourceWhereClauses
 }
 
 /**
@@ -782,8 +782,6 @@ function optimizeFromWithTracking(
     return new QueryRefClass(subQuery, from.alias)
   }
 
-  // Must be queryRef due to type system
-
   // SAFETY CHECK: Only check safety when pushing WHERE clauses into existing subqueries
   // We need to be careful about pushing WHERE clauses into subqueries that already have
   // aggregates, HAVING, or ORDER BY + LIMIT since that could change their semantics
@@ -793,6 +791,12 @@ function optimizeFromWithTracking(
     return new QueryRefClass(deepCopyQuery(from.query), from.alias)
   }
 
+  // Skip pushdown when a clause references a field that only exists via a renamed
+  // projection inside the subquery; leaving it outside preserves the alias mapping.
+  if (referencesAliasWithRemappedSelect(from.query, whereClause, from.alias)) {
+    return new QueryRefClass(deepCopyQuery(from.query), from.alias)
+  }
+
   // Add the WHERE clause to the existing subquery
   // Create a deep copy to ensure immutability
   const existingWhere = from.query.where || []
@@ -943,6 +947,72 @@ function whereReferencesComputedSelectFields(
   return false
 }
 
+/**
+ * Detects whether a WHERE clause references the subquery alias through fields that
+ * are re-exposed under different names (renamed SELECT projections or fnSelect output).
+ * In those cases we keep the clause at the outer level to avoid alias remapping bugs.
+ * TODO: in future we should handle this by rewriting the clause to use the subquery's
+ * internal field references, but it likely needs a wider refactor to do cleanly.
+ */
+function referencesAliasWithRemappedSelect(
+  subquery: QueryIR,
+  whereClause: BasicExpression<boolean>,
+  outerAlias: string
+): boolean {
+  const refs = collectRefs(whereClause)
+  // Only care about clauses that actually reference the outer alias.
+  if (refs.every((ref) => ref.path[0] !== outerAlias)) {
+    return false
+  }
+
+  // fnSelect always rewrites the row shape, so alias-safe pushdown is impossible.
+  if (subquery.fnSelect) {
+    return true
+  }
+
+  const select = subquery.select
+  // Without an explicit SELECT the clause still refers to the original collection.
+  if (!select) {
+    return false
+  }
+
+  for (const ref of refs) {
+    const path = ref.path
+    // Need at least alias + field to matter.
+    if (path.length < 2) continue
+    if (path[0] !== outerAlias) continue
+
+    const projected = select[path[1]!]
+    // Unselected fields can't be remapped, so skip - only care about fields in the SELECT.
+    if (!projected) continue
+
+    // Non-PropRef projections are computed values; cannot push down.
+    if (!(projected instanceof PropRef)) {
+      return true
+    }
+
+    // If the projection is just the alias (whole row) without a specific field,
+    // we can't verify whether the field we're referencing is being preserved or remapped.
+    if (projected.path.length < 2) {
+      return true
+    }
+
+    const [innerAlias, innerField] = projected.path
+
+    // Safe only when the projection points straight back to the same alias or the
+    // underlying source alias and preserves the field name.
+    if (innerAlias !== outerAlias && innerAlias !== subquery.from.alias) {
+      return true
+    }
+
+    if (innerField !== path[1]) {
+      return true
+    }
+  }
+
+  return false
+}
+
 /**
  * Helper function to combine multiple expressions with AND.
  *

package/src/scheduler.ts

@@ -0,0 +1,198 @@
+/**
+ * Identifier used to scope scheduled work. Maps to a transaction id for live queries.
+ */
+export type SchedulerContextId = string | symbol
+
+/**
+ * Options for {@link Scheduler.schedule}. Jobs are identified by `jobId` within a context
+ * and may declare dependencies.
+ */
+interface ScheduleOptions {
+  contextId?: SchedulerContextId
+  jobId: unknown
+  dependencies?: Iterable<unknown>
+  run: () => void
+}
+
+/**
+ * State per context. Queue preserves order, jobs hold run functions, dependencies track
+ * prerequisites, and completed records which jobs have run during the current flush.
+ */
+interface SchedulerContextState {
+  queue: Array<unknown>
+  jobs: Map<unknown, () => void>
+  dependencies: Map<unknown, Set<unknown>>
+  completed: Set<unknown>
+}
+
+/**
+ * Scoped scheduler that coalesces work by context and job.
+ *
+ * - **context** (e.g. transaction id) defines the batching boundary; work is queued until flushed.
+ * - **job id** deduplicates work within a context; scheduling the same job replaces the previous run function.
+ * - Without a context id, work executes immediately.
+ *
+ * Callers manage their own state; the scheduler only orchestrates execution order.
+ */
+export class Scheduler {
+  private contexts = new Map<SchedulerContextId, SchedulerContextState>()
+  private clearListeners = new Set<(contextId: SchedulerContextId) => void>()
+
+  /**
+   * Get or create the state bucket for a context.
+   */
+  private getOrCreateContext(
+    contextId: SchedulerContextId
+  ): SchedulerContextState {
+    let context = this.contexts.get(contextId)
+    if (!context) {
+      context = {
+        queue: [],
+        jobs: new Map(),
+        dependencies: new Map(),
+        completed: new Set(),
+      }
+      this.contexts.set(contextId, context)
+    }
+    return context
+  }
+
+  /**
+   * Schedule work. Without a context id, executes immediately.
+   * Otherwise queues the job to be flushed once dependencies are satisfied.
+   * Scheduling the same jobId again replaces the previous run function.
+   */
+  schedule({ contextId, jobId, dependencies, run }: ScheduleOptions): void {
+    if (typeof contextId === `undefined`) {
+      run()
+      return
+    }
+
+    const context = this.getOrCreateContext(contextId)
+
+    // If this is a new job, add it to the queue
+    if (!context.jobs.has(jobId)) {
+      context.queue.push(jobId)
+    }
+
+    // Store or replace the run function
+    context.jobs.set(jobId, run)
+
+    // Update dependencies
+    if (dependencies) {
+      const depSet = new Set<unknown>(dependencies)
+      depSet.delete(jobId)
+      context.dependencies.set(jobId, depSet)
+    } else if (!context.dependencies.has(jobId)) {
+      context.dependencies.set(jobId, new Set())
+    }
+
+    // Clear completion status since we're rescheduling
+    context.completed.delete(jobId)
+  }
+
+  /**
+   * Flush all queued work for a context. Jobs with unmet dependencies are retried.
+   * Throws if a pass completes without running any job (dependency cycle).
+   */
+  flush(contextId: SchedulerContextId): void {
+    const context = this.contexts.get(contextId)
+    if (!context) return
+
+    const { queue, jobs, dependencies, completed } = context
+
+    while (queue.length > 0) {
+      let ranThisPass = false
+      const jobsThisPass = queue.length
+
+      for (let i = 0; i < jobsThisPass; i++) {
+        const jobId = queue.shift()!
+        const run = jobs.get(jobId)
+        if (!run) {
+          dependencies.delete(jobId)
+          completed.delete(jobId)
+          continue
+        }
+
+        const deps = dependencies.get(jobId)
+        let ready = !deps
+        if (deps) {
+          ready = true
+          for (const dep of deps) {
+            if (dep !== jobId && !completed.has(dep)) {
+              ready = false
+              break
+            }
+          }
+        }
+
+        if (ready) {
+          jobs.delete(jobId)
+          dependencies.delete(jobId)
+          // Run the job. If it throws, we don't mark it complete, allowing the
+          // error to propagate while maintaining scheduler state consistency.
+          run()
+          completed.add(jobId)
+          ranThisPass = true
+        } else {
+          queue.push(jobId)
+        }
+      }
+
+      if (!ranThisPass) {
+        throw new Error(
+          `Scheduler detected unresolved dependencies for context ${String(
+            contextId
+          )}.`
+        )
+      }
+    }
+
+    this.contexts.delete(contextId)
+  }
+
+  /**
+   * Flush all contexts with pending work. Useful during tear-down.
+   */
+  flushAll(): void {
+    for (const contextId of Array.from(this.contexts.keys())) {
+      this.flush(contextId)
+    }
+  }
+
+  /** Clear all scheduled jobs for a context. */
+  clear(contextId: SchedulerContextId): void {
+    this.contexts.delete(contextId)
+    // Notify listeners that this context was cleared
+    this.clearListeners.forEach((listener) => listener(contextId))
+  }
+
+  /** Register a listener to be notified when a context is cleared. */
+  onClear(listener: (contextId: SchedulerContextId) => void): () => void {
+    this.clearListeners.add(listener)
+    return () => this.clearListeners.delete(listener)
+  }
+
+  /** Check if a context has pending jobs. */
+  hasPendingJobs(contextId: SchedulerContextId): boolean {
+    const context = this.contexts.get(contextId)
+    return !!context && context.jobs.size > 0
+  }
+
+  /** Remove a single job from a context and clean up its dependencies. */
+  clearJob(contextId: SchedulerContextId, jobId: unknown): void {
+    const context = this.contexts.get(contextId)
+    if (!context) return
+
+    context.jobs.delete(jobId)
+    context.dependencies.delete(jobId)
+    context.completed.delete(jobId)
+    context.queue = context.queue.filter((id) => id !== jobId)
+
+    if (context.jobs.size === 0) {
+      this.contexts.delete(contextId)
+    }
+  }
+}
+
+export const transactionScopedScheduler = new Scheduler()

package/src/transactions.ts

@@ -5,6 +5,7 @@ import {
   TransactionNotPendingCommitError,
   TransactionNotPendingMutateError,
 } from "./errors"
+import { transactionScopedScheduler } from "./scheduler.js"
 import type { Deferred } from "./deferred"
 import type {
   MutationFn,
@@ -179,11 +180,21 @@ export function getActiveTransaction(): Transaction | undefined {
 }
 
 function registerTransaction(tx: Transaction<any>) {
+  // Clear any stale work that may have been left behind if a previous mutate
+  // scope aborted before we could flush.
+  transactionScopedScheduler.clear(tx.id)
   transactionStack.push(tx)
 }
 
 function unregisterTransaction(tx: Transaction<any>) {
-  transactionStack = transactionStack.filter((t) => t.id !== tx.id)
+  // Always flush pending work for this transaction before removing it from
+  // the ambient stack – this runs even if the mutate callback throws.
+  // If flush throws (e.g., due to a job error), we still clean up the stack.
+  try {
+    transactionScopedScheduler.flush(tx.id)
+  } finally {
+    transactionStack = transactionStack.filter((t) => t.id !== tx.id)
+  }
 }
 
 function removeFromPendingList(tx: Transaction<any>) {

package/src/types.ts

@@ -3,6 +3,7 @@ import type { Collection } from "./collection/index.js"
 import type { StandardSchemaV1 } from "@standard-schema/spec"
 import type { Transaction } from "./transactions"
 import type { BasicExpression, OrderBy } from "./query/ir.js"
+import type { EventEmitter } from "./event-emitter.js"
 
 /**
  * Helper type to extract the output type from a standard schema
@@ -150,17 +151,83 @@ export type Row<TExtensions = never> = Record<string, Value<TExtensions>>
 
 export type OperationType = `insert` | `update` | `delete`
 
-export type OnLoadMoreOptions = {
+/**
+ * Subscription status values
+ */
+export type SubscriptionStatus = `ready` | `loadingSubset`
+
+/**
+ * Event emitted when subscription status changes
+ */
+export interface SubscriptionStatusChangeEvent {
+  type: `status:change`
+  subscription: Subscription
+  previousStatus: SubscriptionStatus
+  status: SubscriptionStatus
+}
+
+/**
+ * Event emitted when subscription status changes to a specific status
+ */
+export interface SubscriptionStatusEvent<T extends SubscriptionStatus> {
+  type: `status:${T}`
+  subscription: Subscription
+  previousStatus: SubscriptionStatus
+  status: T
+}
+
+/**
+ * Event emitted when subscription is unsubscribed
+ */
+export interface SubscriptionUnsubscribedEvent {
+  type: `unsubscribed`
+  subscription: Subscription
+}
+
+/**
+ * All subscription events
+ */
+export type SubscriptionEvents = {
+  "status:change": SubscriptionStatusChangeEvent
+  "status:ready": SubscriptionStatusEvent<`ready`>
+  "status:loadingSubset": SubscriptionStatusEvent<`loadingSubset`>
+  unsubscribed: SubscriptionUnsubscribedEvent
+}
+
+/**
+ * Public interface for a collection subscription
+ * Used by sync implementations to track subscription lifecycle
+ */
+export interface Subscription extends EventEmitter<SubscriptionEvents> {
+  /** Current status of the subscription */
+  readonly status: SubscriptionStatus
+}
+
+export type LoadSubsetOptions = {
+  /** The where expression to filter the data */
   where?: BasicExpression<boolean>
+  /** The order by clause to sort the data */
   orderBy?: OrderBy
+  /** The limit of the data to load */
   limit?: number
+  /**
+   * The subscription that triggered the load.
+   * Advanced sync implementations can use this for:
+   * - LRU caching keyed by subscription
+   * - Reference counting to track active subscriptions
+   * - Subscribing to subscription events (e.g., finalization/unsubscribe)
+   * @optional Available when called from CollectionSubscription, may be undefined for direct calls
+   */
+  subscription?: Subscription
 }
 
+export type LoadSubsetFn = (options: LoadSubsetOptions) => true | Promise<void>
+
 export type CleanupFn = () => void
 
 export type SyncConfigRes = {
   cleanup?: CleanupFn
-  onLoadMore?: (options: OnLoadMoreOptions) => void | Promise<void>
+  loadSubset?: LoadSubsetFn
 }
 export interface SyncConfig<
   T extends object = Record<string, unknown>,
@@ -242,7 +309,7 @@ export interface InsertConfig {
 export type UpdateMutationFnParams<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
 > = {
   transaction: TransactionWithMutations<T, `update`>
   collection: Collection<T, TKey, TUtils>
@@ -251,7 +318,7 @@ export type UpdateMutationFnParams<
 export type InsertMutationFnParams<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
 > = {
   transaction: TransactionWithMutations<T, `insert`>
   collection: Collection<T, TKey, TUtils>
@@ -259,7 +326,7 @@ export type InsertMutationFnParams<
 export type DeleteMutationFnParams<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
 > = {
   transaction: TransactionWithMutations<T, `delete`>
   collection: Collection<T, TKey, TUtils>
@@ -268,21 +335,21 @@ export type DeleteMutationFnParams<
 export type InsertMutationFn<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
   TReturn = any,
 > = (params: InsertMutationFnParams<T, TKey, TUtils>) => Promise<TReturn>
 
 export type UpdateMutationFn<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
   TReturn = any,
 > = (params: UpdateMutationFnParams<T, TKey, TUtils>) => Promise<TReturn>
 
 export type DeleteMutationFn<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
   TReturn = any,
 > = (params: DeleteMutationFnParams<T, TKey, TUtils>) => Promise<TReturn>
 
@@ -313,6 +380,8 @@ export type CollectionStatus =
   /** Collection has been cleaned up and resources freed */
   | `cleaned-up`
 
+export type SyncMode = `eager` | `on-demand`
+
 export interface BaseCollectionConfig<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
@@ -321,7 +390,7 @@ export interface BaseCollectionConfig<
   // then it would conflict with the overloads of createCollection which
   // requires either T to be provided or a schema to be provided but not both!
   TSchema extends StandardSchemaV1 = never,
-  TUtils extends UtilsRecord = Record<string, Fn>,
+  TUtils extends UtilsRecord = UtilsRecord,
   TReturn = any,
 > {
   // If an id isn't passed in, a UUID will be
@@ -374,6 +443,15 @@ export interface BaseCollectionConfig<
    * compare: (x, y) => x.createdAt.getTime() - y.createdAt.getTime()
    */
   compare?: (x: T, y: T) => number
+  /**
+   * The mode of sync to use for the collection.
+   * @default `eager`
+   * @description
+   * - `eager`: syncs all data immediately on preload
+   * - `on-demand`: syncs data in incremental snapshots when the collection is queried
+   * The exact implementation of the sync mode is up to the sync implementation.
+   */
+  syncMode?: SyncMode
   /**
    * Optional asynchronous handler function called before an insert operation
    * @param params Object containing transaction and collection information
@@ -503,13 +581,16 @@ export interface BaseCollectionConfig<
    * }
    */
   onDelete?: DeleteMutationFn<T, TKey, TUtils, TReturn>
+
+  utils?: TUtils
 }
 
 export interface CollectionConfig<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
   TSchema extends StandardSchemaV1 = never,
-> extends BaseCollectionConfig<T, TKey, TSchema> {
+  TUtils extends UtilsRecord = UtilsRecord,
+> extends BaseCollectionConfig<T, TKey, TSchema, TUtils> {
   sync: SyncConfig<T, TKey>
 }
 
@@ -533,7 +614,8 @@ export type CollectionConfigSingleRowOption<
   T extends object = Record<string, unknown>,
   TKey extends string | number = string | number,
   TSchema extends StandardSchemaV1 = never,
-> = CollectionConfig<T, TKey, TSchema> & MaybeSingleResult
+  TUtils extends UtilsRecord = {},
+> = CollectionConfig<T, TKey, TSchema, TUtils> & MaybeSingleResult
 
 export type ChangesPayload<T extends object = Record<string, unknown>> = Array<
   ChangeMessage<T>