@tanstack/db 0.4.8 → 0.4.10

package/src/collection/subscription.ts

@@ -1,13 +1,20 @@
 import { ensureIndexForExpression } from "../indexes/auto-index.js"
 import { and, gt, lt } from "../query/builder/functions.js"
 import { Value } from "../query/ir.js"
+import { EventEmitter } from "../event-emitter.js"
 import {
   createFilterFunctionFromExpression,
   createFilteredCallback,
 } from "./change-events.js"
 import type { BasicExpression, OrderBy } from "../query/ir.js"
 import type { IndexInterface } from "../indexes/base-index.js"
-import type { ChangeMessage } from "../types.js"
+import type {
+  ChangeMessage,
+  Subscription,
+  SubscriptionEvents,
+  SubscriptionStatus,
+  SubscriptionUnsubscribedEvent,
+} from "../types.js"
 import type { CollectionImpl } from "./index.js"
 
 type RequestSnapshotOptions = {
@@ -22,13 +29,17 @@ type RequestLimitedSnapshotOptions = {
 }
 
 type CollectionSubscriptionOptions = {
+  includeInitialState?: boolean
   /** Pre-compiled expression for filtering changes */
   whereExpression?: BasicExpression<boolean>
   /** Callback to call when the subscription is unsubscribed */
-  onUnsubscribe?: () => void
+  onUnsubscribe?: (event: SubscriptionUnsubscribedEvent) => void
 }
 
-export class CollectionSubscription {
+export class CollectionSubscription
+  extends EventEmitter<SubscriptionEvents>
+  implements Subscription
+{
   private loadedInitialState = false
 
   // Flag to indicate that we have sent at least 1 snapshot.
@@ -42,11 +53,24 @@ export class CollectionSubscription {
 
   private orderByIndex: IndexInterface<string | number> | undefined
 
+  // Status tracking
+  private _status: SubscriptionStatus = `ready`
+  private pendingLoadSubsetPromises: Set<Promise<void>> = new Set()
+
+  public get status(): SubscriptionStatus {
+    return this._status
+  }
+
   constructor(
     private collection: CollectionImpl<any, any, any, any, any>,
     private callback: (changes: Array<ChangeMessage<any, any>>) => void,
     private options: CollectionSubscriptionOptions
   ) {
+    super()
+    if (options.onUnsubscribe) {
+      this.on(`unsubscribed`, (event) => options.onUnsubscribe!(event))
+    }
+
     // Auto-index for where expressions if enabled
     if (options.whereExpression) {
       ensureIndexForExpression(options.whereExpression, this.collection)
@@ -71,6 +95,53 @@ export class CollectionSubscription {
     this.orderByIndex = index
   }
 
+  /**
+   * Set subscription status and emit events if changed
+   */
+  private setStatus(newStatus: SubscriptionStatus) {
+    if (this._status === newStatus) {
+      return // No change
+    }
+
+    const previousStatus = this._status
+    this._status = newStatus
+
+    // Emit status:change event
+    this.emitInner(`status:change`, {
+      type: `status:change`,
+      subscription: this,
+      previousStatus,
+      status: newStatus,
+    })
+
+    // Emit specific status event
+    const eventKey: `status:${SubscriptionStatus}` = `status:${newStatus}`
+    this.emitInner(eventKey, {
+      type: eventKey,
+      subscription: this,
+      previousStatus,
+      status: newStatus,
+    } as SubscriptionEvents[typeof eventKey])
+  }
+
+  /**
+   * Track a loadSubset promise and manage loading status
+   */
+  private trackLoadSubsetPromise(syncResult: Promise<void> | true) {
+    // Track the promise if it's actually a promise (async work)
+    if (syncResult instanceof Promise) {
+      this.pendingLoadSubsetPromises.add(syncResult)
+      this.setStatus(`loadingSubset`)
+
+      syncResult.finally(() => {
+        this.pendingLoadSubsetPromises.delete(syncResult)
+        if (this.pendingLoadSubsetPromises.size === 0) {
+          this.setStatus(`ready`)
+        }
+      })
+    }
+  }
+
   hasLoadedInitialState() {
     return this.loadedInitialState
   }
@@ -121,10 +192,13 @@ export class CollectionSubscription {
 
     // Request the sync layer to load more data
     // don't await it, we will load the data into the collection when it comes in
-    this.collection.syncMore({
+    const syncResult = this.collection._sync.loadSubset({
       where: stateOpts.where,
+      subscription: this,
     })
 
+    this.trackLoadSubsetPromise(syncResult)
+
     // Also load data immediately from the collection
     const snapshot = this.collection.currentStateAsChanges(stateOpts)
 
@@ -215,11 +289,14 @@ export class CollectionSubscription {
 
     // Request the sync layer to load more data
     // don't await it, we will load the data into the collection when it comes in
-    this.collection.syncMore({
+    const syncResult = this.collection._sync.loadSubset({
       where: whereWithValueFilter,
       limit,
       orderBy,
+      subscription: this,
     })
+
+    this.trackLoadSubsetPromise(syncResult)
   }
 
   /**
@@ -264,6 +341,11 @@ export class CollectionSubscription {
   }
 
   unsubscribe() {
-    this.options.onUnsubscribe?.()
+    this.emitInner(`unsubscribed`, {
+      type: `unsubscribed`,
+      subscription: this,
+    })
+    // Clear all event listeners to prevent memory leaks
+    this.clearListeners()
   }
 }

package/src/collection/sync.ts

@@ -1,4 +1,5 @@
 import {
+  CollectionConfigurationError,
   CollectionIsInErrorStateError,
   DuplicateKeySyncError,
   NoPendingSyncTransactionCommitError,
@@ -13,12 +14,13 @@ import type {
   ChangeMessage,
   CleanupFn,
   CollectionConfig,
-  OnLoadMoreOptions,
+  LoadSubsetOptions,
   SyncConfigRes,
 } from "../types"
 import type { CollectionImpl } from "./index.js"
 import type { CollectionStateManager } from "./state"
 import type { CollectionLifecycleManager } from "./lifecycle"
+import type { CollectionEventsManager } from "./events.js"
 
 export class CollectionSyncManager<
   TOutput extends object = Record<string, unknown>,
@@ -29,31 +31,38 @@ export class CollectionSyncManager<
   private collection!: CollectionImpl<TOutput, TKey, any, TSchema, TInput>
   private state!: CollectionStateManager<TOutput, TKey, TSchema, TInput>
   private lifecycle!: CollectionLifecycleManager<TOutput, TKey, TSchema, TInput>
+  private _events!: CollectionEventsManager
   private config!: CollectionConfig<TOutput, TKey, TSchema>
   private id: string
+  private syncMode: `eager` | `on-demand`
 
   public preloadPromise: Promise<void> | null = null
   public syncCleanupFn: (() => void) | null = null
-  public syncOnLoadMoreFn:
-    | ((options: OnLoadMoreOptions) => void | Promise<void>)
+  public syncLoadSubsetFn:
+    | ((options: LoadSubsetOptions) => true | Promise<void>)
     | null = null
 
+  private pendingLoadSubsetPromises: Set<Promise<void>> = new Set()
+
   /**
    * Creates a new CollectionSyncManager instance
    */
   constructor(config: CollectionConfig<TOutput, TKey, TSchema>, id: string) {
     this.config = config
     this.id = id
+    this.syncMode = config.syncMode ?? `eager`
   }
 
   setDeps(deps: {
     collection: CollectionImpl<TOutput, TKey, any, TSchema, TInput>
     state: CollectionStateManager<TOutput, TKey, TSchema, TInput>
     lifecycle: CollectionLifecycleManager<TOutput, TKey, TSchema, TInput>
+    events: CollectionEventsManager
   }) {
     this.collection = deps.collection
     this.state = deps.state
     this.lifecycle = deps.lifecycle
+    this._events = deps.events
   }
 
   /**
@@ -189,8 +198,16 @@ export class CollectionSyncManager<
       // Store cleanup function if provided
       this.syncCleanupFn = syncRes?.cleanup ?? null
 
-      // Store onLoadMore function if provided
-      this.syncOnLoadMoreFn = syncRes?.onLoadMore ?? null
+      // Store loadSubset function if provided
+      this.syncLoadSubsetFn = syncRes?.loadSubset ?? null
+
+      // Validate: on-demand mode requires a loadSubset function
+      if (this.syncMode === `on-demand` && !this.syncLoadSubsetFn) {
+        throw new CollectionConfigurationError(
+          `Collection "${this.id}" is configured with syncMode "on-demand" but the sync function did not return a loadSubset handler. ` +
+            `Either provide a loadSubset handler or use syncMode "eager".`
+        )
+      }
     } catch (error) {
       this.lifecycle.setStatus(`error`)
       throw error
@@ -239,16 +256,71 @@ export class CollectionSyncManager<
     return this.preloadPromise
   }
 
+  /**
+   * Gets whether the collection is currently loading more data
+   */
+  public get isLoadingSubset(): boolean {
+    return this.pendingLoadSubsetPromises.size > 0
+  }
+
+  /**
+   * Tracks a load promise for isLoadingSubset state.
+   * @internal This is for internal coordination (e.g., live-query glue code), not for general use.
+   */
+  public trackLoadPromise(promise: Promise<void>): void {
+    const loadingStarting = !this.isLoadingSubset
+    this.pendingLoadSubsetPromises.add(promise)
+
+    if (loadingStarting) {
+      this._events.emit(`loadingSubset:change`, {
+        type: `loadingSubset:change`,
+        collection: this.collection,
+        isLoadingSubset: true,
+        previousIsLoadingSubset: false,
+        loadingSubsetTransition: `start`,
+      })
+    }
+
+    promise.finally(() => {
+      const loadingEnding =
+        this.pendingLoadSubsetPromises.size === 1 &&
+        this.pendingLoadSubsetPromises.has(promise)
+      this.pendingLoadSubsetPromises.delete(promise)
+
+      if (loadingEnding) {
+        this._events.emit(`loadingSubset:change`, {
+          type: `loadingSubset:change`,
+          collection: this.collection,
+          isLoadingSubset: false,
+          previousIsLoadingSubset: true,
+          loadingSubsetTransition: `end`,
+        })
+      }
+    })
+  }
+
   /**
    * Requests the sync layer to load more data.
    * @param options Options to control what data is being loaded
    * @returns If data loading is asynchronous, this method returns a promise that resolves when the data is loaded.
-   *          If data loading is synchronous, the data is loaded when the method returns.
+   *          Returns true if no sync function is configured, if syncMode is 'eager', or if there is no work to do.
    */
-  public syncMore(options: OnLoadMoreOptions): void | Promise<void> {
-    if (this.syncOnLoadMoreFn) {
-      return this.syncOnLoadMoreFn(options)
+  public loadSubset(options: LoadSubsetOptions): Promise<void> | true {
+    // Bypass loadSubset when syncMode is 'eager'
+    if (this.syncMode === `eager`) {
+      return true
+    }
+
+    if (this.syncLoadSubsetFn) {
+      const result = this.syncLoadSubsetFn(options)
+      // If the result is a promise, track it
+      if (result instanceof Promise) {
+        this.trackLoadPromise(result)
+        return result
+      }
     }
+
+    return true
   }
 
   public cleanup(): void {

package/src/errors.ts

@@ -349,9 +349,23 @@ export class LimitOffsetRequireOrderByError extends QueryCompilationError {
   }
 }
 
+/**
+ * Error thrown when a collection input stream is not found during query compilation.
+ * In self-joins, each alias (e.g., 'employee', 'manager') requires its own input stream.
+ */
 export class CollectionInputNotFoundError extends QueryCompilationError {
-  constructor(collectionId: string) {
-    super(`Input for collection "${collectionId}" not found in inputs map`)
+  constructor(
+    alias: string,
+    collectionId?: string,
+    availableKeys?: Array<string>
+  ) {
+    const details = collectionId
+      ? `alias "${alias}" (collection "${collectionId}")`
+      : `collection "${alias}"`
+    const availableKeysMsg = availableKeys?.length
+      ? `. Available keys: ${availableKeys.join(`, `)}`
+      : ``
+    super(`Input for ${details} not found in inputs map${availableKeysMsg}`)
   }
 }
 
@@ -399,32 +413,32 @@ export class UnsupportedJoinTypeError extends JoinError {
   }
 }
 
-export class InvalidJoinConditionSameTableError extends JoinError {
-  constructor(tableAlias: string) {
+export class InvalidJoinConditionSameSourceError extends JoinError {
+  constructor(sourceAlias: string) {
     super(
-      `Invalid join condition: both expressions refer to the same table "${tableAlias}"`
+      `Invalid join condition: both expressions refer to the same source "${sourceAlias}"`
     )
   }
 }
 
-export class InvalidJoinConditionTableMismatchError extends JoinError {
+export class InvalidJoinConditionSourceMismatchError extends JoinError {
   constructor() {
-    super(`Invalid join condition: expressions must reference table aliases`)
+    super(`Invalid join condition: expressions must reference source aliases`)
   }
 }
 
-export class InvalidJoinConditionLeftTableError extends JoinError {
-  constructor(tableAlias: string) {
+export class InvalidJoinConditionLeftSourceError extends JoinError {
+  constructor(sourceAlias: string) {
     super(
-      `Invalid join condition: left expression refers to an unavailable table "${tableAlias}"`
+      `Invalid join condition: left expression refers to an unavailable source "${sourceAlias}"`
     )
   }
 }
 
-export class InvalidJoinConditionRightTableError extends JoinError {
-  constructor(tableAlias: string) {
+export class InvalidJoinConditionRightSourceError extends JoinError {
+  constructor(sourceAlias: string) {
     super(
-      `Invalid join condition: right expression does not refer to the joined table "${tableAlias}"`
+      `Invalid join condition: right expression does not refer to the joined source "${sourceAlias}"`
     )
   }
 }
@@ -563,3 +577,67 @@ export class CannotCombineEmptyExpressionListError extends QueryOptimizerError {
     super(`Cannot combine empty expression list`)
   }
 }
+
+/**
+ * Internal error when the query optimizer fails to convert a WHERE clause to a collection filter.
+ */
+export class WhereClauseConversionError extends QueryOptimizerError {
+  constructor(collectionId: string, alias: string) {
+    super(
+      `Failed to convert WHERE clause to collection filter for collection '${collectionId}' alias '${alias}'. This indicates a bug in the query optimization logic.`
+    )
+  }
+}
+
+/**
+ * Error when a subscription cannot be found during lazy join processing.
+ * For subqueries, aliases may be remapped (e.g., 'activeUser' → 'user').
+ */
+export class SubscriptionNotFoundError extends QueryCompilationError {
+  constructor(
+    resolvedAlias: string,
+    originalAlias: string,
+    collectionId: string,
+    availableAliases: Array<string>
+  ) {
+    super(
+      `Internal error: subscription for alias '${resolvedAlias}' (remapped from '${originalAlias}', collection '${collectionId}') is missing in join pipeline. Available aliases: ${availableAliases.join(`, `)}. This indicates a bug in alias tracking.`
+    )
+  }
+}
+
+/**
+ * Error thrown when aggregate expressions are used outside of a GROUP BY context.
+ */
+export class AggregateNotSupportedError extends QueryCompilationError {
+  constructor() {
+    super(
+      `Aggregate expressions are not supported in this context. Use GROUP BY clause for aggregates.`
+    )
+  }
+}
+
+/**
+ * Internal error when the compiler returns aliases that don't have corresponding input streams.
+ * This should never happen since all aliases come from user declarations.
+ */
+export class MissingAliasInputsError extends QueryCompilationError {
+  constructor(missingAliases: Array<string>) {
+    super(
+      `Internal error: compiler returned aliases without inputs: ${missingAliases.join(`, `)}. ` +
+        `This indicates a bug in query compilation. Please report this issue.`
+    )
+  }
+}
+
+/**
+ * Error thrown when setWindow is called on a collection without an ORDER BY clause.
+ */
+export class SetWindowRequiresOrderByError extends QueryCompilationError {
+  constructor() {
+    super(
+      `setWindow() can only be called on collections with an ORDER BY clause. ` +
+        `Add .orderBy() to your query to enable window movement.`
+    )
+  }
+}

package/src/event-emitter.ts

@@ -0,0 +1,118 @@
+/**
+ * Generic type-safe event emitter
+ * @template TEvents - Record of event names to event payload types
+ */
+export class EventEmitter<TEvents extends Record<string, any>> {
+  private listeners = new Map<
+    keyof TEvents,
+    Set<(event: TEvents[keyof TEvents]) => void>
+  >()
+
+  /**
+   * Subscribe to an event
+   * @param event - Event name to listen for
+   * @param callback - Function to call when event is emitted
+   * @returns Unsubscribe function
+   */
+  on<T extends keyof TEvents>(
+    event: T,
+    callback: (event: TEvents[T]) => void
+  ): () => void {
+    if (!this.listeners.has(event)) {
+      this.listeners.set(event, new Set())
+    }
+    this.listeners.get(event)!.add(callback as (event: any) => void)
+
+    return () => {
+      this.listeners.get(event)?.delete(callback as (event: any) => void)
+    }
+  }
+
+  /**
+   * Subscribe to an event once (automatically unsubscribes after first emission)
+   * @param event - Event name to listen for
+   * @param callback - Function to call when event is emitted
+   * @returns Unsubscribe function
+   */
+  once<T extends keyof TEvents>(
+    event: T,
+    callback: (event: TEvents[T]) => void
+  ): () => void {
+    const unsubscribe = this.on(event, (eventPayload) => {
+      callback(eventPayload)
+      unsubscribe()
+    })
+    return unsubscribe
+  }
+
+  /**
+   * Unsubscribe from an event
+   * @param event - Event name to stop listening for
+   * @param callback - Function to remove
+   */
+  off<T extends keyof TEvents>(
+    event: T,
+    callback: (event: TEvents[T]) => void
+  ): void {
+    this.listeners.get(event)?.delete(callback as (event: any) => void)
+  }
+
+  /**
+   * Wait for an event to be emitted
+   * @param event - Event name to wait for
+   * @param timeout - Optional timeout in milliseconds
+   * @returns Promise that resolves with the event payload
+   */
+  waitFor<T extends keyof TEvents>(
+    event: T,
+    timeout?: number
+  ): Promise<TEvents[T]> {
+    return new Promise((resolve, reject) => {
+      let timeoutId: NodeJS.Timeout | undefined
+      const unsubscribe = this.on(event, (eventPayload) => {
+        if (timeoutId) {
+          clearTimeout(timeoutId)
+          timeoutId = undefined
+        }
+        resolve(eventPayload)
+        unsubscribe()
+      })
+      if (timeout) {
+        timeoutId = setTimeout(() => {
+          timeoutId = undefined
+          unsubscribe()
+          reject(new Error(`Timeout waiting for event ${String(event)}`))
+        }, timeout)
+      }
+    })
+  }
+
+  /**
+   * Emit an event to all listeners
+   * @param event - Event name to emit
+   * @param eventPayload - Event payload
+   * @internal For use by subclasses - subclasses should wrap this with a public emit if needed
+   */
+  protected emitInner<T extends keyof TEvents>(
+    event: T,
+    eventPayload: TEvents[T]
+  ): void {
+    this.listeners.get(event)?.forEach((listener) => {
+      try {
+        listener(eventPayload)
+      } catch (error) {
+        // Re-throw in a microtask to surface the error
+        queueMicrotask(() => {
+          throw error
+        })
+      }
+    })
+  }
+
+  /**
+   * Clear all listeners
+   */
+  protected clearListeners(): void {
+    this.listeners.clear()
+  }
+}

package/src/local-only.ts

@@ -1,15 +1,12 @@
 import type {
   BaseCollectionConfig,
   CollectionConfig,
-  DeleteMutationFn,
   DeleteMutationFnParams,
   InferSchemaOutput,
-  InsertMutationFn,
   InsertMutationFnParams,
   OperationType,
   PendingMutation,
   SyncConfig,
-  UpdateMutationFn,
   UpdateMutationFnParams,
   UtilsRecord,
 } from "./types"
@@ -67,13 +64,7 @@ type LocalOnlyCollectionOptionsResult<
   T extends object,
   TKey extends string | number,
   TSchema extends StandardSchemaV1 | never = never,
-> = Omit<
-  CollectionConfig<T, TKey, TSchema>,
-  `onInsert` | `onUpdate` | `onDelete`
-> & {
-  onInsert?: InsertMutationFn<T, TKey, LocalOnlyCollectionUtils>
-  onUpdate?: UpdateMutationFn<T, TKey, LocalOnlyCollectionUtils>
-  onDelete?: DeleteMutationFn<T, TKey, LocalOnlyCollectionUtils>
+> = CollectionConfig<T, TKey, TSchema> & {
   utils: LocalOnlyCollectionUtils
 }
 
@@ -191,7 +182,7 @@ export function localOnlyCollectionOptions<
   const { initialData, onInsert, onUpdate, onDelete, ...restConfig } = config
 
   // Create the sync configuration with transaction confirmation capability
-  const syncResult = createLocalOnlySync(initialData)
+  const syncResult = createLocalOnlySync<T, TKey>(initialData)
 
   /**
    * Create wrapper handlers that call user handlers first, then confirm transactions
@@ -279,9 +270,11 @@ export function localOnlyCollectionOptions<
     onDelete: wrappedOnDelete,
     utils: {
       acceptMutations,
-    } as LocalOnlyCollectionUtils,
+    },
     startSync: true,
     gcTime: 0,
+  } as LocalOnlyCollectionOptionsResult<T, TKey, TSchema> & {
+    schema?: StandardSchemaV1
   }
 }
 

package/src/query/builder/types.ts

@@ -107,7 +107,7 @@ export type SchemaFromSource<T extends Source> = Prettify<{
  * GetAliases - Extracts all table aliases available in a query context
  *
  * Simple utility type that returns the keys of the schema, representing
- * all table/collection aliases that can be referenced in the current query.
+ * all table/source aliases that can be referenced in the current query.
  */
 export type GetAliases<TContext extends Context> = keyof TContext[`schema`]