@tanstack/db 0.0.27 → 0.0.30

package/src/indexes/base-index.ts

@@ -0,0 +1,119 @@
+import { compileSingleRowExpression } from "../query/compiler/evaluators.js"
+import { comparisonFunctions } from "../query/builder/functions.js"
+import type { BasicExpression } from "../query/ir.js"
+
+/**
+ * Operations that indexes can support, imported from available comparison functions
+ */
+export const IndexOperation = comparisonFunctions
+
+/**
+ * Type for index operation values
+ */
+export type IndexOperation = (typeof comparisonFunctions)[number]
+
+/**
+ * Statistics about index usage and performance
+ */
+export interface IndexStats {
+  readonly entryCount: number
+  readonly lookupCount: number
+  readonly averageLookupTime: number
+  readonly lastUpdated: Date
+}
+
+/**
+ * Base abstract class that all index types extend
+ */
+export abstract class BaseIndex<
+  TKey extends string | number = string | number,
+> {
+  public readonly id: number
+  public readonly name?: string
+  public readonly expression: BasicExpression
+  public abstract readonly supportedOperations: Set<IndexOperation>
+
+  protected lookupCount = 0
+  protected totalLookupTime = 0
+  protected lastUpdated = new Date()
+
+  constructor(
+    id: number,
+    expression: BasicExpression,
+    name?: string,
+    options?: any
+  ) {
+    this.id = id
+    this.expression = expression
+    this.name = name
+    this.initialize(options)
+  }
+
+  // Abstract methods that each index type must implement
+  abstract add(key: TKey, item: any): void
+  abstract remove(key: TKey, item: any): void
+  abstract update(key: TKey, oldItem: any, newItem: any): void
+  abstract build(entries: Iterable<[TKey, any]>): void
+  abstract clear(): void
+  abstract lookup(operation: IndexOperation, value: any): Set<TKey>
+  abstract get keyCount(): number
+
+  // Common methods
+  supports(operation: IndexOperation): boolean {
+    return this.supportedOperations.has(operation)
+  }
+
+  matchesField(fieldPath: Array<string>): boolean {
+    return (
+      this.expression.type === `ref` &&
+      this.expression.path.length === fieldPath.length &&
+      this.expression.path.every((part, i) => part === fieldPath[i])
+    )
+  }
+
+  getStats(): IndexStats {
+    return {
+      entryCount: this.keyCount,
+      lookupCount: this.lookupCount,
+      averageLookupTime:
+        this.lookupCount > 0 ? this.totalLookupTime / this.lookupCount : 0,
+      lastUpdated: this.lastUpdated,
+    }
+  }
+
+  // Protected methods for subclasses
+  protected abstract initialize(options?: any): void
+
+  protected evaluateIndexExpression(item: any): any {
+    const evaluator = compileSingleRowExpression(this.expression)
+    return evaluator(item as Record<string, unknown>)
+  }
+
+  protected trackLookup(startTime: number): void {
+    const duration = performance.now() - startTime
+    this.lookupCount++
+    this.totalLookupTime += duration
+  }
+
+  protected updateTimestamp(): void {
+    this.lastUpdated = new Date()
+  }
+}
+
+/**
+ * Type for index constructor
+ */
+export type IndexConstructor<TKey extends string | number = string | number> =
+  new (
+    id: number,
+    expression: BasicExpression,
+    name?: string,
+    options?: any
+  ) => BaseIndex<TKey>
+
+/**
+ * Index resolver can be either a class constructor or async loader
+ */
+export type IndexResolver<TKey extends string | number = string | number> =
+  | IndexConstructor<TKey>
+  | (() => Promise<IndexConstructor<TKey>>)

package/src/indexes/btree-index.ts

@@ -0,0 +1,263 @@
+import { ascComparator } from "../utils/comparison.js"
+import { BTree } from "../utils/btree.js"
+import { BaseIndex } from "./base-index.js"
+import type { BasicExpression } from "../query/ir.js"
+import type { IndexOperation } from "./base-index.js"
+
+/**
+ * Options for Ordered index
+ */
+export interface BTreeIndexOptions {
+  compareFn?: (a: any, b: any) => number
+}
+
+/**
+ * Options for range queries
+ */
+export interface RangeQueryOptions {
+  from?: any
+  to?: any
+  fromInclusive?: boolean
+  toInclusive?: boolean
+}
+
+/**
+ * B+Tree index for sorted data with range queries
+ * This maintains items in sorted order and provides efficient range operations
+ */
+export class BTreeIndex<
+  TKey extends string | number = string | number,
+> extends BaseIndex<TKey> {
+  public readonly supportedOperations = new Set<IndexOperation>([
+    `eq`,
+    `gt`,
+    `gte`,
+    `lt`,
+    `lte`,
+    `in`,
+  ])
+
+  // Internal data structures - private to hide implementation details
+  // The `orderedEntries` B+ tree is used for efficient range queries
+  // The `valueMap` is used for O(1) lookups of PKs by indexed value
+  private orderedEntries: BTree<any, undefined> // we don't associate values with the keys of the B+ tree (the keys are indexed values)
+  private valueMap = new Map<any, Set<TKey>>() // instead we store a mapping of indexed values to a set of PKs
+  private indexedKeys = new Set<TKey>()
+  private compareFn: (a: any, b: any) => number = ascComparator
+
+  constructor(
+    id: number,
+    expression: BasicExpression,
+    name?: string,
+    options?: any
+  ) {
+    super(id, expression, name, options)
+    this.compareFn = options?.compareFn ?? ascComparator
+    this.orderedEntries = new BTree(this.compareFn)
+  }
+
+  protected initialize(_options?: BTreeIndexOptions): void {}
+
+  /**
+   * Adds a value to the index
+   */
+  add(key: TKey, item: any): void {
+    let indexedValue: any
+    try {
+      indexedValue = this.evaluateIndexExpression(item)
+    } catch (error) {
+      throw new Error(
+        `Failed to evaluate index expression for key ${key}: ${error}`
+      )
+    }
+
+    // Check if this value already exists
+    if (this.valueMap.has(indexedValue)) {
+      // Add to existing set
+      this.valueMap.get(indexedValue)!.add(key)
+    } else {
+      // Create new set for this value
+      const keySet = new Set<TKey>([key])
+      this.valueMap.set(indexedValue, keySet)
+      this.orderedEntries.set(indexedValue, undefined)
+    }
+
+    this.indexedKeys.add(key)
+    this.updateTimestamp()
+  }
+
+  /**
+   * Removes a value from the index
+   */
+  remove(key: TKey, item: any): void {
+    let indexedValue: any
+    try {
+      indexedValue = this.evaluateIndexExpression(item)
+    } catch (error) {
+      console.warn(
+        `Failed to evaluate index expression for key ${key} during removal:`,
+        error
+      )
+      return
+    }
+
+    if (this.valueMap.has(indexedValue)) {
+      const keySet = this.valueMap.get(indexedValue)!
+      keySet.delete(key)
+
+      // If set is now empty, remove the entry entirely
+      if (keySet.size === 0) {
+        this.valueMap.delete(indexedValue)
+
+        // Remove from ordered entries
+        this.orderedEntries.delete(indexedValue)
+      }
+    }
+
+    this.indexedKeys.delete(key)
+    this.updateTimestamp()
+  }
+
+  /**
+   * Updates a value in the index
+   */
+  update(key: TKey, oldItem: any, newItem: any): void {
+    this.remove(key, oldItem)
+    this.add(key, newItem)
+  }
+
+  /**
+   * Builds the index from a collection of entries
+   */
+  build(entries: Iterable<[TKey, any]>): void {
+    this.clear()
+
+    for (const [key, item] of entries) {
+      this.add(key, item)
+    }
+  }
+
+  /**
+   * Clears all data from the index
+   */
+  clear(): void {
+    this.orderedEntries.clear()
+    this.valueMap.clear()
+    this.indexedKeys.clear()
+    this.updateTimestamp()
+  }
+
+  /**
+   * Performs a lookup operation
+   */
+  lookup(operation: IndexOperation, value: any): Set<TKey> {
+    const startTime = performance.now()
+
+    let result: Set<TKey>
+
+    switch (operation) {
+      case `eq`:
+        result = this.equalityLookup(value)
+        break
+      case `gt`:
+        result = this.rangeQuery({ from: value, fromInclusive: false })
+        break
+      case `gte`:
+        result = this.rangeQuery({ from: value, fromInclusive: true })
+        break
+      case `lt`:
+        result = this.rangeQuery({ to: value, toInclusive: false })
+        break
+      case `lte`:
+        result = this.rangeQuery({ to: value, toInclusive: true })
+        break
+      case `in`:
+        result = this.inArrayLookup(value)
+        break
+      default:
+        throw new Error(`Operation ${operation} not supported by BTreeIndex`)
+    }
+
+    this.trackLookup(startTime)
+    return result
+  }
+
+  /**
+   * Gets the number of indexed keys
+   */
+  get keyCount(): number {
+    return this.indexedKeys.size
+  }
+
+  // Public methods for backward compatibility (used by tests)
+
+  /**
+   * Performs an equality lookup
+   */
+  equalityLookup(value: any): Set<TKey> {
+    return new Set(this.valueMap.get(value) ?? [])
+  }
+
+  /**
+   * Performs a range query with options
+   * This is more efficient for compound queries like "WHERE a > 5 AND a < 10"
+   */
+  rangeQuery(options: RangeQueryOptions = {}): Set<TKey> {
+    const { from, to, fromInclusive = true, toInclusive = true } = options
+    const result = new Set<TKey>()
+
+    const fromKey = from ?? this.orderedEntries.minKey()
+    const toKey = to ?? this.orderedEntries.maxKey()
+
+    this.orderedEntries.forRange(
+      fromKey,
+      toKey,
+      toInclusive,
+      (indexedValue, _) => {
+        if (!fromInclusive && this.compareFn(indexedValue, from) === 0) {
+          // the B+ tree `forRange` method does not support exclusive lower bounds
+          // so we need to exclude it manually
+          return
+        }
+
+        const keys = this.valueMap.get(indexedValue)
+        if (keys) {
+          keys.forEach((key) => result.add(key))
+        }
+      }
+    )
+
+    return result
+  }
+
+  /**
+   * Performs an IN array lookup
+   */
+  inArrayLookup(values: Array<any>): Set<TKey> {
+    const result = new Set<TKey>()
+
+    for (const value of values) {
+      const keys = this.valueMap.get(value)
+      if (keys) {
+        keys.forEach((key) => result.add(key))
+      }
+    }
+
+    return result
+  }
+
+  // Getter methods for testing compatibility
+  get indexedKeysSet(): Set<TKey> {
+    return this.indexedKeys
+  }
+
+  get orderedEntriesArray(): Array<[any, Set<TKey>]> {
+    return this.orderedEntries
+      .keysArray()
+      .map((key) => [key, this.valueMap.get(key) ?? new Set()])
+  }
+
+  get valueMapData(): Map<any, Set<TKey>> {
+    return this.valueMap
+  }
+}

package/src/indexes/index-options.ts

@@ -0,0 +1,42 @@
+import type { IndexConstructor, IndexResolver } from "./base-index.js"
+
+/**
+ * Enhanced index options that support both sync and async resolvers
+ */
+export interface IndexOptions<TResolver extends IndexResolver = IndexResolver> {
+  name?: string
+  indexType?: TResolver
+  options?: TResolver extends IndexConstructor<any>
+    ? TResolver extends new (
+        id: string,
+        expr: any,
+        name?: string,
+        options?: infer O
+      ) => any
+      ? O
+      : never
+    : TResolver extends () => Promise<infer TCtor>
+      ? TCtor extends new (
+          id: string,
+          expr: any,
+          name?: string,
+          options?: infer O
+        ) => any
+        ? O
+        : never
+      : never
+}
+
+/**
+ * Utility type to extract the constructed index type from a resolver
+ */
+export type ResolvedIndexType<TResolver extends IndexResolver> =
+  TResolver extends IndexConstructor<any>
+    ? InstanceType<TResolver>
+    : TResolver extends () => Promise<IndexConstructor<any>>
+      ? TResolver extends () => Promise<infer TCtor>
+        ? TCtor extends IndexConstructor<any>
+          ? InstanceType<TCtor>
+          : never
+        : never
+      : never

package/src/indexes/lazy-index.ts

@@ -0,0 +1,251 @@
+import type {
+  BaseIndex,
+  IndexConstructor,
+  IndexResolver,
+} from "./base-index.js"
+import type { BasicExpression } from "../query/ir.js"
+
+/**
+ * Utility to determine if a resolver is a constructor or async loader
+ */
+function isConstructor<TKey extends string | number>(
+  resolver: IndexResolver<TKey>
+): resolver is IndexConstructor<TKey> {
+  // Check if it's a function with a prototype (constructor)
+  return (
+    typeof resolver === `function` &&
+    resolver.prototype !== undefined &&
+    resolver.prototype.constructor === resolver
+  )
+}
+
+/**
+ * Resolve index constructor from resolver
+ */
+async function resolveIndexConstructor<TKey extends string | number>(
+  resolver: IndexResolver<TKey>
+): Promise<IndexConstructor<TKey>> {
+  if (isConstructor(resolver)) {
+    return resolver
+  } else {
+    // It's an async loader function
+    return await resolver()
+  }
+}
+
+/**
+ * Wrapper that defers index creation until first sync
+ */
+export class LazyIndexWrapper<TKey extends string | number = string | number> {
+  private indexPromise: Promise<BaseIndex<TKey>> | null = null
+  private resolvedIndex: BaseIndex<TKey> | null = null
+
+  constructor(
+    private id: number,
+    private expression: BasicExpression,
+    private name: string | undefined,
+    private resolver: IndexResolver<TKey>,
+    private options: any,
+    private collectionEntries?: Iterable<[TKey, any]>
+  ) {
+    // For synchronous constructors, resolve immediately
+    if (isConstructor(this.resolver)) {
+      this.resolvedIndex = new this.resolver(
+        this.id,
+        this.expression,
+        this.name,
+        this.options
+      )
+      // Build with initial data if provided
+      if (this.collectionEntries) {
+        this.resolvedIndex.build(this.collectionEntries)
+      }
+    }
+  }
+
+  /**
+   * Resolve the actual index
+   */
+  async resolve(): Promise<BaseIndex<TKey>> {
+    if (this.resolvedIndex) {
+      return this.resolvedIndex
+    }
+
+    if (!this.indexPromise) {
+      this.indexPromise = this.createIndex()
+    }
+
+    this.resolvedIndex = await this.indexPromise
+    return this.resolvedIndex
+  }
+
+  /**
+   * Check if already resolved
+   */
+  isResolved(): boolean {
+    return this.resolvedIndex !== null
+  }
+
+  /**
+   * Get resolved index (throws if not ready)
+   */
+  getResolved(): BaseIndex<TKey> {
+    if (!this.resolvedIndex) {
+      throw new Error(
+        `Index ${this.id} has not been resolved yet. Ensure collection is synced.`
+      )
+    }
+    return this.resolvedIndex
+  }
+
+  /**
+   * Get the index ID
+   */
+  getId(): number {
+    return this.id
+  }
+
+  /**
+   * Get the index name
+   */
+  getName(): string | undefined {
+    return this.name
+  }
+
+  /**
+   * Get the index expression
+   */
+  getExpression(): BasicExpression {
+    return this.expression
+  }
+
+  private async createIndex(): Promise<BaseIndex<TKey>> {
+    const IndexClass = await resolveIndexConstructor(this.resolver)
+    return new IndexClass(this.id, this.expression, this.name, this.options)
+  }
+}
+
+/**
+ * Proxy that provides synchronous interface while index loads asynchronously
+ */
+export class IndexProxy<TKey extends string | number = string | number> {
+  constructor(
+    private indexId: number,
+    private lazyIndex: LazyIndexWrapper<TKey>
+  ) {}
+
+  /**
+   * Get the resolved index (throws if not ready)
+   */
+  get index(): BaseIndex<TKey> {
+    return this.lazyIndex.getResolved()
+  }
+
+  /**
+   * Check if index is ready
+   */
+  get isReady(): boolean {
+    return this.lazyIndex.isResolved()
+  }
+
+  /**
+   * Wait for index to be ready
+   */
+  async whenReady(): Promise<BaseIndex<TKey>> {
+    return await this.lazyIndex.resolve()
+  }
+
+  /**
+   * Get the index ID
+   */
+  get id(): number {
+    return this.indexId
+  }
+
+  /**
+   * Get the index name (throws if not ready)
+   */
+  get name(): string | undefined {
+    if (this.isReady) {
+      return this.index.name
+    }
+    return this.lazyIndex.getName()
+  }
+
+  /**
+   * Get the index expression (available immediately)
+   */
+  get expression(): BasicExpression {
+    return this.lazyIndex.getExpression()
+  }
+
+  /**
+   * Check if index supports an operation (throws if not ready)
+   */
+  supports(operation: any): boolean {
+    return this.index.supports(operation)
+  }
+
+  /**
+   * Get index statistics (throws if not ready)
+   */
+  getStats() {
+    return this.index.getStats()
+  }
+
+  /**
+   * Check if index matches a field path (available immediately)
+   */
+  matchesField(fieldPath: Array<string>): boolean {
+    const expr = this.expression
+    return (
+      expr.type === `ref` &&
+      expr.path.length === fieldPath.length &&
+      expr.path.every((part, i) => part === fieldPath[i])
+    )
+  }
+
+  /**
+   * Get the key count (throws if not ready)
+   */
+  get keyCount(): number {
+    return this.index.keyCount
+  }
+
+  // Test compatibility properties - delegate to resolved index
+  get indexedKeysSet(): Set<TKey> {
+    const resolved = this.index as any
+    return resolved.indexedKeysSet
+  }
+
+  get orderedEntriesArray(): Array<[any, Set<TKey>]> {
+    const resolved = this.index as any
+    return resolved.orderedEntriesArray
+  }
+
+  get valueMapData(): Map<any, Set<TKey>> {
+    const resolved = this.index as any
+    return resolved.valueMapData
+  }
+
+  // BTreeIndex compatibility methods
+  equalityLookup(value: any): Set<TKey> {
+    const resolved = this.index as any
+    return resolved.equalityLookup?.(value) ?? new Set()
+  }
+
+  rangeQuery(options: any): Set<TKey> {
+    const resolved = this.index as any
+    return resolved.rangeQuery?.(options) ?? new Set()
+  }
+
+  inArrayLookup(values: Array<any>): Set<TKey> {
+    const resolved = this.index as any
+    return resolved.inArrayLookup?.(values) ?? new Set()
+  }
+
+  // Internal method for the collection to get the lazy wrapper
+  _getLazyWrapper(): LazyIndexWrapper<TKey> {
+    return this.lazyIndex
+  }
+}