@tanstack/db-ivm 0.1.13 → 0.1.15

package/src/operators/output.ts

@@ -1,8 +1,8 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { MultiSet } from "../multiset.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { MultiSet } from '../multiset.js'
 
 /**
  * Operator that outputs the messages in the stream
@@ -14,7 +14,7 @@ export class OutputOperator<T> extends UnaryOperator<T> {
     id: number,
     inputA: DifferenceStreamReader<T>,
     outputWriter: DifferenceStreamWriter<T>,
-    fn: (data: MultiSet<T>) => void
+    fn: (data: MultiSet<T>) => void,
   ) {
     super(id, inputA, outputWriter)
     this.#fn = fn
@@ -33,18 +33,18 @@ export class OutputOperator<T> extends UnaryOperator<T> {
  * @param fn - The function to call with each message
  */
 export function output<T>(
-  fn: (data: MultiSet<T>) => void
+  fn: (data: MultiSet<T>) => void,
 ): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const outputStream = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new OutputOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       outputStream.writer,
-      fn
+      fn,
     )
     stream.graph.addOperator(operator)
     return outputStream

package/src/operators/pipe.ts

@@ -1,4 +1,4 @@
-import type { IStreamBuilder, PipedOperator } from "../types.js"
+import type { IStreamBuilder, PipedOperator } from '../types.js'
 
 // Don't judge, this is the only way to type this function.
 // rxjs has very similar code to type its pipe function

package/src/operators/reduce.ts

@@ -1,9 +1,9 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { MultiSet } from "../multiset.js"
-import { Index } from "../indexes.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder, KeyValue } from "../types.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { MultiSet } from '../multiset.js'
+import { Index } from '../indexes.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, KeyValue } from '../types.js'
 
 /**
  * Base operator for reduction operations (version-free)
@@ -17,7 +17,7 @@ export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
     id: number,
     inputA: DifferenceStreamReader<[K, V1]>,
     output: DifferenceStreamWriter<[K, V2]>,
-    f: (values: Array<[V1, number]>) => Array<[V2, number]>
+    f: (values: Array<[V1, number]>) => Array<[V2, number]>,
   ) {
     super(id, inputA, output)
     this.#f = f
@@ -109,13 +109,13 @@ export function reduce<
   return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<KType, R>> => {
     const output = new StreamBuilder<KeyValue<KType, R>>(
       stream.graph,
-      new DifferenceStreamWriter<KeyValue<KType, R>>()
+      new DifferenceStreamWriter<KeyValue<KType, R>>(),
     )
     const operator = new ReduceOperator<KType, V1Type, R>(
       stream.graph.getNextOperatorId(),
       stream.connectReader() as DifferenceStreamReader<KeyValue<KType, V1Type>>,
       output.writer,
-      f
+      f,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/tap.ts

@@ -1,8 +1,8 @@
-import { DifferenceStreamWriter, LinearUnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { MultiSet } from "../multiset.js"
+import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { MultiSet } from '../multiset.js'
 
 /**
  * Operator that applies a function to each multi-set in the input stream
@@ -14,7 +14,7 @@ export class TapOperator<T> extends LinearUnaryOperator<T, T> {
     id: number,
     inputA: DifferenceStreamReader<T>,
     output: DifferenceStreamWriter<T>,
-    f: (data: MultiSet<T>) => void
+    f: (data: MultiSet<T>) => void,
   ) {
     super(id, inputA, output)
     this.#f = f
@@ -36,13 +36,13 @@ export function tap<T>(f: (data: MultiSet<T>) => void): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new TapOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
-      f
+      f,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/topK.ts

@@ -1,6 +1,6 @@
-import { MultiSet } from "../multiset.js"
-import { reduce } from "./reduce.js"
-import type { IStreamBuilder, KeyValue, PipedOperator } from "../types"
+import { MultiSet } from '../multiset.js'
+import { reduce } from './reduce.js'
+import type { IStreamBuilder, KeyValue, PipedOperator } from '../types'
 
 interface TopKOptions {
   limit?: number
@@ -24,7 +24,7 @@ export function topK<
   T,
 >(
   comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions
+  options?: TopKOptions,
 ): PipedOperator<T, T> {
   const limit = options?.limit ?? Infinity
   const offset = options?.offset ?? 0
@@ -38,7 +38,7 @@ export function topK<
           .getInner()
           .sort((a, b) => comparator(a[0] as V1Type, b[0] as V1Type))
         return sortedValues.slice(offset, offset + limit)
-      })
+      }),
     )
     return reduced as IStreamBuilder<T>
   }
@@ -62,13 +62,13 @@ export function topKWithIndex<
   T,
 >(
   comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions
+  options?: TopKOptions,
 ): PipedOperator<T, KeyValue<KType, [V1Type, number]>> {
   const limit = options?.limit ?? Infinity
   const offset = options?.offset ?? 0
 
   return (
-    stream: IStreamBuilder<T>
+    stream: IStreamBuilder<T>,
   ): IStreamBuilder<KeyValue<KType, [V1Type, number]>> => {
     const reduced = stream.pipe(
       reduce<KType, V1Type, [V1Type, number], T>((values) => {
@@ -84,7 +84,7 @@ export function topKWithIndex<
             multiplicity,
           ])
         return sortedValues
-      })
+      }),
     )
     return reduced
   }

package/src/operators/topKArray.ts

@@ -0,0 +1,255 @@
+import { generateKeyBetween } from 'fractional-indexing'
+import { binarySearch, compareKeys, diffHalfOpen } from '../utils.js'
+import type { HRange } from '../utils.js'
+
+// Abstraction for fractionally indexed values
+export type FractionalIndex = string
+export type IndexedValue<V> = [V, FractionalIndex]
+
+export function indexedValue<V>(
+  value: V,
+  index: FractionalIndex,
+): IndexedValue<V> {
+  return [value, index]
+}
+
+export function getValue<V>(indexedVal: IndexedValue<V>): V {
+  return indexedVal[0]
+}
+
+export function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {
+  return indexedVal[1]
+}
+
+/**
+ * Creates a comparator for [key, value] tuples that first compares values,
+ * then uses the row key as a stable tie-breaker.
+ */
+export function createKeyedComparator<K extends string | number, T>(
+  comparator: (a: T, b: T) => number,
+): (a: [K, T], b: [K, T]) => number {
+  return ([aKey, aVal], [bKey, bVal]) => {
+    // First compare on the value
+    const valueComparison = comparator(aVal, bVal)
+    if (valueComparison !== 0) {
+      return valueComparison
+    }
+    // If the values are equal, use the row key as tie-breaker
+    // This provides stable, deterministic ordering since keys are string | number
+    return compareKeys(aKey, bKey)
+  }
+}
+
+export type TopKChanges<V> = {
+  /** Indicates which element moves into the topK (if any) */
+  moveIn: IndexedValue<V> | null
+  /** Indicates which element moves out of the topK (if any) */
+  moveOut: IndexedValue<V> | null
+}
+
+export type TopKMoveChanges<V> = {
+  /** Flag that marks whether there were any changes to the topK */
+  changes: boolean
+  /** Indicates which elements move into the topK (if any) */
+  moveIns: Array<IndexedValue<V>>
+  /** Indicates which elements move out of the topK (if any) */
+  moveOuts: Array<IndexedValue<V>>
+}
+
+/**
+ * A topK data structure that supports insertions and deletions
+ * and returns changes to the topK.
+ */
+export interface TopK<V> {
+  size: number
+  insert: (value: V) => TopKChanges<V>
+  delete: (value: V) => TopKChanges<V>
+}
+
+/**
+ * Implementation of a topK data structure.
+ * Uses a sorted array internally to store the values and keeps a topK window over that array.
+ * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted
+ * at the start of the array which causes all the elements to shift to the right/left.
+ */
+export class TopKArray<V> implements TopK<V> {
+  #sortedValues: Array<IndexedValue<V>> = []
+  #comparator: (a: V, b: V) => number
+  #topKStart: number
+  #topKEnd: number
+
+  constructor(
+    offset: number,
+    limit: number,
+    comparator: (a: V, b: V) => number,
+  ) {
+    this.#topKStart = offset
+    this.#topKEnd = offset + limit
+    this.#comparator = comparator
+  }
+
+  get size(): number {
+    const offset = this.#topKStart
+    const limit = this.#topKEnd - this.#topKStart
+    const available = this.#sortedValues.length - offset
+    return Math.max(0, Math.min(limit, available))
+  }
+
+  /**
+   * Moves the topK window
+   */
+  move({
+    offset,
+    limit,
+  }: {
+    offset?: number
+    limit?: number
+  }): TopKMoveChanges<V> {
+    const oldOffset = this.#topKStart
+    const oldLimit = this.#topKEnd - this.#topKStart
+
+    // `this.#topKEnd` can be `Infinity` if it has no limit
+    // but `diffHalfOpen` expects a finite range
+    // so we restrict it to the size of the topK if topKEnd is infinite
+    const oldRange: HRange = [
+      this.#topKStart,
+      this.#topKEnd === Infinity ? this.#topKStart + this.size : this.#topKEnd,
+    ]
+
+    this.#topKStart = offset ?? oldOffset
+    this.#topKEnd = this.#topKStart + (limit ?? oldLimit) // can be `Infinity` if limit is `Infinity`
+
+    // Also handle `Infinity` in the newRange
+    const newRange: HRange = [
+      this.#topKStart,
+      this.#topKEnd === Infinity
+        ? Math.max(this.#topKStart + this.size, oldRange[1]) // since the new limit is Infinity we need to take everything (so we need to take the biggest (finite) topKEnd)
+        : this.#topKEnd,
+    ]
+    const { onlyInA, onlyInB } = diffHalfOpen(oldRange, newRange)
+
+    const moveIns: Array<IndexedValue<V>> = []
+    onlyInB.forEach((index) => {
+      const value = this.#sortedValues[index]
+      if (value) {
+        moveIns.push(value)
+      }
+    })
+
+    const moveOuts: Array<IndexedValue<V>> = []
+    onlyInA.forEach((index) => {
+      const value = this.#sortedValues[index]
+      if (value) {
+        moveOuts.push(value)
+      }
+    })
+
+    // It could be that there are changes (i.e. moveIns or moveOuts)
+    // but that the collection is lazy so we don't have the data yet that needs to move in/out
+    // so `moveIns` and `moveOuts` will be empty but `changes` will be true
+    // this will tell the caller that it needs to run the graph to load more data
+    return { moveIns, moveOuts, changes: onlyInA.length + onlyInB.length > 0 }
+  }
+
+  insert(value: V): TopKChanges<V> {
+    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
+
+    // Lookup insert position
+    const index = this.#findIndex(value)
+    // Generate fractional index based on the fractional indices of the elements before and after it
+    const indexBefore =
+      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)
+    const indexAfter =
+      index === this.#sortedValues.length
+        ? null
+        : getIndex(this.#sortedValues[index]!)
+    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)
+
+    // Insert the value at the correct position
+    const val = indexedValue(value, fractionalIndex)
+    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !
+    this.#sortedValues.splice(index, 0, val)
+
+    // Check if the topK changed
+    if (index < this.#topKEnd) {
+      // The inserted element is either before the top K or within the top K
+      // If it is before the top K then it moves the element that was right before the topK into the topK
+      // If it is within the top K then the inserted element moves into the top K
+      // In both cases the last element of the old top K now moves out of the top K
+      const moveInIndex = Math.max(index, this.#topKStart)
+      if (moveInIndex < this.#sortedValues.length) {
+        // We actually have a topK
+        // because in some cases there may not be enough elements in the array to reach the start of the topK
+        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK
+        result.moveIn = this.#sortedValues[moveInIndex]!
+
+        // We need to remove the element that falls out of the top K
+        // The element that falls out of the top K has shifted one to the right
+        // because of the element we inserted, so we find it at index topKEnd
+        if (this.#topKEnd < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[this.#topKEnd]!
+        }
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Deletes a value that may or may not be in the topK.
+   * IMPORTANT: this assumes that the value is present in the collection
+   *            if it's not the case it will remove the element
+   *            that is on the position where the provided `value` would be.
+   */
+  delete(value: V): TopKChanges<V> {
+    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
+
+    // Lookup delete position
+    const index = this.#findIndex(value)
+    // Remove the value at that position
+    const [removedElem] = this.#sortedValues.splice(index, 1)
+
+    // Check if the topK changed
+    if (index < this.#topKEnd) {
+      // The removed element is either before the top K or within the top K
+      // If it is before the top K then the first element of the topK moves out of the topK
+      // If it is within the top K then the removed element moves out of the topK
+      result.moveOut = removedElem!
+      if (index < this.#topKStart) {
+        // The removed element is before the topK
+        // so actually, the first element of the topK moves out of the topK
+        // and not the element that we removed
+        // The first element of the topK is now at index topKStart - 1
+        // since we removed an element before the topK
+        const moveOutIndex = this.#topKStart - 1
+        if (moveOutIndex < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[moveOutIndex]!
+        } else {
+          // No value is moving out of the topK
+          // because there are no elements in the topK
+          result.moveOut = null
+        }
+      }
+
+      // Since we removed an element that was before or in the topK
+      // the first element after the topK moved one position to the left
+      // and thus falls into the topK now
+      const moveInIndex = this.#topKEnd - 1
+      if (moveInIndex < this.#sortedValues.length) {
+        result.moveIn = this.#sortedValues[moveInIndex]!
+      }
+    }
+
+    return result
+  }
+
+  // TODO: see if there is a way to refactor the code for insert and delete in the topK above
+  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right
+  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern
+
+  #findIndex(value: V): number {
+    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>
+      this.#comparator(getValue(a), getValue(b)),
+    )
+  }
+}

package/src/operators/topKState.ts

@@ -0,0 +1,111 @@
+import { TopKArray } from './topKArray.js'
+import type {
+  IndexedValue,
+  TopK,
+  TopKChanges,
+  TopKMoveChanges,
+} from './topKArray.js'
+
+/**
+ * Helper class that manages the state for a single topK window.
+ * Encapsulates the multiplicity tracking and topK data structure,
+ * providing a clean interface for processing elements and moving the window.
+ *
+ * This class is used by both TopKWithFractionalIndexOperator (single instance)
+ * and GroupedTopKWithFractionalIndexOperator (one instance per group).
+ */
+export class TopKState<K extends string | number, T> {
+  #multiplicities: Map<K, number> = new Map()
+  #topK: TopK<[K, T]>
+
+  constructor(topK: TopK<[K, T]>) {
+    this.#topK = topK
+  }
+
+  get size(): number {
+    return this.#topK.size
+  }
+
+  get isEmpty(): boolean {
+    return this.#multiplicities.size === 0 && this.#topK.size === 0
+  }
+
+  /**
+   * Process an element update (insert or delete based on multiplicity change).
+   * Returns the changes to the topK window.
+   */
+  processElement(key: K, value: T, multiplicity: number): TopKChanges<[K, T]> {
+    const { oldMultiplicity, newMultiplicity } = this.#updateMultiplicity(
+      key,
+      multiplicity,
+    )
+
+    if (oldMultiplicity <= 0 && newMultiplicity > 0) {
+      // The value was invisible but should now be visible
+      return this.#topK.insert([key, value])
+    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
+      // The value was visible but should now be invisible
+      return this.#topK.delete([key, value])
+    }
+    // The value was invisible and remains invisible,
+    // or was visible and remains visible - no topK change
+    return { moveIn: null, moveOut: null }
+  }
+
+  /**
+   * Move the topK window. Only works with TopKArray implementation.
+   */
+  move(options: { offset?: number; limit?: number }): TopKMoveChanges<[K, T]> {
+    if (!(this.#topK instanceof TopKArray)) {
+      throw new Error(
+        `Cannot move B+-tree implementation of TopK with fractional index`,
+      )
+    }
+    return this.#topK.move(options)
+  }
+
+  #updateMultiplicity(
+    key: K,
+    multiplicity: number,
+  ): { oldMultiplicity: number; newMultiplicity: number } {
+    if (multiplicity === 0) {
+      const current = this.#multiplicities.get(key) ?? 0
+      return { oldMultiplicity: current, newMultiplicity: current }
+    }
+
+    const oldMultiplicity = this.#multiplicities.get(key) ?? 0
+    const newMultiplicity = oldMultiplicity + multiplicity
+    if (newMultiplicity === 0) {
+      this.#multiplicities.delete(key)
+    } else {
+      this.#multiplicities.set(key, newMultiplicity)
+    }
+    return { oldMultiplicity, newMultiplicity }
+  }
+}
+
+/**
+ * Handles a moveIn change by adding it to the result array.
+ */
+export function handleMoveIn<K extends string | number, T>(
+  moveIn: IndexedValue<[K, T]> | null,
+  result: Array<[[K, IndexedValue<T>], number]>,
+): void {
+  if (moveIn) {
+    const [[key, value], index] = moveIn
+    result.push([[key, [value, index]], 1])
+  }
+}
+
+/**
+ * Handles a moveOut change by adding it to the result array.
+ */
+export function handleMoveOut<K extends string | number, T>(
+  moveOut: IndexedValue<[K, T]> | null,
+  result: Array<[[K, IndexedValue<T>], number]>,
+): void {
+  if (moveOut) {
+    const [[key, value], index] = moveOut
+    result.push([[key, [value, index]], -1])
+  }
+}