@tanstack/db-ivm 0.1.12 → 0.1.14

package/src/operators/reduce.ts

@@ -1,9 +1,9 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { MultiSet } from "../multiset.js"
-import { Index } from "../indexes.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder, KeyValue } from "../types.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { MultiSet } from '../multiset.js'
+import { Index } from '../indexes.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, KeyValue } from '../types.js'
 
 /**
  * Base operator for reduction operations (version-free)
@@ -17,7 +17,7 @@ export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
     id: number,
     inputA: DifferenceStreamReader<[K, V1]>,
     output: DifferenceStreamWriter<[K, V2]>,
-    f: (values: Array<[V1, number]>) => Array<[V2, number]>
+    f: (values: Array<[V1, number]>) => Array<[V2, number]>,
   ) {
     super(id, inputA, output)
     this.#f = f
@@ -109,13 +109,13 @@ export function reduce<
   return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<KType, R>> => {
     const output = new StreamBuilder<KeyValue<KType, R>>(
       stream.graph,
-      new DifferenceStreamWriter<KeyValue<KType, R>>()
+      new DifferenceStreamWriter<KeyValue<KType, R>>(),
     )
     const operator = new ReduceOperator<KType, V1Type, R>(
       stream.graph.getNextOperatorId(),
       stream.connectReader() as DifferenceStreamReader<KeyValue<KType, V1Type>>,
       output.writer,
-      f
+      f,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/tap.ts

@@ -1,8 +1,8 @@
-import { DifferenceStreamWriter, LinearUnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { MultiSet } from "../multiset.js"
+import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { MultiSet } from '../multiset.js'
 
 /**
  * Operator that applies a function to each multi-set in the input stream
@@ -14,7 +14,7 @@ export class TapOperator<T> extends LinearUnaryOperator<T, T> {
     id: number,
     inputA: DifferenceStreamReader<T>,
     output: DifferenceStreamWriter<T>,
-    f: (data: MultiSet<T>) => void
+    f: (data: MultiSet<T>) => void,
   ) {
     super(id, inputA, output)
     this.#f = f
@@ -36,13 +36,13 @@ export function tap<T>(f: (data: MultiSet<T>) => void): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new TapOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
-      f
+      f,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/topK.ts

@@ -1,6 +1,6 @@
-import { MultiSet } from "../multiset.js"
-import { reduce } from "./reduce.js"
-import type { IStreamBuilder, KeyValue, PipedOperator } from "../types"
+import { MultiSet } from '../multiset.js'
+import { reduce } from './reduce.js'
+import type { IStreamBuilder, KeyValue, PipedOperator } from '../types'
 
 interface TopKOptions {
   limit?: number
@@ -24,7 +24,7 @@ export function topK<
   T,
 >(
   comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions
+  options?: TopKOptions,
 ): PipedOperator<T, T> {
   const limit = options?.limit ?? Infinity
   const offset = options?.offset ?? 0
@@ -38,7 +38,7 @@ export function topK<
           .getInner()
           .sort((a, b) => comparator(a[0] as V1Type, b[0] as V1Type))
         return sortedValues.slice(offset, offset + limit)
-      })
+      }),
     )
     return reduced as IStreamBuilder<T>
   }
@@ -62,13 +62,13 @@ export function topKWithIndex<
   T,
 >(
   comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions
+  options?: TopKOptions,
 ): PipedOperator<T, KeyValue<KType, [V1Type, number]>> {
   const limit = options?.limit ?? Infinity
   const offset = options?.offset ?? 0
 
   return (
-    stream: IStreamBuilder<T>
+    stream: IStreamBuilder<T>,
   ): IStreamBuilder<KeyValue<KType, [V1Type, number]>> => {
     const reduced = stream.pipe(
       reduce<KType, V1Type, [V1Type, number], T>((values) => {
@@ -84,7 +84,7 @@ export function topKWithIndex<
             multiplicity,
           ])
         return sortedValues
-      })
+      }),
     )
     return reduced
   }

package/src/operators/topKWithFractionalIndex.ts

@@ -1,22 +1,18 @@
-import { generateKeyBetween } from "fractional-indexing"
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { MultiSet } from "../multiset.js"
-import {
-  binarySearch,
-  diffHalfOpen,
-  globalObjectIdGenerator,
-} from "../utils.js"
-import type { HRange } from "../utils.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
+import { generateKeyBetween } from 'fractional-indexing'
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { MultiSet } from '../multiset.js'
+import { binarySearch, compareKeys, diffHalfOpen } from '../utils.js'
+import type { HRange } from '../utils.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
 
 export interface TopKWithFractionalIndexOptions {
   limit?: number
   offset?: number
   setSizeCallback?: (getSize: () => number) => void
   setWindowFn?: (
-    windowFn: (options: { offset?: number; limit?: number }) => void
+    windowFn: (options: { offset?: number; limit?: number }) => void,
   ) => void
 }
 
@@ -61,7 +57,7 @@ class TopKArray<V> implements TopK<V> {
   constructor(
     offset: number,
     limit: number,
-    comparator: (a: V, b: V) => number
+    comparator: (a: V, b: V) => number,
   ) {
     this.#topKStart = offset
     this.#topKEnd = offset + limit
@@ -229,7 +225,7 @@ class TopKArray<V> implements TopK<V> {
 
   #findIndex(value: V): number {
     return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>
-      this.#comparator(getValue(a), getValue(b))
+      this.#comparator(getValue(a), getValue(b)),
     )
   }
 }
@@ -239,43 +235,34 @@ class TopKArray<V> implements TopK<V> {
  * This operator maintains fractional indices for sorted elements
  * and only updates indices when elements move position
  */
-export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
-  [K, T],
-  [K, IndexedValue<T>]
-> {
+export class TopKWithFractionalIndexOperator<
+  K extends string | number,
+  T,
+> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {
   #index: Map<K, number> = new Map() // maps keys to their multiplicity
 
   /**
    * topK data structure that supports insertions and deletions
    * and returns changes to the topK.
+   * Elements are stored as [key, value] tuples for stable tie-breaking.
    */
-  #topK: TopK<TaggedValue<K, T>>
+  #topK: TopK<[K, T]>
 
   constructor(
     id: number,
     inputA: DifferenceStreamReader<[K, T]>,
     output: DifferenceStreamWriter<[K, IndexedValue<T>]>,
     comparator: (a: T, b: T) => number,
-    options: TopKWithFractionalIndexOptions
+    options: TopKWithFractionalIndexOptions,
   ) {
     super(id, inputA, output)
     const limit = options.limit ?? Infinity
     const offset = options.offset ?? 0
-    const compareTaggedValues = (
-      a: TaggedValue<K, T>,
-      b: TaggedValue<K, T>
-    ) => {
-      // First compare on the value
-      const valueComparison = comparator(getVal(a), getVal(b))
-      if (valueComparison !== 0) {
-        return valueComparison
-      }
-      // If the values are equal, compare on the tag (object identity)
-      const tieBreakerA = getTag(a)
-      const tieBreakerB = getTag(b)
-      return tieBreakerA - tieBreakerB
-    }
-    this.#topK = this.createTopK(offset, limit, compareTaggedValues)
+    this.#topK = this.createTopK(
+      offset,
+      limit,
+      createKeyedComparator(comparator),
+    )
     options.setSizeCallback?.(() => this.#topK.size)
     options.setWindowFn?.(this.moveTopK.bind(this))
   }
@@ -283,8 +270,8 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
   protected createTopK(
     offset: number,
     limit: number,
-    comparator: (a: TaggedValue<K, T>, b: TaggedValue<K, T>) => number
-  ): TopK<TaggedValue<K, T>> {
+    comparator: (a: [K, T], b: [K, T]) => number,
+  ): TopK<[K, T]> {
     return new TopKArray(offset, limit, comparator)
   }
 
@@ -295,7 +282,7 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
   moveTopK({ offset, limit }: { offset?: number; limit?: number }) {
     if (!(this.#topK instanceof TopKArray)) {
       throw new Error(
-        `Cannot move B+-tree implementation of TopK with fractional index`
+        `Cannot move B+-tree implementation of TopK with fractional index`,
       )
     }
 
@@ -332,24 +319,22 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
     key: K,
     value: T,
     multiplicity: number,
-    result: Array<[[K, IndexedValue<T>], number]>
+    result: Array<[[K, IndexedValue<T>], number]>,
   ): void {
     const { oldMultiplicity, newMultiplicity } = this.addKey(key, multiplicity)
 
-    let res: TopKChanges<TaggedValue<K, T>> = {
+    let res: TopKChanges<[K, T]> = {
       moveIn: null,
       moveOut: null,
     }
     if (oldMultiplicity <= 0 && newMultiplicity > 0) {
       // The value was invisible but should now be visible
       // Need to insert it into the array of sorted values
-      const taggedValue = tagValue(key, value)
-      res = this.#topK.insert(taggedValue)
+      res = this.#topK.insert([key, value])
     } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
       // The value was visible but should now be invisible
       // Need to remove it from the array of sorted values
-      const taggedValue = tagValue(key, value)
-      res = this.#topK.delete(taggedValue)
+      res = this.#topK.delete([key, value])
     } else {
       // The value was invisible and it remains invisible
       // or it was visible and remains visible
@@ -363,28 +348,22 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
   }
 
   private handleMoveIn(
-    moveIn: IndexedValue<TaggedValue<K, T>> | null,
-    result: Array<[[K, IndexedValue<T>], number]>
+    moveIn: IndexedValue<[K, T]> | null,
+    result: Array<[[K, IndexedValue<T>], number]>,
   ) {
     if (moveIn) {
-      const index = getIndex(moveIn)
-      const taggedValue = getValue(moveIn)
-      const k = getKey(taggedValue)
-      const val = getVal(taggedValue)
-      result.push([[k, [val, index]], 1])
+      const [[key, value], index] = moveIn
+      result.push([[key, [value, index]], 1])
     }
   }
 
   private handleMoveOut(
-    moveOut: IndexedValue<TaggedValue<K, T>> | null,
-    result: Array<[[K, IndexedValue<T>], number]>
+    moveOut: IndexedValue<[K, T]> | null,
+    result: Array<[[K, IndexedValue<T>], number]>,
   ) {
     if (moveOut) {
-      const index = getIndex(moveOut)
-      const taggedValue = getValue(moveOut)
-      const k = getKey(taggedValue)
-      const val = getVal(taggedValue)
-      result.push([[k, [val, index]], -1])
+      const [[key, value], index] = moveOut
+      result.push([[key, [value, index]], -1])
     }
   }
 
@@ -394,7 +373,7 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
 
   private addKey(
     key: K,
-    multiplicity: number
+    multiplicity: number,
   ): { oldMultiplicity: number; newMultiplicity: number } {
     const oldMultiplicity = this.getMultiplicity(key)
     const newMultiplicity = oldMultiplicity + multiplicity
@@ -417,25 +396,25 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
  * @param options - An optional object containing limit and offset properties
  * @returns A piped operator that orders the elements and limits the number of results
  */
-export function topKWithFractionalIndex<KType, T>(
+export function topKWithFractionalIndex<KType extends string | number, T>(
   comparator: (a: T, b: T) => number,
-  options?: TopKWithFractionalIndexOptions
+  options?: TopKWithFractionalIndexOptions,
 ): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {
   const opts = options || {}
 
   return (
-    stream: IStreamBuilder<[KType, T]>
+    stream: IStreamBuilder<[KType, T]>,
   ): IStreamBuilder<[KType, IndexedValue<T>]> => {
     const output = new StreamBuilder<[KType, IndexedValue<T>]>(
       stream.graph,
-      new DifferenceStreamWriter<[KType, IndexedValue<T>]>()
+      new DifferenceStreamWriter<[KType, IndexedValue<T>]>(),
     )
     const operator = new TopKWithFractionalIndexOperator<KType, T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
       comparator,
-      opts
+      opts,
     )
     stream.graph.addOperator(operator)
     return output
@@ -448,7 +427,7 @@ export type IndexedValue<V> = [V, FractionalIndex]
 
 export function indexedValue<V>(
   value: V,
-  index: FractionalIndex
+  index: FractionalIndex,
 ): IndexedValue<V> {
   return [value, index]
 }
@@ -461,21 +440,21 @@ export function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {
   return indexedVal[1]
 }
 
-export type Tag = number
-export type TaggedValue<K, V> = [K, V, Tag]
-
-function tagValue<K, V>(key: K, value: V): TaggedValue<K, V> {
-  return [key, value, globalObjectIdGenerator.getId(key)]
-}
-
-function getKey<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): K {
-  return tieBreakerTaggedValue[0]
-}
-
-function getVal<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): V {
-  return tieBreakerTaggedValue[1]
-}
-
-function getTag<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): Tag {
-  return tieBreakerTaggedValue[2]
+/**
+ * Creates a comparator for [key, value] tuples that first compares values,
+ * then uses the row key as a stable tie-breaker.
+ */
+function createKeyedComparator<K extends string | number, T>(
+  comparator: (a: T, b: T) => number,
+): (a: [K, T], b: [K, T]) => number {
+  return ([aKey, aVal], [bKey, bVal]) => {
+    // First compare on the value
+    const valueComparison = comparator(aVal, bVal)
+    if (valueComparison !== 0) {
+      return valueComparison
+    }
+    // If the values are equal, use the row key as tie-breaker
+    // This provides stable, deterministic ordering since keys are string | number
+    return compareKeys(aKey, bKey)
+  }
 }

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -1,20 +1,19 @@
-import { generateKeyBetween } from "fractional-indexing"
-import { DifferenceStreamWriter } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
+import { generateKeyBetween } from 'fractional-indexing'
+import { DifferenceStreamWriter } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
 import {
   TopKWithFractionalIndexOperator,
   getIndex,
   getValue,
   indexedValue,
-} from "./topKWithFractionalIndex.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
+} from './topKWithFractionalIndex.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
 import type {
   IndexedValue,
-  TaggedValue,
   TopK,
   TopKChanges,
   TopKWithFractionalIndexOptions,
-} from "./topKWithFractionalIndex.js"
+} from './topKWithFractionalIndex.js'
 
 interface BTree<Key, Value> {
   nextLowerPair: (key: Key) => [Key, Value] | undefined
@@ -30,7 +29,7 @@ interface BTreeClass {
   new <Key, Value>(
     entries?: Array<[Key, Value]>,
     compare?: (a: Key, b: Key) => number,
-    maxNodeSize?: number
+    maxNodeSize?: number,
   ): BTree<Key, Value>
 }
 
@@ -61,11 +60,11 @@ class TopKTree<V> implements TopK<V> {
   constructor(
     offset: number,
     limit: number,
-    comparator: (a: V, b: V) => number
+    comparator: (a: V, b: V) => number,
   ) {
     if (BTree === undefined) {
       throw new Error(
-        `B+ tree not loaded. You need to call loadBTree() before using TopKTree.`
+        `B+ tree not loaded. You need to call loadBTree() before using TopKTree.`,
       )
     }
 
@@ -243,17 +242,17 @@ class TopKTree<V> implements TopK<V> {
  * and only updates indices when elements move position
  */
 export class TopKWithFractionalIndexBTreeOperator<
-  K,
+  K extends string | number,
   T,
 > extends TopKWithFractionalIndexOperator<K, T> {
   protected override createTopK(
     offset: number,
     limit: number,
-    comparator: (a: TaggedValue<K, T>, b: TaggedValue<K, T>) => number
-  ): TopK<TaggedValue<K, T>> {
+    comparator: (a: [K, T], b: [K, T]) => number,
+  ): TopK<[K, T]> {
     if (BTree === undefined) {
       throw new Error(
-        `B+ tree not loaded. You need to call loadBTree() before using TopKWithFractionalIndexBTreeOperator.`
+        `B+ tree not loaded. You need to call loadBTree() before using TopKWithFractionalIndexBTreeOperator.`,
       )
     }
     return new TopKTree(offset, limit, comparator)
@@ -275,31 +274,31 @@ export class TopKWithFractionalIndexBTreeOperator<
  * @param options - An optional object containing limit and offset properties
  * @returns A piped operator that orders the elements and limits the number of results
  */
-export function topKWithFractionalIndexBTree<KType, T>(
+export function topKWithFractionalIndexBTree<KType extends string | number, T>(
   comparator: (a: T, b: T) => number,
-  options?: TopKWithFractionalIndexOptions
+  options?: TopKWithFractionalIndexOptions,
 ): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {
   const opts = options || {}
 
   if (BTree === undefined) {
     throw new Error(
-      `B+ tree not loaded. You need to call loadBTree() before using topKWithFractionalIndexBTree.`
+      `B+ tree not loaded. You need to call loadBTree() before using topKWithFractionalIndexBTree.`,
     )
   }
 
   return (
-    stream: IStreamBuilder<[KType, T]>
+    stream: IStreamBuilder<[KType, T]>,
   ): IStreamBuilder<[KType, IndexedValue<T>]> => {
     const output = new StreamBuilder<[KType, IndexedValue<T>]>(
       stream.graph,
-      new DifferenceStreamWriter<[KType, IndexedValue<T>]>()
+      new DifferenceStreamWriter<[KType, IndexedValue<T>]>(),
     )
     const operator = new TopKWithFractionalIndexBTreeOperator<KType, T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
       comparator,
-      opts
+      opts,
     )
     stream.graph.addOperator(operator)
     return output