@tanstack/db-ivm 0.1.19 → 0.1.22

package/src/operators/topKState.ts

@@ -1,4 +1,6 @@
+import { equalHashValues } from '../hashing/hash.js'
 import { TopKArray } from './topKArray.js'
+import type { MultiSet } from '../multiset.js'
 import type {
   IndexedValue,
   TopK,
@@ -6,6 +8,46 @@ import type {
   TopKMoveChanges,
 } from './topKArray.js'
 
+/** Apply a keyed batch's retractions before additions. An outer join can emit
+ * a replacement in the opposite order; key multiplicity alone would hide it.
+ * Consolidate first so a transient value added and removed in this same turn
+ * cannot be mistaken for the final replacement. Use structural relation
+ * identity: map can allocate a fresh value for each side of a cancelling pair.
+ * No state survives the run.
+ */
+export function* topKBatch<K, T>(messages: Array<MultiSet<[K, T]>>) {
+  const keyed = new Map<K, Array<[[K, T], number]>>()
+  for (const message of messages) {
+    for (const [value, weight] of message.getInner()) {
+      const entries = keyed.get(value[0])
+      if (entries) entries.push([value, weight])
+      else keyed.set(value[0], [[value, weight]])
+    }
+  }
+  const batch: Array<[[K, T], number]> = []
+  for (const entries of keyed.values()) {
+    // Ordering distinct keys needs only the caller's comparator. In particular,
+    // do not traverse irrelevant payloads or merge keys on a hash collision.
+    if (entries.length === 1) {
+      batch.push(entries[0]!)
+      continue
+    }
+    // A per-key scan keeps normal two-version replacements cheap and permits
+    // cycles, at the cost of quadratic comparisons for long same-key histories.
+    const consolidated: Array<[[K, T], number]> = []
+    for (const [value, weight] of entries) {
+      const previous = consolidated.find(([candidate]) =>
+        equalHashValues(candidate[1], value[1]),
+      )
+      if (previous) previous[1] += weight
+      else consolidated.push([value, weight])
+    }
+    batch.push(...consolidated)
+  }
+  for (const entry of batch) if (entry[1] < 0) yield entry
+  for (const entry of batch) if (entry[1] > 0) yield entry
+}
+
 /**
  * Helper class that manages the state for a single topK window.
  * Encapsulates the multiplicity tracking and topK data structure,

package/src/operators/topKWithFractionalIndex.ts

@@ -1,7 +1,12 @@
 import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
 import { StreamBuilder } from '../d2.js'
 import { MultiSet } from '../multiset.js'
-import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import {
+  TopKState,
+  handleMoveIn,
+  handleMoveOut,
+  topKBatch,
+} from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
 import type { IndexedValue, TopK } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
@@ -76,11 +81,10 @@ export class TopKWithFractionalIndexOperator<
 
   run(): void {
     const result: Array<[[K, IndexedValue<T>], number]> = []
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item
-        this.processElement(key, value, multiplicity, result)
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages(),
+    )) {
+      this.processElement(key, value, multiplicity, result)
     }
 
     if (result.length > 0) {

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -106,9 +106,11 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#tree.size - 1 < this.#topKStart) {
-      // We don't have a topK yet
-      // so we don't need to do anything
+    if (
+      this.#topKStart === this.#topKEnd ||
+      this.#tree.size - 1 < this.#topKStart
+    ) {
+      // The window is empty, or there aren't enough rows to reach its offset.
       return result
     }
 
@@ -178,11 +180,8 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#comparator(value, getValue(this.#topKFirstElem)) < 0) {
-      // We deleted an element that was before the topK
-      // so the topK has shifted one position to the left
-
-      // the old first element moves out of the topK
+    if (this.#comparator(value, getValue(this.#topKFirstElem)) <= 0) {
+      // Deleting at or before the first selected value advances that boundary.
       result.moveOut = this.#topKFirstElem
       // the element that was right after the first element of the topK
       // is now the new first element of the topK

package/src/utils.ts

@@ -185,6 +185,14 @@ function range(start: number, end: number): Array<number> {
 export function compareKeys(a: string | number, b: string | number): number {
   // Same type: compare directly
   if (typeof a === typeof b) {
+    if (typeof a === `number` && typeof b === `number`) {
+      const aIsNaN = Number.isNaN(a)
+      const bIsNaN = Number.isNaN(b)
+      if (aIsNaN || bIsNaN) {
+        if (aIsNaN && bIsNaN) return 0
+        return aIsNaN ? 1 : -1
+      }
+    }
     if (a < b) return -1
     if (a > b) return 1
     return 0

package/dist/cjs/operators/orderByBTree.d.cts

@@ -1,3 +0,0 @@
-import { KeyValue } from '../types.js';
-import { OrderByOptions } from './orderBy.js';
-export declare function orderByWithFractionalIndexBTree<T extends KeyValue<unknown, unknown>, Ve = unknown>(valueExtractor: (value: T extends KeyValue<unknown, infer V> ? V : never) => Ve, options?: OrderByOptions<Ve>): (stream: import('../types.js').IStreamBuilder<T>) => import('../types.js').IStreamBuilder<[T extends KeyValue<infer K, unknown> ? K : never, [T extends KeyValue<unknown, infer V> ? V : never, string]]>;

package/dist/esm/operators/orderByBTree.d.ts

@@ -1,3 +0,0 @@
-import { KeyValue } from '../types.js';
-import { OrderByOptions } from './orderBy.js';
-export declare function orderByWithFractionalIndexBTree<T extends KeyValue<unknown, unknown>, Ve = unknown>(valueExtractor: (value: T extends KeyValue<unknown, infer V> ? V : never) => Ve, options?: OrderByOptions<Ve>): (stream: import('../types.js').IStreamBuilder<T>) => import('../types.js').IStreamBuilder<[T extends KeyValue<infer K, unknown> ? K : never, [T extends KeyValue<unknown, infer V> ? V : never, string]]>;

package/src/operators/orderByBTree.ts

@@ -1,20 +0,0 @@
-import { orderByWithFractionalIndexBase } from './orderBy.js'
-import { topKWithFractionalIndexBTree } from './topKWithFractionalIndexBTree.js'
-import type { KeyValue } from '../types.js'
-import type { OrderByOptions } from './orderBy.js'
-
-export function orderByWithFractionalIndexBTree<
-  T extends KeyValue<unknown, unknown>,
-  Ve = unknown,
->(
-  valueExtractor: (
-    value: T extends KeyValue<unknown, infer V> ? V : never,
-  ) => Ve,
-  options?: OrderByOptions<Ve>,
-) {
-  return orderByWithFractionalIndexBase(
-    topKWithFractionalIndexBTree,
-    valueExtractor,
-    options,
-  )
-}