@tanstack/db-ivm 0.1.20 → 0.1.24

package/src/operators/groupedTopKWithFractionalIndex.ts

@@ -1,7 +1,12 @@
 import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
 import { StreamBuilder } from '../d2.js'
 import { MultiSet } from '../multiset.js'
-import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import {
+  TopKState,
+  handleMoveIn,
+  handleMoveOut,
+  topKBatch,
+} from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
 import type { IndexedValue, TopK } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
@@ -125,11 +130,10 @@ export class GroupedTopKWithFractionalIndexOperator<
 
   run(): void {
     const result: Array<[[K, IndexedValue<T>], number]> = []
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item
-        this.#processElement(key, value, multiplicity, result)
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages(),
+    )) {
+      this.#processElement(key, value, multiplicity, result)
     }
 
     if (result.length > 0) {

package/src/operators/topKArray.ts

@@ -170,6 +170,8 @@ export class TopKArray<V> implements TopK<V> {
     // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !
     this.#sortedValues.splice(index, 0, val)
 
+    if (this.#topKStart === this.#topKEnd) return result
+
     // Check if the topK changed
     if (index < this.#topKEnd) {
       // The inserted element is either before the top K or within the top K
@@ -209,6 +211,8 @@ export class TopKArray<V> implements TopK<V> {
     // Remove the value at that position
     const [removedElem] = this.#sortedValues.splice(index, 1)
 
+    if (this.#topKStart === this.#topKEnd) return result
+
     // Check if the topK changed
     if (index < this.#topKEnd) {
       // The removed element is either before the top K or within the top K

package/src/operators/topKState.ts

@@ -1,4 +1,6 @@
+import { equalHashValues } from '../hashing/hash.js'
 import { TopKArray } from './topKArray.js'
+import type { MultiSet } from '../multiset.js'
 import type {
   IndexedValue,
   TopK,
@@ -6,6 +8,46 @@ import type {
   TopKMoveChanges,
 } from './topKArray.js'
 
+/** Apply a keyed batch's retractions before additions. An outer join can emit
+ * a replacement in the opposite order; key multiplicity alone would hide it.
+ * Consolidate first so a transient value added and removed in this same turn
+ * cannot be mistaken for the final replacement. Use structural relation
+ * identity: map can allocate a fresh value for each side of a cancelling pair.
+ * No state survives the run.
+ */
+export function* topKBatch<K, T>(messages: Array<MultiSet<[K, T]>>) {
+  const keyed = new Map<K, Array<[[K, T], number]>>()
+  for (const message of messages) {
+    for (const [value, weight] of message.getInner()) {
+      const entries = keyed.get(value[0])
+      if (entries) entries.push([value, weight])
+      else keyed.set(value[0], [[value, weight]])
+    }
+  }
+  const batch: Array<[[K, T], number]> = []
+  for (const entries of keyed.values()) {
+    // Ordering distinct keys needs only the caller's comparator. In particular,
+    // do not traverse irrelevant payloads or merge keys on a hash collision.
+    if (entries.length === 1) {
+      batch.push(entries[0]!)
+      continue
+    }
+    // A per-key scan keeps normal two-version replacements cheap and permits
+    // cycles, at the cost of quadratic comparisons for long same-key histories.
+    const consolidated: Array<[[K, T], number]> = []
+    for (const [value, weight] of entries) {
+      const previous = consolidated.find(([candidate]) =>
+        equalHashValues(candidate[1], value[1]),
+      )
+      if (previous) previous[1] += weight
+      else consolidated.push([value, weight])
+    }
+    batch.push(...consolidated)
+  }
+  for (const entry of batch) if (entry[1] < 0) yield entry
+  for (const entry of batch) if (entry[1] > 0) yield entry
+}
+
 /**
  * Helper class that manages the state for a single topK window.
  * Encapsulates the multiplicity tracking and topK data structure,

package/src/operators/topKWithFractionalIndex.ts

@@ -1,7 +1,12 @@
 import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
 import { StreamBuilder } from '../d2.js'
 import { MultiSet } from '../multiset.js'
-import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import {
+  TopKState,
+  handleMoveIn,
+  handleMoveOut,
+  topKBatch,
+} from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
 import type { IndexedValue, TopK } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
@@ -76,11 +81,10 @@ export class TopKWithFractionalIndexOperator<
 
   run(): void {
     const result: Array<[[K, IndexedValue<T>], number]> = []
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item
-        this.processElement(key, value, multiplicity, result)
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages(),
+    )) {
+      this.processElement(key, value, multiplicity, result)
     }
 
     if (result.length > 0) {

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -106,9 +106,11 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#tree.size - 1 < this.#topKStart) {
-      // We don't have a topK yet
-      // so we don't need to do anything
+    if (
+      this.#topKStart === this.#topKEnd ||
+      this.#tree.size - 1 < this.#topKStart
+    ) {
+      // The window is empty, or there aren't enough rows to reach its offset.
       return result
     }
 
@@ -178,11 +180,8 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#comparator(value, getValue(this.#topKFirstElem)) < 0) {
-      // We deleted an element that was before the topK
-      // so the topK has shifted one position to the left
-
-      // the old first element moves out of the topK
+    if (this.#comparator(value, getValue(this.#topKFirstElem)) <= 0) {
+      // Deleting at or before the first selected value advances that boundary.
       result.moveOut = this.#topKFirstElem
       // the element that was right after the first element of the topK
       // is now the new first element of the topK

package/src/utils.ts

@@ -1,3 +1,5 @@
+import { isTemporal } from './hashing/temporal.js'
+
 /**
  * Simple assertion function for runtime checks.
  * Throws an error if the condition is false.
@@ -11,36 +13,6 @@ export function assert(
   }
 }
 
-/**
- * A map that returns a default value for keys that are not present.
- */
-export class DefaultMap<K, V> extends Map<K, V> {
-  constructor(
-    private defaultValue: () => V,
-    entries?: Iterable<[K, V]>,
-  ) {
-    super(entries)
-  }
-
-  get(key: K): V {
-    if (!this.has(key)) {
-      // this.set(key, this.defaultValue())
-      return this.defaultValue()
-    }
-    return super.get(key)!
-  }
-
-  /**
-   * Update the value for a key using a function.
-   */
-  update(key: K, updater: (value: V) => V): V {
-    const value = this.get(key)
-    const newValue = updater(value)
-    this.set(key, newValue)
-    return newValue
-  }
-}
-
 // JS engines have various limits on how many args can be passed to a function
 // with a spread operator, so we need to split the operation into chunks
 // 32767 is the max for Chrome 14, all others are higher
@@ -78,55 +50,21 @@ export function binarySearch<T>(
   return low
 }
 
-/**
- * Utility for generating unique IDs for objects and values.
- * Uses WeakMap for object reference tracking and consistent hashing for primitives.
- */
-export class ObjectIdGenerator {
-  private objectIds = new WeakMap<object, number>()
-  private nextId = 0
-
-  /**
-   * Get a unique identifier for any value.
-   * - Objects: Uses WeakMap for reference-based identity
-   * - Primitives: Uses consistent string-based hashing
-   */
-  getId(value: any): number {
-    // For primitives, use a simple hash of their string representation
-    if (typeof value !== `object` || value === null) {
-      const str = String(value)
-      let hashValue = 0
-      for (let i = 0; i < str.length; i++) {
-        const char = str.charCodeAt(i)
-        hashValue = (hashValue << 5) - hashValue + char
-        hashValue = hashValue & hashValue // Convert to 32-bit integer
-      }
-      return hashValue
-    }
-
-    // For objects, use WeakMap to assign unique IDs
-    if (!this.objectIds.has(value)) {
-      this.objectIds.set(value, this.nextId++)
-    }
-    return this.objectIds.get(value)!
-  }
-
-  /**
-   * Get a string representation of the ID for use in composite keys.
-   */
-  getStringId(value: any): string {
-    if (value === null) return `null`
-    if (value === undefined) return `undefined`
-    if (typeof value !== `object`) return `str_${String(value)}`
-
-    return `obj_${this.getId(value)}`
-  }
-}
+const objectIds = new WeakMap<object, number>()
+let nextObjectId = 0
 
 /**
- * Global instance for cases where a shared object ID space is needed.
+ * A string identity for composite keys: primitives by value, objects by
+ * reference.
  */
-export const globalObjectIdGenerator = new ObjectIdGenerator()
+export function getStringId(value: unknown): string {
+  if (value === null) return `null`
+  if (value === undefined) return `undefined`
+  if (typeof value !== `object`) return `str_${String(value)}`
+  let id = objectIds.get(value)
+  if (id === undefined) objectIds.set(value, (id = nextObjectId++))
+  return `obj_${id}`
+}
 
 export function* concatIterable<T>(
   ...iterables: Array<Iterable<T>>
@@ -209,6 +147,7 @@ type CanonicalValue =
   | readonly [`bigint`, string]
   | readonly [`string`, string]
   | readonly [`date`, number | `Invalid`]
+  | readonly [`temporal`, string, string]
   | readonly [`regexp`, string, string]
   | readonly [`bytes`, Array<number>]
   | readonly [`array`, Array<CanonicalValue>]
@@ -265,6 +204,7 @@ function isJsonSafeStructuralValue(
       value instanceof Date ||
       value instanceof RegExp ||
       value instanceof Uint8Array ||
+      isTemporal(value) ||
       value instanceof Map ||
       value instanceof Set
     ) {
@@ -348,6 +288,10 @@ function toCanonicalValue(
       return [`bytes`, Array.from(value)]
     }
 
+    if (isTemporal(value)) {
+      return [`temporal`, value[Symbol.toStringTag], value.toString()]
+    }
+
     if (Array.isArray(value)) {
       return [`array`, value.map((item) => toCanonicalValue(item, ancestors))]
     }