@tanstack/db-ivm 0.1.20 → 0.1.22

package/src/hashing/hash.ts

@@ -13,10 +13,12 @@ const UNDEFINED = randomHash()
 const KEY = randomHash()
 const FUNCTIONS = randomHash()
 const DATE_MARKER = randomHash()
-const OBJECT_MARKER = randomHash()
-const ARRAY_MARKER = randomHash()
-const MAP_MARKER = randomHash()
-const SET_MARKER = randomHash()
+const STRUCTURAL_MARKERS = {
+  object: randomHash(),
+  array: randomHash(),
+  map: randomHash(),
+  set: randomHash(),
+}
 const UINT8ARRAY_MARKER = randomHash()
 const TEMPORAL_MARKER = randomHash()
 // Bound structural recursion and value visits. Shared acyclic subtrees are
@@ -51,6 +53,7 @@ function isTemporal(input: object): input is TemporalLike {
 const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128
 
 const hashCache = new WeakMap<object, number>()
+const referenceValues = new WeakSet<object>()
 
 /** @internal Register a mutable handle before it enters a structural value. */
 export function registerOpaqueHash(value: object): void {
@@ -87,24 +90,12 @@ function hashObject(input: object, context: HashContext): number {
     } else if (isTemporal(input)) {
       valueHash = hashTemporal(input)
     } else {
-      let plainObjectInput = input
-      let marker = OBJECT_MARKER
-
-      if (input instanceof Array) {
-        marker = ARRAY_MARKER
-      }
-
-      if (input instanceof Map) {
-        marker = MAP_MARKER
-        plainObjectInput = [...input.entries()]
-      }
-
-      if (input instanceof Set) {
-        marker = SET_MARKER
-        plainObjectInput = [...input.entries()]
-      }
-
-      valueHash = hashPlainObject(plainObjectInput, marker, context)
+      const [kind, plainObjectInput] = structuralShape(input)
+      valueHash = hashPlainObject(
+        plainObjectInput,
+        STRUCTURAL_MARKERS[kind],
+        context,
+      )
     }
   } finally {
     context.activeObjects.delete(input)
@@ -261,6 +252,80 @@ function isBinaryValue(input: object): input is Uint8Array {
   )
 }
 
+function structuralShape(
+  input: object,
+): [keyof typeof STRUCTURAL_MARKERS, object] {
+  if (input instanceof Map) return [`map`, [...input.entries()]]
+  if (input instanceof Set) return [`set`, [...input.values()]]
+  return [input instanceof Array ? `array` : `object`, input]
+}
+
+/** @internal Compare immutable structural values without computing a digest.
+ * Pair memoization also permits cyclic values that structural hashing rejects.
+ * Reference-valued leaves remain opaque, including registered mutable handles.
+ */
+export function equalHashValues(left: unknown, right: unknown): boolean {
+  const compared = new Map<object, Set<object>>()
+  function equal(a: unknown, b: unknown): boolean {
+    if (a === b || (Number.isNaN(a) && Number.isNaN(b))) return true
+    if (
+      a === null ||
+      b === null ||
+      typeof a !== `object` ||
+      typeof b !== `object`
+    )
+      return false
+    if (
+      referenceValues.has(a) ||
+      referenceValues.has(b) ||
+      isReferenceHashedObject(a) ||
+      isReferenceHashedObject(b)
+    )
+      return false
+    if (a instanceof Date || b instanceof Date)
+      return (
+        a instanceof Date &&
+        b instanceof Date &&
+        equal(a.getTime(), b.getTime())
+      )
+    if (isBinaryValue(a) || isBinaryValue(b))
+      return (
+        isBinaryValue(a) &&
+        isBinaryValue(b) &&
+        a.byteLength === b.byteLength &&
+        a.every((value, index) => value === b[index])
+      )
+    if (isTemporal(a) || isTemporal(b))
+      return (
+        isTemporal(a) &&
+        isTemporal(b) &&
+        a[Symbol.toStringTag] === b[Symbol.toStringTag] &&
+        a.toString() === b.toString()
+      )
+
+    // Revisited pairs close cycles and avoid expanding shared subtrees.
+    const peers = compared.get(a)
+    if (peers?.has(b)) return true
+    if (peers) peers.add(b)
+    else compared.set(a, new Set([b]))
+    const [aKind, aShape] = structuralShape(a)
+    const [bKind, bShape] = structuralShape(b)
+    if (aKind !== bKind) return false
+    const keys = (value: object) =>
+      Reflect.ownKeys(value).filter((key) =>
+        Object.prototype.propertyIsEnumerable.call(value, key),
+      )
+    const aKeys = keys(aShape)
+    if (aKeys.length !== keys(bShape).length) return false
+    return aKeys.every(
+      (key) =>
+        Object.prototype.propertyIsEnumerable.call(bShape, key) &&
+        equal(aShape[key as keyof object], bShape[key as keyof object]),
+    )
+  }
+  return equal(left, right)
+}
+
 let nextRefId = 1
 function cachedReferenceHash(fn: object): number {
   let valueHash = hashCache.get(fn)
@@ -268,6 +333,7 @@ function cachedReferenceHash(fn: object): number {
     valueHash = nextRefId ^ FUNCTIONS
     nextRefId++
     hashCache.set(fn, valueHash)
+    referenceValues.add(fn)
   }
   return valueHash
 }

package/src/operators/distinct.ts

@@ -85,10 +85,12 @@ export class DistinctOperator<
 export function distinct<T extends KeyValue<any, any>>(
   by: (value: T) => any = (value: T) => value,
 ) {
-  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
-    const output = new StreamBuilder<T>(
+  return (
+    stream: IStreamBuilder<T>,
+  ): IStreamBuilder<KeyValue<number, GetValue<T>>> => {
+    const output = new StreamBuilder<KeyValue<number, GetValue<T>>>(
       stream.graph,
-      new DifferenceStreamWriter<T>(),
+      new DifferenceStreamWriter<KeyValue<number, GetValue<T>>>(),
     )
     const operator = new DistinctOperator<T>(
       stream.graph.getNextOperatorId(),

package/src/operators/groupedTopKWithFractionalIndex.ts

@@ -1,7 +1,12 @@
 import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
 import { StreamBuilder } from '../d2.js'
 import { MultiSet } from '../multiset.js'
-import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import {
+  TopKState,
+  handleMoveIn,
+  handleMoveOut,
+  topKBatch,
+} from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
 import type { IndexedValue, TopK } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
@@ -125,11 +130,10 @@ export class GroupedTopKWithFractionalIndexOperator<
 
   run(): void {
     const result: Array<[[K, IndexedValue<T>], number]> = []
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item
-        this.#processElement(key, value, multiplicity, result)
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages(),
+    )) {
+      this.#processElement(key, value, multiplicity, result)
     }
 
     if (result.length > 0) {

package/src/operators/topKArray.ts

@@ -170,6 +170,8 @@ export class TopKArray<V> implements TopK<V> {
     // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !
     this.#sortedValues.splice(index, 0, val)
 
+    if (this.#topKStart === this.#topKEnd) return result
+
     // Check if the topK changed
     if (index < this.#topKEnd) {
       // The inserted element is either before the top K or within the top K
@@ -209,6 +211,8 @@ export class TopKArray<V> implements TopK<V> {
     // Remove the value at that position
     const [removedElem] = this.#sortedValues.splice(index, 1)
 
+    if (this.#topKStart === this.#topKEnd) return result
+
     // Check if the topK changed
     if (index < this.#topKEnd) {
       // The removed element is either before the top K or within the top K

package/src/operators/topKState.ts

@@ -1,4 +1,6 @@
+import { equalHashValues } from '../hashing/hash.js'
 import { TopKArray } from './topKArray.js'
+import type { MultiSet } from '../multiset.js'
 import type {
   IndexedValue,
   TopK,
@@ -6,6 +8,46 @@ import type {
   TopKMoveChanges,
 } from './topKArray.js'
 
+/** Apply a keyed batch's retractions before additions. An outer join can emit
+ * a replacement in the opposite order; key multiplicity alone would hide it.
+ * Consolidate first so a transient value added and removed in this same turn
+ * cannot be mistaken for the final replacement. Use structural relation
+ * identity: map can allocate a fresh value for each side of a cancelling pair.
+ * No state survives the run.
+ */
+export function* topKBatch<K, T>(messages: Array<MultiSet<[K, T]>>) {
+  const keyed = new Map<K, Array<[[K, T], number]>>()
+  for (const message of messages) {
+    for (const [value, weight] of message.getInner()) {
+      const entries = keyed.get(value[0])
+      if (entries) entries.push([value, weight])
+      else keyed.set(value[0], [[value, weight]])
+    }
+  }
+  const batch: Array<[[K, T], number]> = []
+  for (const entries of keyed.values()) {
+    // Ordering distinct keys needs only the caller's comparator. In particular,
+    // do not traverse irrelevant payloads or merge keys on a hash collision.
+    if (entries.length === 1) {
+      batch.push(entries[0]!)
+      continue
+    }
+    // A per-key scan keeps normal two-version replacements cheap and permits
+    // cycles, at the cost of quadratic comparisons for long same-key histories.
+    const consolidated: Array<[[K, T], number]> = []
+    for (const [value, weight] of entries) {
+      const previous = consolidated.find(([candidate]) =>
+        equalHashValues(candidate[1], value[1]),
+      )
+      if (previous) previous[1] += weight
+      else consolidated.push([value, weight])
+    }
+    batch.push(...consolidated)
+  }
+  for (const entry of batch) if (entry[1] < 0) yield entry
+  for (const entry of batch) if (entry[1] > 0) yield entry
+}
+
 /**
  * Helper class that manages the state for a single topK window.
  * Encapsulates the multiplicity tracking and topK data structure,

package/src/operators/topKWithFractionalIndex.ts

@@ -1,7 +1,12 @@
 import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
 import { StreamBuilder } from '../d2.js'
 import { MultiSet } from '../multiset.js'
-import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import {
+  TopKState,
+  handleMoveIn,
+  handleMoveOut,
+  topKBatch,
+} from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
 import type { IndexedValue, TopK } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
@@ -76,11 +81,10 @@ export class TopKWithFractionalIndexOperator<
 
   run(): void {
     const result: Array<[[K, IndexedValue<T>], number]> = []
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item
-        this.processElement(key, value, multiplicity, result)
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages(),
+    )) {
+      this.processElement(key, value, multiplicity, result)
     }
 
     if (result.length > 0) {

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -106,9 +106,11 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#tree.size - 1 < this.#topKStart) {
-      // We don't have a topK yet
-      // so we don't need to do anything
+    if (
+      this.#topKStart === this.#topKEnd ||
+      this.#tree.size - 1 < this.#topKStart
+    ) {
+      // The window is empty, or there aren't enough rows to reach its offset.
       return result
     }
 
@@ -178,11 +180,8 @@ class TopKTree<V> implements TopK<V> {
       return result
     }
 
-    if (this.#comparator(value, getValue(this.#topKFirstElem)) < 0) {
-      // We deleted an element that was before the topK
-      // so the topK has shifted one position to the left
-
-      // the old first element moves out of the topK
+    if (this.#comparator(value, getValue(this.#topKFirstElem)) <= 0) {
+      // Deleting at or before the first selected value advances that boundary.
       result.moveOut = this.#topKFirstElem
       // the element that was right after the first element of the topK
       // is now the new first element of the topK