@tanstack/db-ivm 0.1.20 → 0.1.24

package/src/hashing/hash.ts

@@ -1,4 +1,5 @@
 import { MurmurHashStream, getSymbolIdentity, randomHash } from './murmur.js'
+import { isTemporal } from './temporal.js'
 import type { Hasher } from './murmur.js'
 
 /*
@@ -12,45 +13,27 @@ const NULL = randomHash()
 const UNDEFINED = randomHash()
 const KEY = randomHash()
 const FUNCTIONS = randomHash()
-const DATE_MARKER = randomHash()
-const OBJECT_MARKER = randomHash()
-const ARRAY_MARKER = randomHash()
-const MAP_MARKER = randomHash()
-const SET_MARKER = randomHash()
-const UINT8ARRAY_MARKER = randomHash()
-const TEMPORAL_MARKER = randomHash()
+// Identity is distinct even if two random hash values happen to collide.
+const DATE_MARKER = { hash: randomHash() }
+const REGEXP_MARKER = { hash: randomHash() }
+const OBJECT_MARKER = { hash: randomHash() }
+const ARRAY_MARKER = { hash: randomHash() }
+const MAP_MARKER = { hash: randomHash() }
+const SET_MARKER = { hash: randomHash() }
+const UINT8ARRAY_MARKER = { hash: randomHash() }
+const TEMPORAL_MARKER = { hash: randomHash() }
 // Bound structural recursion and value visits. Shared acyclic subtrees are
 // cached; cycles are rejected rather than given context-dependent hashes.
 const MAX_STRUCTURAL_HASH_WORK = 1_000_000
 const MAX_STRUCTURAL_HASH_DEPTH = 768
 
-const temporalTypes = new Set([
-  `Temporal.Duration`,
-  `Temporal.Instant`,
-  `Temporal.PlainDate`,
-  `Temporal.PlainDateTime`,
-  `Temporal.PlainMonthDay`,
-  `Temporal.PlainTime`,
-  `Temporal.PlainYearMonth`,
-  `Temporal.ZonedDateTime`,
-])
-
-interface TemporalLike {
-  [Symbol.toStringTag]: string
-  toString: () => string
-}
-
-function isTemporal(input: object): input is TemporalLike {
-  const tag = (input as Record<symbol, unknown>)[Symbol.toStringTag]
-  return typeof tag === `string` && temporalTypes.has(tag)
-}
-
 // Maximum byte length for Uint8Arrays to hash by content instead of reference
 // Arrays smaller than this will be hashed by content, allowing proper equality comparisons
 // for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays
 const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128
 
 const hashCache = new WeakMap<object, number>()
+const referenceValues = new WeakSet<object>()
 
 /** @internal Register a mutable handle before it enters a structural value. */
 export function registerOpaqueHash(value: object): void {
@@ -80,32 +63,33 @@ function hashObject(input: object, context: HashContext): number {
 
   let valueHash: number | undefined
   try {
-    if (input instanceof Date) {
-      valueHash = hashDate(input)
-    } else if (isBinaryValue(input)) {
-      valueHash = hashUint8Array(input)
-    } else if (isTemporal(input)) {
-      valueHash = hashTemporal(input)
-    } else {
-      let plainObjectInput = input
-      let marker = OBJECT_MARKER
-
-      if (input instanceof Array) {
-        marker = ARRAY_MARKER
-      }
-
-      if (input instanceof Map) {
-        marker = MAP_MARKER
-        plainObjectInput = [...input.entries()]
-      }
-
-      if (input instanceof Set) {
-        marker = SET_MARKER
-        plainObjectInput = [...input.entries()]
+    const [marker, header, body] = objectParts(input)
+    const hasher = new MurmurHashStream()
+    hasher.update(marker.hash)
+    // Header values of a type with a body (array length, RegExp fields)
+    // count toward the work cap; Date, binary, and Temporal headers do not.
+    for (const value of header)
+      if (body) updateHasher(hasher, value, context)
+      else hasher.update(value as string | number)
+    if (body) {
+      // Sorted string keys, then enumerable symbol keys by identity.
+      const keys: Array<string | symbol> = Object.keys(body).sort(keySort)
+      keys.push(
+        ...Object.getOwnPropertySymbols(body)
+          .filter((key) =>
+            Object.prototype.propertyIsEnumerable.call(body, key),
+          )
+          .sort(
+            (left, right) => getSymbolIdentity(left) - getSymbolIdentity(right),
+          ),
+      )
+      for (const key of keys) {
+        hasher.update(KEY)
+        hasher.update(key)
+        updateHasher(hasher, body[key as keyof typeof body], context)
       }
-
-      valueHash = hashPlainObject(plainObjectInput, marker, context)
     }
+    valueHash = hasher.digest()
   } finally {
     context.activeObjects.delete(input)
   }
@@ -114,59 +98,39 @@ function hashObject(input: object, context: HashContext): number {
   return valueHash
 }
 
-function hashDate(input: Date): number {
-  const hasher = new MurmurHashStream()
-  hasher.update(DATE_MARKER)
-  hasher.update(input.getTime())
-  return hasher.digest()
-}
-
-function hashUint8Array(input: Uint8Array): number {
-  const hasher = new MurmurHashStream()
-  hasher.update(UINT8ARRAY_MARKER)
-  // Hash the byte length first to differentiate arrays of different sizes
-  hasher.update(input.byteLength)
-  // Hash each byte in the array
-  for (let i = 0; i < input.byteLength; i++) {
-    hasher.writeByte(input[i]!)
-  }
-  return hasher.digest()
-}
-
-function hashTemporal(input: TemporalLike): number {
-  const hasher = new MurmurHashStream()
-  hasher.update(TEMPORAL_MARKER)
-  hasher.update(input[Symbol.toStringTag])
-  hasher.update(input.toString())
-  return hasher.digest()
-}
-
-function hashPlainObject(
-  input: object,
-  marker: number,
-  context: HashContext,
-): number {
-  const hasher = new MurmurHashStream()
-
-  // Mark the type of the input
-  hasher.update(marker)
-  const keys = Object.keys(input)
-  keys.sort(keySort)
-  for (const key of keys) {
-    hasher.update(KEY)
-    hasher.update(key)
-    updateHasher(hasher, input[key as keyof typeof input], context)
-  }
-  const symbolKeys = Object.getOwnPropertySymbols(input)
-    .filter((key) => Object.prototype.propertyIsEnumerable.call(input, key))
-    .sort((left, right) => getSymbolIdentity(left) - getSymbolIdentity(right))
-  for (const key of symbolKeys) {
-    hasher.update(KEY)
-    hasher.update(key)
-    updateHasher(hasher, input[key as keyof typeof input], context)
+/**
+ * The one type dispatch shared by structural hashing and equality: a type
+ * marker, primitive header values, and an optional object whose enumerable
+ * own properties are compared structurally.
+ */
+type ObjectParts = [
+  marker: Readonly<{ hash: number }>,
+  // A RegExp `lastIndex` can hold any value; the other headers are strings
+  // or numbers.
+  header: ReadonlyArray<unknown>,
+  body?: object,
+]
+
+function objectParts(input: object): ObjectParts {
+  if (input instanceof Date) {
+    const time = input.getTime()
+    // Invalid dates are equal to each other; headers compare with `===`.
+    return [DATE_MARKER, [Number.isNaN(time) ? `invalid` : time]]
   }
-
-  return hasher.digest()
+  if (isBinaryValue(input))
+    return [
+      UINT8ARRAY_MARKER,
+      [String.fromCharCode.apply(null, input as unknown as Array<number>)],
+    ]
+  if (isTemporal(input))
+    return [TEMPORAL_MARKER, [input[Symbol.toStringTag], input.toString()]]
+  if (input instanceof RegExp)
+    return [REGEXP_MARKER, [input.source, input.flags, input.lastIndex], input]
+  if (input instanceof Map) return [MAP_MARKER, [], [...input.entries()]]
+  if (input instanceof Set) return [SET_MARKER, [], [...input.values()]]
+  return input instanceof Array
+    ? [ARRAY_MARKER, [input.length], input]
+    : [OBJECT_MARKER, [], input]
 }
 
 function updateHasher(
@@ -204,56 +168,50 @@ function updateHasher(
       // Functions are assigned a globally unique ID
       // and that ID is cached in the weak map
       hasher.update(cachedReferenceHash(input))
-      return
-    default:
-      console.warn(
-        `Ignored input during hashing because it is of type ${typeof input} which is not supported`,
-      )
   }
 }
 
 function getCachedHash(input: object, context?: HashContext): number {
-  if (!context) {
-    const cached = hashCache.get(input)
-    if (cached !== undefined) return cached
-    if (isReferenceHashedObject(input)) return cachedReferenceHash(input)
-
-    // Only an uncached structural root needs graph traversal state. Commit its
-    // cache entries after success so a failed traversal cannot poison retries.
-    context = {
-      activeObjects: new Set(),
-      work: 0,
-      pendingHashes: new Map(),
-    }
-    const result = hashObject(input, context)
-    for (const [object, valueHash] of context.pendingHashes) {
-      hashCache.set(object, valueHash)
-    }
-    return result
-  }
-
-  if (context.activeObjects.has(input)) {
-    throw new TypeError(`Cannot hash cyclic structural values`)
-  }
+  // Active objects are neither cached nor pending, so the cycle check below
+  // still sees every back-reference.
+  const cached = hashCache.get(input) ?? context?.pendingHashes.get(input)
+  if (cached !== undefined) return cached
 
   // Opaque leaves cannot contain structural back-references. Resolve them
   // before entering structural recursion, even when they have user properties.
   if (isReferenceHashedObject(input)) return cachedReferenceHash(input)
 
-  const valueHash = hashCache.get(input) ?? context.pendingHashes.get(input)
-  if (valueHash !== undefined) return valueHash
+  if (context) {
+    if (context.activeObjects.has(input)) {
+      throw new TypeError(`Cannot hash cyclic structural values`)
+    }
+    return hashObject(input, context)
+  }
 
-  return hashObject(input, context)
+  // Only an uncached structural root needs graph traversal state. Commit its
+  // cache entries after success so a failed traversal cannot poison retries.
+  const root: HashContext = {
+    activeObjects: new Set(),
+    work: 0,
+    pendingHashes: new Map(),
+  }
+  const result = hashObject(input, root)
+  for (const [object, valueHash] of root.pendingHashes) {
+    hashCache.set(object, valueHash)
+  }
+  return result
 }
 
 function isReferenceHashedObject(input: object): boolean {
   return (
-    input instanceof File ||
+    referenceValues.has(input) ||
+    (typeof File !== `undefined` && input instanceof File) ||
     (isBinaryValue(input) &&
       input.byteLength > UINT8ARRAY_CONTENT_HASH_THRESHOLD)
   )
 }
 
+// Buffer can come from another realm than Uint8Array (for example under jsdom).
 function isBinaryValue(input: object): input is Uint8Array {
   return (
     (typeof Buffer !== `undefined` && input instanceof Buffer) ||
@@ -261,6 +219,59 @@ function isBinaryValue(input: object): input is Uint8Array {
   )
 }
 
+/** @internal Compare immutable structural values without computing a digest.
+ * Pair memoization also permits cyclic values that structural hashing rejects.
+ * Reference-valued leaves remain opaque, including registered mutable handles.
+ */
+export function equalHashValues(left: unknown, right: unknown): boolean {
+  const compared = new Map<object, Set<object>>()
+  const keys = (value: object) =>
+    Reflect.ownKeys(value).filter((key) =>
+      Object.prototype.propertyIsEnumerable.call(value, key),
+    )
+  function equal(a: unknown, b: unknown): boolean {
+    if (a === b || (Number.isNaN(a) && Number.isNaN(b))) return true
+    if (
+      a === null ||
+      b === null ||
+      typeof a !== `object` ||
+      typeof b !== `object` ||
+      isReferenceHashedObject(a) ||
+      isReferenceHashedObject(b)
+    )
+      return false
+    // Arrays from another realm take the object marker but keep this check.
+    if (Array.isArray(a) && Array.isArray(b) && a.length !== b.length)
+      return false
+
+    // Revisited pairs close cycles and avoid expanding shared subtrees. Check
+    // before objectParts copies Map or Set entries; record only pairs with a
+    // body, after their markers and headers match.
+    const peers = compared.get(a)
+    if (peers?.has(b)) return true
+    const [aMarker, aHeader, aBody] = objectParts(a)
+    const [bMarker, bHeader, bBody] = objectParts(b)
+    if (
+      aMarker !== bMarker ||
+      !aHeader.every((value, index) => value === bHeader[index])
+    )
+      return false
+    if (!aBody || !bBody) return true
+    if (peers) peers.add(b)
+    else compared.set(a, new Set([b]))
+    const aKeys = keys(aBody)
+    return (
+      aKeys.length === keys(bBody).length &&
+      aKeys.every(
+        (key) =>
+          Object.prototype.propertyIsEnumerable.call(bBody, key) &&
+          equal(aBody[key as keyof object], bBody[key as keyof object]),
+      )
+    )
+  }
+  return equal(left, right)
+}
+
 let nextRefId = 1
 function cachedReferenceHash(fn: object): number {
   let valueHash = hashCache.get(fn)
@@ -268,6 +279,7 @@ function cachedReferenceHash(fn: object): number {
     valueHash = nextRefId ^ FUNCTIONS
     nextRefId++
     hashCache.set(fn, valueHash)
+    referenceValues.add(fn)
   }
   return valueHash
 }

package/src/hashing/temporal.ts

@@ -0,0 +1,25 @@
+const temporalTypes = new Set([
+  `Temporal.Duration`,
+  `Temporal.Instant`,
+  `Temporal.PlainDate`,
+  `Temporal.PlainDateTime`,
+  `Temporal.PlainMonthDay`,
+  `Temporal.PlainTime`,
+  `Temporal.PlainYearMonth`,
+  `Temporal.ZonedDateTime`,
+])
+
+export interface TemporalLike {
+  [Symbol.toStringTag]: string
+  toString: () => string
+}
+
+export function isTemporal(input: object): input is TemporalLike {
+  const tag = (input as Record<symbol, unknown>)[Symbol.toStringTag]
+  // An own tag on a plain object does not make it a Temporal value.
+  return (
+    typeof tag === `string` &&
+    temporalTypes.has(tag) &&
+    Object.getPrototypeOf(input)?.[Symbol.toStringTag] === tag
+  )
+}

package/src/multiset.ts

@@ -1,8 +1,4 @@
-import {
-  DefaultMap,
-  chunkedArrayPush,
-  globalObjectIdGenerator,
-} from './utils.js'
+import { chunkedArrayPush, getStringId } from './utils.js'
 import { hash } from './hashing/index.js'
 
 export type MultiSetArray<T> = Array<[T, number]>
@@ -69,139 +65,59 @@ export class MultiSet<T> {
    * Produce as output a collection that is logically equivalent to the input
    * but which combines identical instances of the same record into one
    * (record, multiplicity) pair.
-   */
-  consolidate(): MultiSet<T> {
-    // Check if this looks like a keyed multiset (first item is a tuple of length 2)
-    if (this.#inner.length > 0) {
-      const firstItem = this.#inner[0]?.[0]
-      if (Array.isArray(firstItem) && firstItem.length === 2) {
-        return this.#consolidateKeyed()
-      }
-    }
-
-    // Fall back to original method for unkeyed data
-    return this.#consolidateUnkeyed()
-  }
-
-  /**
-   * Private method for consolidating keyed multisets where keys are strings/numbers
-   * and values are compared by reference equality.
    *
-   * This method provides significant performance improvements over the hash-based approach
-   * by using WeakMap for object reference tracking and avoiding expensive serialization.
+   * Record identity depends on the shape of the whole multiset:
+   *
+   * - Keyed: every record is a `[key, value]` pair with a string or number
+   *   key. Keys and primitive values compare by value, and object values by
+   *   reference. A value that is an array of length 2 is a join tuple, and
+   *   its two elements compare by the same rule. The keyed identity text
+   *   has known collisions: https://github.com/TanStack/db/issues/1948
+   * - Unkeyed, one primitive type: every record is a string, or every record
+   *   is a number. Records compare as `Map` keys do, and a `-0` record is
+   *   returned as `0`.
+   * - Otherwise, records compare by structure (`hash`).
    *
-   * Special handling for join operations: When values are tuples of length 2 (common in joins),
-   * we unpack them and compare each element individually to maintain proper equality semantics.
+   * Each identity keeps its first record in input order with the summed
+   * multiplicity. Identities whose sum is zero are dropped. The input is not
+   * changed.
    */
-  #consolidateKeyed(): MultiSet<T> {
-    const consolidated = new Map<string, number>()
-    const values = new Map<string, T>()
-
-    // Use global object ID generator for consistent reference equality
-
-    /**
-     * Special handler for tuples (arrays of length 2) commonly produced by join operations.
-     * Unpacks the tuple and generates an ID based on both elements to ensure proper
-     * consolidation of join results like ['A', null] and [null, 'X'].
-     */
-    const getTupleId = (tuple: Array<any>): string => {
-      if (tuple.length !== 2) {
-        throw new Error(`Expected tuple of length 2`)
-      }
-      const [first, second] = tuple
-      return `${globalObjectIdGenerator.getStringId(first)}|${globalObjectIdGenerator.getStringId(second)}`
-    }
-
-    // Process each item in the multiset
-    for (const [data, multiplicity] of this.#inner) {
-      // Verify this is still a keyed item (should be [key, value] pair)
-      if (!Array.isArray(data) || data.length !== 2) {
-        // Found non-keyed item, fall back to unkeyed consolidation
-        return this.#consolidateUnkeyed()
-      }
-
-      const [key, value] = data
-
-      // Verify key is string or number as expected for keyed multisets
-      if (typeof key !== `string` && typeof key !== `number`) {
-        // Found non-string/number key, fall back to unkeyed consolidation
-        return this.#consolidateUnkeyed()
-      }
-
-      // Generate value ID with special handling for join tuples
-      let valueId: string
-      if (Array.isArray(value) && value.length === 2) {
-        // Special case: value is a tuple from join operations
-        valueId = getTupleId(value)
-      } else {
-        // Regular case: use reference/value equality
-        valueId = globalObjectIdGenerator.getStringId(value)
-      }
-
-      // Create composite key and consolidate
-      const compositeKey = key + `|` + valueId
-      consolidated.set(
-        compositeKey,
-        (consolidated.get(compositeKey) || 0) + multiplicity,
+  consolidate(): MultiSet<T> {
+    const inner = this.#inner
+    // Keyed multisets hold [string | number key, value] pairs. Their values are
+    // compared by reference, which avoids hashing. Join tuples (values of
+    // length 2) are unpacked so ['A', null] and [null, 'X'] stay distinct.
+    if (
+      inner.every(
+        ([data]) =>
+          Array.isArray(data) &&
+          data.length === 2 &&
+          (typeof data[0] === `string` || typeof data[0] === `number`),
       )
-
-      // Store the original data for the first occurrence
-      if (!values.has(compositeKey)) {
-        values.set(compositeKey, data as T)
-      }
+    ) {
+      return consolidateBy(inner, (data) => {
+        const [key, value] = data as [string | number, unknown]
+        const valueId =
+          Array.isArray(value) && value.length === 2
+            ? `${getStringId(value[0])}|${getStringId(value[1])}`
+            : getStringId(value)
+        return key + `|` + valueId
+      })
     }
 
-    // Build result array, filtering out zero multiplicities
-    const result: MultiSetArray<T> = []
-    for (const [compositeKey, multiplicity] of consolidated) {
-      if (multiplicity !== 0) {
-        result.push([values.get(compositeKey)!, multiplicity])
-      }
-    }
-
-    return new MultiSet(result)
-  }
-
-  /**
-   * Private method for consolidating unkeyed multisets using the original approach.
-   */
-  #consolidateUnkeyed(): MultiSet<T> {
-    const consolidated = new DefaultMap<string | number, number>(() => 0)
-    const values = new Map<string, any>()
-
-    let hasString = false
-    let hasNumber = false
-    let hasOther = false
-    for (const [data, _] of this.#inner) {
-      if (typeof data === `string`) {
-        hasString = true
-      } else if (typeof data === `number`) {
-        hasNumber = true
-      } else {
-        hasOther = true
-        break
-      }
-    }
-
-    const requireJson = hasOther || (hasString && hasNumber)
-
-    for (const [data, multiplicity] of this.#inner) {
-      const key = requireJson ? hash(data) : (data as string | number)
-      if (requireJson && !values.has(key as string)) {
-        values.set(key as string, data)
-      }
-      consolidated.update(key, (count) => count + multiplicity)
-    }
-
-    const result: MultiSetArray<T> = []
-    for (const [key, multiplicity] of consolidated.entries()) {
-      if (multiplicity !== 0) {
-        const parsedKey = requireJson ? values.get(key as string) : key
-        result.push([parsedKey as T, multiplicity])
-      }
-    }
-
-    return new MultiSet(result)
+    // Unkeyed data of a single primitive type is its own identity, and the
+    // result holds the Map key, so -0 becomes 0. Anything else is compared
+    // structurally by hash.
+    let primitiveType: string | undefined
+    const requireHash = inner.some(([data]) => {
+      const type = typeof data
+      if (type !== `string` && type !== `number`) return true
+      primitiveType ??= type
+      return type !== primitiveType
+    })
+    return requireHash
+      ? consolidateBy(inner, hash)
+      : consolidateBy(inner, (data) => data, true)
   }
 
   extend(other: MultiSet<T> | MultiSetArray<T>): void {
@@ -219,3 +135,28 @@ export class MultiSet<T> {
     return this.#inner
   }
 }
+
+/**
+ * Sums multiplicities of records with the same identity, keeping the first
+ * record seen for each identity (or its Map key, which writes -0 as 0, when
+ * `keepIdentity` is set) and dropping records whose sum is zero.
+ */
+function consolidateBy<T>(
+  inner: MultiSetArray<T>,
+  identityOf: (data: T) => unknown,
+  keepIdentity = false,
+): MultiSet<T> {
+  const consolidated = new Map<unknown, [T, number]>()
+  for (const [data, multiplicity] of inner) {
+    const identity = identityOf(data)
+    const entry = consolidated.get(identity)
+    if (entry) entry[1] += multiplicity
+    else consolidated.set(identity, [data, multiplicity])
+  }
+  const result: MultiSetArray<T> = []
+  for (const [identity, [data, multiplicity]] of consolidated) {
+    if (multiplicity !== 0)
+      result.push([keepIdentity ? (identity as T) : data, multiplicity])
+  }
+  return new MultiSet(result)
+}

package/src/operators/distinct.ts

@@ -85,10 +85,12 @@ export class DistinctOperator<
 export function distinct<T extends KeyValue<any, any>>(
   by: (value: T) => any = (value: T) => value,
 ) {
-  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
-    const output = new StreamBuilder<T>(
+  return (
+    stream: IStreamBuilder<T>,
+  ): IStreamBuilder<KeyValue<number, GetValue<T>>> => {
+    const output = new StreamBuilder<KeyValue<number, GetValue<T>>>(
       stream.graph,
-      new DifferenceStreamWriter<T>(),
+      new DifferenceStreamWriter<KeyValue<number, GetValue<T>>>(),
     )
     const operator = new DistinctOperator<T>(
       stream.graph.getNextOperatorId(),

package/src/operators/groupBy.ts

@@ -237,7 +237,10 @@ export function min<T, V extends CanMinMax>(
     reduce: (values) => {
       let minValue: V | undefined
       for (const [value, _multiplicity] of values) {
-        if (!minValue || (value && value < minValue)) {
+        if (
+          value !== undefined &&
+          (minValue === undefined || value < minValue)
+        ) {
           minValue = value
         }
       }
@@ -267,7 +270,10 @@ export function max<T, V extends CanMinMax>(
     reduce: (values) => {
       let maxValue: V | undefined
       for (const [value, _multiplicity] of values) {
-        if (!maxValue || (value && value > maxValue)) {
+        if (
+          value !== undefined &&
+          (maxValue === undefined || value > maxValue)
+        ) {
           maxValue = value
         }
       }