@tanstack/db-ivm 0.1.12 → 0.1.14

package/src/hashing/hash.ts

@@ -1,5 +1,5 @@
-import { MurmurHashStream, randomHash } from "./murmur.js"
-import type { Hasher } from "./murmur.js"
+import { MurmurHashStream, randomHash } from './murmur.js'
+import type { Hasher } from './murmur.js'
 
 /*
  * Implementation of structural hashing based on the Composites polyfill implementation:
@@ -17,6 +17,12 @@ const OBJECT_MARKER = randomHash()
 const ARRAY_MARKER = randomHash()
 const MAP_MARKER = randomHash()
 const SET_MARKER = randomHash()
+const UINT8ARRAY_MARKER = randomHash()
+
+// Maximum byte length for Uint8Arrays to hash by content instead of reference
+// Arrays smaller than this will be hashed by content, allowing proper equality comparisons
+// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays
+const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128
 
 const hashCache = new WeakMap<object, number>()
 
@@ -35,6 +41,24 @@ function hashObject(input: object): number {
   let valueHash: number | undefined
   if (input instanceof Date) {
     valueHash = hashDate(input)
+  } else if (
+    // Check if input is a Uint8Array or Buffer
+    (typeof Buffer !== `undefined` && input instanceof Buffer) ||
+    input instanceof Uint8Array
+  ) {
+    // For small Uint8Arrays/Buffers (e.g., ULIDs, UUIDs), hash by content
+    // to enable proper equality comparisons. For large arrays, hash by reference
+    // to avoid performance costs.
+    if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {
+      valueHash = hashUint8Array(input)
+    } else {
+      // Deeply hashing large arrays would be too costly
+      // so we track them by reference and cache them in a weak map
+      return cachedReferenceHash(input)
+    }
+  } else if (input instanceof File) {
+    // Files are always hashed by reference due to their potentially large size
+    return cachedReferenceHash(input)
   } else {
     let plainObjectInput = input
     let marker = OBJECT_MARKER
@@ -53,17 +77,6 @@ function hashObject(input: object): number {
       plainObjectInput = [...input.entries()]
     }
 
-    if (
-      (typeof Buffer !== `undefined` && input instanceof Buffer) ||
-      input instanceof Uint8Array ||
-      input instanceof File
-    ) {
-      // Deeply hashing these objects would be too costly
-      // but we also don't want to ignore them
-      // so we track them by reference and cache them in a weak map
-      return cachedReferenceHash(input)
-    }
-
     valueHash = hashPlainObject(plainObjectInput, marker)
   }
 
@@ -78,6 +91,18 @@ function hashDate(input: Date): number {
   return hasher.digest()
 }
 
+function hashUint8Array(input: Uint8Array): number {
+  const hasher = new MurmurHashStream()
+  hasher.update(UINT8ARRAY_MARKER)
+  // Hash the byte length first to differentiate arrays of different sizes
+  hasher.update(input.byteLength)
+  // Hash each byte in the array
+  for (let i = 0; i < input.byteLength; i++) {
+    hasher.writeByte(input[i]!)
+  }
+  return hasher.digest()
+}
+
 function hashPlainObject(input: object, marker: number): number {
   const hasher = new MurmurHashStream()
 
@@ -125,7 +150,7 @@ function updateHasher(hasher: Hasher, input: unknown): void {
       return
     default:
       console.warn(
-        `Ignored input during hashing because it is of type ${typeof input} which is not supported`
+        `Ignored input during hashing because it is of type ${typeof input} which is not supported`,
       )
   }
 }

package/src/hashing/index.ts

@@ -1,2 +1,2 @@
-export { hash } from "./hash.js"
-export type { Hash, Hasher } from "./murmur.js"
+export { hash } from './hash.js'
+export type { Hash, Hasher } from './murmur.js'

package/src/hashing/murmur.ts

@@ -51,7 +51,7 @@ export class MurmurHashStream implements Hasher {
     this.hash = Math.imul(this.hash, 5) + 0xe6546b64
   }
 
-  private _writeByte(byte: number): void {
+  writeByte(byte: number): void {
     this.carry |= (byte & 0xff) << (8 * this.carryBytes)
     this.carryBytes++
     this.length++
@@ -74,8 +74,8 @@ export class MurmurHashStream implements Hasher {
 
         for (let i = 0; i < description.length; i++) {
           const code = description.charCodeAt(i)
-          this._writeByte(code & 0xff)
-          this._writeByte((code >>> 8) & 0xff)
+          this.writeByte(code & 0xff)
+          this.writeByte((code >>> 8) & 0xff)
         }
         return
       }
@@ -83,20 +83,20 @@ export class MurmurHashStream implements Hasher {
         this.update(STRING_MARKER)
         for (let i = 0; i < chunk.length; i++) {
           const code = chunk.charCodeAt(i)
-          this._writeByte(code & 0xff)
-          this._writeByte((code >>> 8) & 0xff)
+          this.writeByte(code & 0xff)
+          this.writeByte((code >>> 8) & 0xff)
         }
         return
       case `number`:
         dv.setFloat64(0, chunk, true) // fixed little-endian
-        this._writeByte(u8[0]!)
-        this._writeByte(u8[1]!)
-        this._writeByte(u8[2]!)
-        this._writeByte(u8[3]!)
-        this._writeByte(u8[4]!)
-        this._writeByte(u8[5]!)
-        this._writeByte(u8[6]!)
-        this._writeByte(u8[7]!)
+        this.writeByte(u8[0]!)
+        this.writeByte(u8[1]!)
+        this.writeByte(u8[2]!)
+        this.writeByte(u8[3]!)
+        this.writeByte(u8[4]!)
+        this.writeByte(u8[5]!)
+        this.writeByte(u8[6]!)
+        this.writeByte(u8[7]!)
         return
       case `bigint`: {
         let value = chunk
@@ -107,10 +107,10 @@ export class MurmurHashStream implements Hasher {
           this.update(BIG_INT_MARKER)
         }
         while (value > 0n) {
-          this._writeByte(Number(value & 0xffn))
+          this.writeByte(Number(value & 0xffn))
           value >>= 8n
         }
-        if (chunk === 0n) this._writeByte(0)
+        if (chunk === 0n) this.writeByte(0)
         return
       }
       default:

package/src/index.ts

@@ -1,4 +1,5 @@
-export * from "./d2.js"
-export * from "./multiset.js"
-export * from "./operators/index.js"
-export * from "./types.js"
+export * from './d2.js'
+export * from './multiset.js'
+export * from './operators/index.js'
+export * from './types.js'
+export { compareKeys } from './utils.js'

package/src/indexes.ts

@@ -33,9 +33,9 @@
  * Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'
  */
 
-import { MultiSet } from "./multiset.js"
-import { hash } from "./hashing/index.js"
-import type { Hash } from "./hashing/index.js"
+import { MultiSet } from './multiset.js'
+import { hash } from './hashing/index.js'
+import type { Hash } from './hashing/index.js'
 
 // We use a symbol to represent the absence of a prefix, unprefixed values a stored
 // against this key.
@@ -183,7 +183,7 @@ export class Index<TKey, TValue, TPrefix = any> {
     return `Index(${JSON.stringify(
       [...this.entries()],
       undefined,
-      indent ? 2 : undefined
+      indent ? 2 : undefined,
     )})`
   }
 
@@ -325,7 +325,7 @@ export class Index<TKey, TValue, TPrefix = any> {
         key,
         mapOrSingleValue,
         value,
-        multiplicity
+        multiplicity,
       )
       return
     }
@@ -362,7 +362,7 @@ export class Index<TKey, TValue, TPrefix = any> {
     key: TKey,
     currentSingleValue: SingleValue<TValue>,
     newValue: TValue,
-    multiplicity: number
+    multiplicity: number,
   ) {
     const [currentValue, currentMultiplicity] = currentSingleValue
 
@@ -438,7 +438,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns A multiset of the joined values.
    */
   join<TValue2>(
-    other: Index<TKey, TValue2>
+    other: Index<TKey, TValue2>,
   ): MultiSet<[TKey, [TValue, TValue2]]> {
     const result: Array<[[TKey, [TValue, TValue2]], number]> = []
     // We want to iterate over the smaller of the two indexes to reduce the
@@ -499,7 +499,7 @@ function getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {
  * @returns True if the value is a single value, false otherwise.
  */
 function isSingleValue<TValue>(
-  value: SingleValue<TValue> | unknown
+  value: SingleValue<TValue> | unknown,
 ): value is SingleValue<TValue> {
   return Array.isArray(value)
 }

package/src/multiset.ts

@@ -2,8 +2,8 @@ import {
   DefaultMap,
   chunkedArrayPush,
   globalObjectIdGenerator,
-} from "./utils.js"
-import { hash } from "./hashing/index.js"
+} from './utils.js'
+import { hash } from './hashing/index.js'
 
 export type MultiSetArray<T> = Array<[T, number]>
 export type KeyedData<T> = [key: string, value: T]
@@ -35,7 +35,7 @@ export class MultiSet<T> {
    */
   map<U>(f: (data: T) => U): MultiSet<U> {
     return new MultiSet(
-      this.#inner.map(([data, multiplicity]) => [f(data), multiplicity])
+      this.#inner.map(([data, multiplicity]) => [f(data), multiplicity]),
     )
   }
 
@@ -51,7 +51,7 @@ export class MultiSet<T> {
    */
   negate(): MultiSet<T> {
     return new MultiSet(
-      this.#inner.map(([data, multiplicity]) => [data, -multiplicity])
+      this.#inner.map(([data, multiplicity]) => [data, -multiplicity]),
     )
   }
 
@@ -142,7 +142,7 @@ export class MultiSet<T> {
       const compositeKey = key + `|` + valueId
       consolidated.set(
         compositeKey,
-        (consolidated.get(compositeKey) || 0) + multiplicity
+        (consolidated.get(compositeKey) || 0) + multiplicity,
       )
 
       // Store the original data for the first occurrence

package/src/operators/concat.ts

@@ -1,6 +1,6 @@
-import { BinaryOperator, DifferenceStreamWriter } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
+import { BinaryOperator, DifferenceStreamWriter } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
 
 /**
  * Operator that concatenates two input streams
@@ -22,7 +22,7 @@ export class ConcatOperator<T, T2> extends BinaryOperator<T | T2> {
  * @param other - The other stream to concatenate
  */
 export function concat<T, T2>(
-  other: IStreamBuilder<T2>
+  other: IStreamBuilder<T2>,
 ): PipedOperator<T, T | T2> {
   return (stream: IStreamBuilder<T | T2>): IStreamBuilder<T | T2> => {
     if (stream.graph !== other.graph) {
@@ -30,13 +30,13 @@ export function concat<T, T2>(
     }
     const output = new StreamBuilder<T | T2>(
       stream.graph,
-      new DifferenceStreamWriter<T | T2>()
+      new DifferenceStreamWriter<T | T2>(),
     )
     const operator = new ConcatOperator<T, T2>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       other.connectReader(),
-      output.writer
+      output.writer,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/consolidate.ts

@@ -1,7 +1,7 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { MultiSet } from "../multiset.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { MultiSet } from '../multiset.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
 
 /**
  * Operator that consolidates collections
@@ -36,12 +36,12 @@ export function consolidate<T>(): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new ConsolidateOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
-      output.writer
+      output.writer,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/count.ts

@@ -1,8 +1,8 @@
-import { DifferenceStreamWriter } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { ReduceOperator } from "./reduce.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder, KeyValue } from "../types.js"
+import { DifferenceStreamWriter } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { ReduceOperator } from './reduce.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, KeyValue } from '../types.js'
 
 /**
  * Operator that counts elements by key (version-free)
@@ -11,7 +11,7 @@ export class CountOperator<K, V> extends ReduceOperator<K, V, number> {
   constructor(
     id: number,
     inputA: DifferenceStreamReader<[K, V]>,
-    output: DifferenceStreamWriter<[K, number]>
+    output: DifferenceStreamWriter<[K, number]>,
   ) {
     const countInner = (vals: Array<[V, number]>): Array<[number, number]> => {
       let totalCount = 0
@@ -34,16 +34,16 @@ export function count<
   T,
 >() {
   return (
-    stream: IStreamBuilder<T>
+    stream: IStreamBuilder<T>,
   ): IStreamBuilder<KeyValue<KType, number>> => {
     const output = new StreamBuilder<KeyValue<KType, number>>(
       stream.graph,
-      new DifferenceStreamWriter<KeyValue<KType, number>>()
+      new DifferenceStreamWriter<KeyValue<KType, number>>(),
     )
     const operator = new CountOperator<KType, VType>(
       stream.graph.getNextOperatorId(),
       stream.connectReader() as DifferenceStreamReader<KeyValue<KType, VType>>,
-      output.writer
+      output.writer,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/debug.ts

@@ -1,7 +1,7 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
-import type { DifferenceStreamReader } from "../graph.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+import type { DifferenceStreamReader } from '../graph.js'
 
 /**
  * Operator that logs debug information about the stream
@@ -15,7 +15,7 @@ export class DebugOperator<T> extends UnaryOperator<T> {
     inputA: DifferenceStreamReader<T>,
     output: DifferenceStreamWriter<T>,
     name: string,
-    indent: boolean = false
+    indent: boolean = false,
   ) {
     super(id, inputA, output)
     this.#name = name
@@ -37,19 +37,19 @@ export class DebugOperator<T> extends UnaryOperator<T> {
  */
 export function debug<T>(
   name: string,
-  indent: boolean = false
+  indent: boolean = false,
 ): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new DebugOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
       name,
-      indent
+      indent,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/distinct.ts

@@ -1,10 +1,10 @@
-import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import { hash } from "../hashing/index.js"
-import { MultiSet } from "../multiset.js"
-import type { Hash } from "../hashing/index.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder, KeyValue } from "../types.js"
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { hash } from '../hashing/index.js'
+import { MultiSet } from '../multiset.js'
+import type { Hash } from '../hashing/index.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, KeyValue } from '../types.js'
 
 type Multiplicity = number
 
@@ -23,7 +23,7 @@ export class DistinctOperator<
     id: number,
     input: DifferenceStreamReader<T>,
     output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,
-    by: (value: T) => any = (value: T) => value
+    by: (value: T) => any = (value: T) => value,
   ) {
     super(id, input, output)
     this.#by = by
@@ -83,18 +83,18 @@ export class DistinctOperator<
  * Removes duplicate values
  */
 export function distinct<T extends KeyValue<any, any>>(
-  by: (value: T) => any = (value: T) => value
+  by: (value: T) => any = (value: T) => value,
 ) {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new DistinctOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
-      by
+      by,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/filter.ts

@@ -1,8 +1,8 @@
-import { DifferenceStreamWriter, LinearUnaryOperator } from "../graph.js"
-import { StreamBuilder } from "../d2.js"
-import type { IStreamBuilder, PipedOperator } from "../types.js"
-import type { DifferenceStreamReader } from "../graph.js"
-import type { MultiSet } from "../multiset.js"
+import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { MultiSet } from '../multiset.js'
 
 /**
  * Operator that filters elements from the input stream
@@ -14,7 +14,7 @@ export class FilterOperator<T> extends LinearUnaryOperator<T, T> {
     id: number,
     inputA: DifferenceStreamReader<T>,
     output: DifferenceStreamWriter<T>,
-    f: (data: T) => boolean
+    f: (data: T) => boolean,
   ) {
     super(id, inputA, output)
     this.#f = f
@@ -33,13 +33,13 @@ export function filter<T>(f: (data: T) => boolean): PipedOperator<T, T> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,
-      new DifferenceStreamWriter<T>()
+      new DifferenceStreamWriter<T>(),
     )
     const operator = new FilterOperator<T>(
       stream.graph.getNextOperatorId(),
       stream.connectReader(),
       output.writer,
-      f
+      f,
     )
     stream.graph.addOperator(operator)
     return output

package/src/operators/filterBy.ts

@@ -1,7 +1,7 @@
-import { map } from "./map.js"
-import { innerJoin } from "./join.js"
-import { consolidate } from "./consolidate.js"
-import type { IStreamBuilder, KeyValue, PipedOperator } from "../types.js"
+import { map } from './map.js'
+import { innerJoin } from './join.js'
+import { consolidate } from './consolidate.js'
+import type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'
 
 /**
  * Filters the elements of a keyed stream, by keys of another stream.
@@ -15,16 +15,16 @@ export function filterBy<
   V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
   T,
 >(
-  other: IStreamBuilder<KeyValue<K, unknown>>
+  other: IStreamBuilder<KeyValue<K, unknown>>,
 ): PipedOperator<T, KeyValue<K, V1>> {
   return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<K, V1>> => {
     const otherKeys = other.pipe(
-      map(([key, _]) => [key, null] as KeyValue<K, null>)
+      map(([key, _]) => [key, null] as KeyValue<K, null>),
     )
     return stream.pipe(
       innerJoin(otherKeys),
       map(([key, [value, _]]) => [key, value] as KeyValue<K, V1>),
-      consolidate()
+      consolidate(),
     )
   }
 }