@tanstack/db-ivm 0.1.25 → 0.2.0

package/src/hashing/murmur.ts

@@ -63,11 +63,8 @@ export interface Hasher {
   digest: () => number
 }
 
-// Allocate these once, outside the hot path
-const buf = new ArrayBuffer(8)
-// dv and u8 are 2 different views on the same buffer `buf`
-const dv = new DataView(buf)
-const u8 = new Uint8Array(buf)
+// Allocate this once, outside the hot path
+const dv = new DataView(new ArrayBuffer(8))
 
 /**
  * This implementation of Murmur hash uses a random initial seed and random markers.
@@ -106,6 +103,20 @@ export class MurmurHashStream implements Hasher {
     }
   }
 
+  /** Write four bytes, low byte first, as four `writeByte` calls would. */
+  private writeWord(word: number): void {
+    this.length += 4
+    if (this.carryBytes === 0) {
+      this._mix(word >>> 0)
+      return
+    }
+    // The carry's bytes come first. The word's low bytes complete that block,
+    // and its high bytes become the new carry, which keeps its byte count.
+    const shift = 8 * this.carryBytes
+    this._mix((this.carry | (word << shift)) >>> 0)
+    this.carry = word >>> (32 - shift)
+  }
+
   update(chunk: symbol | string | number | bigint): void {
     switch (typeof chunk) {
       case `symbol`: {
@@ -115,22 +126,25 @@ export class MurmurHashStream implements Hasher {
       }
       case `string`:
         this.update(STRING_MARKER)
-        for (let i = 0; i < chunk.length; i++) {
-          const code = chunk.charCodeAt(i)
-          this.writeByte(code & 0xff)
-          this.writeByte((code >>> 8) & 0xff)
+        {
+          // Each code unit is two bytes, so a pair of them is one word.
+          const pairs = chunk.length - (chunk.length % 2)
+          for (let i = 0; i < pairs; i += 2) {
+            this.writeWord(
+              chunk.charCodeAt(i) | (chunk.charCodeAt(i + 1) << 16),
+            )
+          }
+          if (pairs < chunk.length) {
+            const code = chunk.charCodeAt(pairs)
+            this.writeByte(code & 0xff)
+            this.writeByte((code >>> 8) & 0xff)
+          }
         }
         return
       case `number`:
         dv.setFloat64(0, chunk, true) // fixed little-endian
-        this.writeByte(u8[0]!)
-        this.writeByte(u8[1]!)
-        this.writeByte(u8[2]!)
-        this.writeByte(u8[3]!)
-        this.writeByte(u8[4]!)
-        this.writeByte(u8[5]!)
-        this.writeByte(u8[6]!)
-        this.writeByte(u8[7]!)
+        this.writeWord(dv.getUint32(0, true))
+        this.writeWord(dv.getUint32(4, true))
         return
       case `bigint`: {
         let value = chunk

package/src/indexes.ts

@@ -58,8 +58,13 @@ class PrefixMap<TValue, TPrefix> extends Map<
 > {
   /**
    * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.
+   * With `prefixIdentity`, values that share a prefix are the same value.
    */
-  addValue(value: TValue, multiplicity: number): boolean {
+  addValue(
+    value: TValue,
+    multiplicity: number,
+    prefixIdentity = false,
+  ): boolean {
     if (multiplicity === 0) return this.size === 0
 
     const prefix = getPrefix<TValue, TPrefix>(value)
@@ -73,7 +78,11 @@ class PrefixMap<TValue, TPrefix> extends Map<
         throw new Error(`Mismatching prefixes, this should never happen`)
       }
 
-      if (currentValue === value || hash(currentValue) === hash(value)) {
+      if (
+        currentValue === value ||
+        (prefixIdentity && prefix !== NO_PREFIX) ||
+        hash(currentValue) === hash(value)
+      ) {
         // Same value, update multiplicity
         const newMultiplicity = currentMultiplicity + multiplicity
         if (newMultiplicity === 0) {
@@ -150,10 +159,21 @@ export class Index<TKey, TValue, TPrefix = any> {
    *   hash to identify identical values, storing them in a third level value map.
    */
   #inner: IndexMap<TKey, TValue, TPrefix>
-  #consolidatedMultiplicity: Map<TKey, number> = new Map() // sum of multiplicities per key
+  // Sum of multiplicities per key, for presence. Only joins read presence; a
+  // reduce index is built without it.
+  #consolidatedMultiplicity: Map<TKey, number> | undefined
+  /**
+   * When true, the producer guarantees that a value's string or number
+   * prefix determines the value: two values under one key with an equal
+   * prefix are equal, so they merge without a structural hash.
+   */
+  #prefixIdentity: boolean
 
-  constructor() {
+  constructor(options?: { prefixIdentity?: boolean; trackPresence?: boolean }) {
     this.#inner = new Map()
+    if (options?.trackPresence ?? true)
+      this.#consolidatedMultiplicity = new Map()
+    this.#prefixIdentity = options?.prefixIdentity ?? false
   }
 
   /**
@@ -194,6 +214,12 @@ export class Index<TKey, TValue, TPrefix = any> {
     return this.#inner.size
   }
 
+  #presence(): Map<TKey, number> {
+    if (this.#consolidatedMultiplicity === undefined)
+      throw new Error(`This Index does not track presence`)
+    return this.#consolidatedMultiplicity
+  }
+
   /**
    * This method checks if the index has a given key.
    * @param key - The key to check.
@@ -209,7 +235,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns True if the key has non-zero consolidated multiplicity, false otherwise.
    */
   hasPresence(key: TKey): boolean {
-    return (this.#consolidatedMultiplicity.get(key) || 0) !== 0
+    return (this.#presence().get(key) || 0) !== 0
   }
 
   /**
@@ -218,7 +244,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns The consolidated multiplicity for the key.
    */
   getConsolidatedMultiplicity(key: TKey): number {
-    return this.#consolidatedMultiplicity.get(key) || 0
+    return this.#presence().get(key) || 0
   }
 
   /**
@@ -226,7 +252,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns An iterator of keys with non-zero consolidated multiplicity.
    */
   getPresenceKeys(): Iterable<TKey> {
-    return this.#consolidatedMultiplicity.keys()
+    return this.#presence().keys()
   }
 
   /**
@@ -235,7 +261,22 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns An array of value tuples [value, multiplicity].
    */
   get(key: TKey): Array<[TValue, number]> {
-    return [...this.getIterator(key)]
+    // Built directly, in getIterator's order: join and reduce call this for
+    // every changed key.
+    const mapOrSingleValue = this.#inner.get(key)
+    if (mapOrSingleValue === undefined) return []
+    if (isSingleValue(mapOrSingleValue)) return [mapOrSingleValue]
+    if (mapOrSingleValue instanceof ValueMap)
+      return [...mapOrSingleValue.values()]
+    const result: Array<[TValue, number]> = []
+    for (const singleValueOrValueMap of mapOrSingleValue.values()) {
+      if (isSingleValue(singleValueOrValueMap))
+        result.push(singleValueOrValueMap)
+      else
+        for (const valueTuple of singleValueOrValueMap.values())
+          result.push(valueTuple)
+    }
+    return result
   }
 
   /**
@@ -303,12 +344,15 @@ export class Index<TKey, TValue, TPrefix = any> {
     if (multiplicity === 0) return
 
     // Update consolidated multiplicity tracking
-    const newConsolidatedMultiplicity =
-      (this.#consolidatedMultiplicity.get(key) || 0) + multiplicity
-    if (newConsolidatedMultiplicity === 0) {
-      this.#consolidatedMultiplicity.delete(key)
-    } else {
-      this.#consolidatedMultiplicity.set(key, newConsolidatedMultiplicity)
+    const presence = this.#consolidatedMultiplicity
+    if (presence) {
+      const newConsolidatedMultiplicity =
+        (presence.get(key) || 0) + multiplicity
+      if (newConsolidatedMultiplicity === 0) {
+        presence.delete(key)
+      } else {
+        presence.set(key, newConsolidatedMultiplicity)
+      }
     }
 
     const mapOrSingleValue = this.#inner.get(key)
@@ -348,7 +392,11 @@ export class Index<TKey, TValue, TPrefix = any> {
       }
     } else {
       // Handle existing PrefixMap
-      const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
+      const isEmpty = mapOrSingleValue.addValue(
+        value,
+        multiplicity,
+        this.#prefixIdentity,
+      )
       if (isEmpty) {
         this.#inner.delete(key)
       }
@@ -384,7 +432,8 @@ export class Index<TKey, TValue, TPrefix = any> {
     // Check if they're the same value by prefix/suffix comparison
     if (
       isSamePrefix(currentPrefix, newPrefix) &&
-      (currentValue === newValue || hash(currentValue) === hash(newValue))
+      ((this.#prefixIdentity && newPrefix !== NO_PREFIX) ||
+        hash(currentValue) === hash(newValue))
     ) {
       const newMultiplicity = currentMultiplicity + multiplicity
       if (newMultiplicity === 0) {

package/src/operators/groupBy.ts

@@ -1,3 +1,4 @@
+import { structuralIdentity } from '../hashing/hash.js'
 import { serializeValue } from '../utils.js'
 import { map } from './map.js'
 import { reduce } from './reduce.js'
@@ -5,20 +6,12 @@ import type { IStreamBuilder, KeyValue } from '../types.js'
 
 type GroupKey = Record<string, unknown>
 
-type BasicAggregateFunction<T, R, V = unknown> = {
+type AggregateFunction<T, R, V = unknown> = {
   preMap: (data: T) => V
   reduce: (values: Array<[V, number]>) => V
   postMap?: (result: V) => R
 }
 
-type PipedAggregateFunction<T, R> = {
-  pipe: (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<string, R>>
-}
-
-type AggregateFunction<T, R, V = unknown> =
-  | BasicAggregateFunction<T, R, V>
-  | PipedAggregateFunction<T, R>
-
 type ExtractAggregateReturnType<T, A> =
   A extends AggregateFunction<T, infer R, any> ? R : never
 
@@ -26,12 +19,6 @@ type AggregatesReturnType<T, A> = {
   [K in keyof A]: ExtractAggregateReturnType<T, A[K]>
 }
 
-function isPipedAggregateFunction<T, R>(
-  aggregate: AggregateFunction<T, R>,
-): aggregate is PipedAggregateFunction<T, R> {
-  return `pipe` in aggregate
-}
-
 /**
  * Groups data by key and applies multiple aggregate operations
  * @param keyExtractor Function to extract grouping key from data
@@ -44,27 +31,19 @@ export function groupBy<
 >(keyExtractor: (data: T) => K, aggregates: A = {} as A) {
   type ResultType = K & AggregatesReturnType<T, A>
 
-  const basicAggregates = Object.fromEntries(
-    Object.entries(aggregates).filter(
-      ([_, aggregate]) => !isPipedAggregateFunction(aggregate),
-    ),
-  ) as Record<string, BasicAggregateFunction<T, any, any>>
-
-  // @ts-expect-error - TODO: we don't use this yet, but we will
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  const pipedAggregates = Object.fromEntries(
-    Object.entries(aggregates).filter(([_, aggregate]) =>
-      isPipedAggregateFunction(aggregate),
-    ),
-  ) as Record<string, PipedAggregateFunction<T, any>>
-
   return (
     stream: IStreamBuilder<T>,
   ): IStreamBuilder<KeyValue<string, ResultType>> => {
     // Special key to store the original key object
     const KEY_SENTINEL = Symbol(`original_group_key`)
 
-    // First map to extract keys and pre-aggregate values
+    const aggregateEntries = Object.entries(aggregates)
+
+    // First map to extract keys and pre-aggregate values. Each contribution
+    // leads with an identity built from its group key and pre-aggregate
+    // values when every one of them has one. Within one group the reduce
+    // index then merges equal contributions by that identity instead of
+    // hashing them. A null identity falls back to the structural hash.
     const withKeysAndValues = stream.pipe(
       map((data) => {
         const key = keyExtractor(data)
@@ -73,51 +52,61 @@ export function groupBy<
         // Create values object with pre-aggregated values
         const values: Record<string | symbol, unknown> = {}
 
-        // Store the original key object
+        // Store the original key object. Contributions whose keys serialize
+        // alike but hash apart must not merge, so the key is part of the
+        // identity.
         values[KEY_SENTINEL] = key
 
         // Add pre-aggregated values
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          values[name] = aggregate.preMap(data)
+        let identity: string | null = structuralIdentity(key) ?? null
+        for (const [name, aggregate] of aggregateEntries) {
+          const value = aggregate.preMap(data)
+          values[name] = value
+          if (identity === null) continue
+          const part = structuralIdentity(value)
+          identity = part === undefined ? null : `${identity}|${part}`
         }
 
-        return [keyString, values] as KeyValue<
+        return [keyString, [identity, values]] as KeyValue<
           string,
-          Record<string | symbol, unknown>
+          [string | null, Record<string | symbol, unknown>]
         >
       }),
     )
 
     // Then reduce to compute aggregates
     const reduced = withKeysAndValues.pipe(
-      reduce((values) => {
-        // Calculate total multiplicity to check if the group should exist
-        let totalMultiplicity = 0
-        for (const [_, multiplicity] of values) {
-          totalMultiplicity += multiplicity
-        }
+      reduce(
+        (contributions) => {
+          // Calculate total multiplicity to check if the group should exist
+          let totalMultiplicity = 0
+          for (const [_, multiplicity] of contributions) {
+            totalMultiplicity += multiplicity
+          }
 
-        // If total multiplicity is 0 or negative, the group should be removed completely
-        if (totalMultiplicity <= 0) {
-          return []
-        }
+          // If total multiplicity is 0 or negative, the group should be removed completely
+          if (totalMultiplicity <= 0) {
+            return []
+          }
 
-        const result: Record<PropertyKey, unknown> = {}
+          const result: Record<PropertyKey, unknown> = {}
 
-        // Get the original key from first value in group
-        const originalKey = values[0]?.[0]?.[KEY_SENTINEL]
-        result[KEY_SENTINEL] = originalKey
+          // Get the original key from first value in group
+          const originalKey = contributions[0]?.[0][1][KEY_SENTINEL]
+          result[KEY_SENTINEL] = originalKey
 
-        // Apply each aggregate function
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          const preValues = values.map(
-            ([v, m]) => [v[name], m] as [any, number],
-          )
-          result[name] = aggregate.reduce(preValues)
-        }
+          // Apply each aggregate function
+          for (const [name, aggregate] of aggregateEntries) {
+            const preValues = contributions.map(
+              ([[, v], m]) => [v[name], m] as [any, number],
+            )
+            result[name] = aggregate.reduce(preValues)
+          }
 
-        return [[result, 1]]
-      }),
+          return [[result, 1]]
+        },
+        { prefixIdentity: true },
+      ),
     )
 
     // Finally map to extract the key and include all values
@@ -133,7 +122,7 @@ export function groupBy<
         Object.assign(result, key)
 
         // Apply postMap if provided
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
+        for (const [name, aggregate] of aggregateEntries) {
           if (aggregate.postMap) {
             result[name] = aggregate.postMap(values[name])
           } else {

package/src/operators/groupedTopKWithFractionalIndex.ts

@@ -8,7 +8,7 @@ import {
   topKBatch,
 } from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
-import type { IndexedValue, TopK } from './topKArray.js'
+import type { IndexedValue } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
 import type { IStreamBuilder, PipedOperator } from '../types.js'
 
@@ -61,18 +61,6 @@ export class GroupedTopKWithFractionalIndexOperator<
     options.setWindowFn?.(this.#moveTopK.bind(this))
   }
 
-  /**
-   * Creates a new TopK data structure for a group.
-   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).
-   */
-  protected createTopK(
-    offset: number,
-    limit: number,
-    comparator: (a: [K, T], b: [K, T]) => number,
-  ): TopK<[K, T]> {
-    return new TopKArray(offset, limit, comparator)
-  }
-
   #getTotalSize(): number {
     let size = 0
     for (const state of this.#groupStates.values()) {
@@ -84,8 +72,9 @@ export class GroupedTopKWithFractionalIndexOperator<
   #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {
     let state = this.#groupStates.get(groupKey)
     if (!state) {
-      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)
-      state = new TopKState(topK)
+      state = new TopKState(
+        new TopKArray(this.#offset, this.#limit, this.#comparator),
+      )
       this.#groupStates.set(groupKey, state)
     }
     return state

package/src/operators/index.ts

@@ -2,20 +2,14 @@ export * from './pipe.js'
 export * from './map.js'
 export * from './tap.js'
 export * from './filter.js'
-export * from './negate.js'
 export * from './concat.js'
-export * from './debug.js'
 export * from './output.js'
 export * from './consolidate.js'
 export * from './join.js'
 export * from './reduce.js'
-export * from './count.js'
 export * from './distinct.js'
-export * from './keying.js'
-export * from './topK.js'
 export * from './topKWithFractionalIndex.js'
 export * from './groupedTopKWithFractionalIndex.js'
 export * from './orderBy.js'
 export * from './groupedOrderBy.js'
-export * from './filterBy.js'
 export { groupBy, groupByOperators } from './groupBy.js'

package/src/operators/join.ts

@@ -254,7 +254,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
  */
 export function join<
   K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
+  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
   V2,
   T,
 >(
@@ -282,93 +282,3 @@ export function join<
     return output
   }
 }
-
-/**
- * Joins two input streams (inner join)
- * @param other - The other stream to join with
- */
-export function innerJoin<
-  K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
-  V2,
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, V2>>,
-): PipedOperator<T, KeyValue<K, [V1, V2]>> {
-  return join(other, `inner`) as unknown as PipedOperator<
-    T,
-    KeyValue<K, [V1, V2]>
-  >
-}
-
-/**
- * Joins two input streams (anti join)
- * @param other - The other stream to join with
- */
-export function antiJoin<
-  K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
-  V2,
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, V2>>,
-): PipedOperator<T, KeyValue<K, [V1, null]>> {
-  return join(other, `anti`) as unknown as PipedOperator<
-    T,
-    KeyValue<K, [V1, null]>
-  >
-}
-
-/**
- * Joins two input streams (left join)
- * @param other - The other stream to join with
- */
-export function leftJoin<
-  K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
-  V2,
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, V2>>,
-): PipedOperator<T, KeyValue<K, [V1, V2 | null]>> {
-  return join(other, `left`) as unknown as PipedOperator<
-    T,
-    KeyValue<K, [V1, V2 | null]>
-  >
-}
-
-/**
- * Joins two input streams (right join)
- * @param other - The other stream to join with
- */
-export function rightJoin<
-  K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
-  V2,
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, V2>>,
-): PipedOperator<T, KeyValue<K, [V1 | null, V2]>> {
-  return join(other, `right`) as unknown as PipedOperator<
-    T,
-    KeyValue<K, [V1 | null, V2]>
-  >
-}
-
-/**
- * Joins two input streams (full join)
- * @param other - The other stream to join with
- */
-export function fullJoin<
-  K,
-  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
-  V2,
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, V2>>,
-): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
-  return join(other, `full`) as unknown as PipedOperator<
-    T,
-    KeyValue<K, [V1 | null, V2 | null]>
-  >
-}