@tanstack/db-ivm 0.1.26 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/db-ivm",
-  "version": "0.1.26",
+  "version": "0.2.0",
   "description": "Incremental View Maintenance for TanStack DB based on Differential Dataflow",
   "author": "Sam Willis",
   "license": "MIT",
@@ -41,14 +41,12 @@
     "src"
   ],
   "dependencies": {
-    "fractional-indexing": "^3.2.0",
-    "sorted-btree": "^1.8.1"
+    "fractional-indexing": "^3.2.0"
   },
   "peerDependencies": {
     "typescript": ">=4.7"
   },
   "devDependencies": {
-    "@types/debug": "^4.1.12",
     "@vitest/coverage-istanbul": "^3.2.4",
     "temporal-polyfill": "^0.3.0"
   },

package/src/hashing/hash.ts

@@ -72,8 +72,18 @@ function hashObject(input: object, context: HashContext): number {
       if (body) updateHasher(hasher, value, context)
       else hasher.update(value as string | number)
     if (body) {
-      // Sorted string keys, then enumerable symbol keys by identity.
-      const keys: Array<string | symbol> = Object.keys(body).sort(keySort)
+      // String keys in UTF-16 code-unit order, then enumerable symbol keys by
+      // identity. Code-unit order is total; locale collation can rank two
+      // distinct keys as equal and leave their insertion order in the hash.
+      const keys: Array<string | symbol> = Object.keys(body)
+      // A dense array with no other keys lists its indices in one order for
+      // every equal array: integer keys come first, so the last key is the
+      // last index. Anything else sorts.
+      const denseArray =
+        Array.isArray(body) &&
+        keys.length === body.length &&
+        (keys.length === 0 || keys[keys.length - 1] === String(body.length - 1))
+      if (!denseArray) keys.sort()
       keys.push(
         ...Object.getOwnPropertySymbols(body)
           .filter((key) =>
@@ -284,9 +294,81 @@ function cachedReferenceHash(fn: object): number {
   return valueHash
 }
 
+// One punctuation tag per objectParts marker, so values of different kinds
+// never share an identity (and no tag reads as an identifier in this source).
+const MARKER_TAGS = new Map<{ hash: number }, string>([
+  [ARRAY_MARKER, `[`],
+  [OBJECT_MARKER, `{`],
+  [DATE_MARKER, `<`],
+  [REGEXP_MARKER, `/`],
+  [MAP_MARKER, `&`],
+  [SET_MARKER, `*`],
+  [UINT8ARRAY_MARKER, `^`],
+  [TEMPORAL_MARKER, `?`],
+])
+
+// Nesting a structural identity follows: a min or max input is an array of
+// exact keys, and an exact key can hold a reference tuple.
+const MAX_IDENTITY_DEPTH = 4
+
 /**
- * Strings sorted lexicographically.
+ * @internal An exact identity string for a value, or undefined when it has
+ * no cheap one. Two values that `equalHashValues` treats as equal get the
+ * same result, both undefined or one string; two that it treats as different
+ * never share a string. The decision uses only what equalHashValues reads:
+ * the type, the `objectParts` marker and headers, and enumerable own keys. groupBy uses it
+ * to merge equal contributions without hashing them.
  */
-function keySort(a: string, b: string): number {
-  return a.localeCompare(b)
+export function structuralIdentity(
+  value: unknown,
+  depth = 0,
+): string | undefined {
+  switch (typeof value) {
+    case `number`:
+      // The hash treats -0 as 0 and every NaN as one value.
+      return `#${value === 0 ? 0 : value}`
+    case `string`:
+      return `"${value.length}:${value}`
+    case `boolean`:
+      return value ? `+` : `-`
+    case `undefined`:
+      return `~`
+    case `bigint`:
+      return `%${value}`
+    case `symbol`:
+      return `$${getSymbolIdentity(value)}`
+    case `object`: {
+      if (value === null) return `!`
+      if (depth >= MAX_IDENTITY_DEPTH || isReferenceHashedObject(value))
+        return undefined
+      const [marker, header, body] = objectParts(value)
+      // equalHashValues compares headers with ===, so a NaN header is never
+      // equal to another value's, and gets no identity.
+      let head = MARKER_TAGS.get(marker)!
+      for (const part of header) {
+        if (Number.isNaN(part)) return undefined
+        head += structuralIdentity(part, depth + 1)
+      }
+      if (!body) return head
+      if (
+        Object.getOwnPropertySymbols(body).some((key) =>
+          Object.prototype.propertyIsEnumerable.call(body, key),
+        )
+      )
+        return undefined
+      let parts = ``
+      for (const key of Object.keys(body).sort()) {
+        const part = structuralIdentity(
+          (body as Record<string, unknown>)[key],
+          depth + 1,
+        )
+        if (part === undefined) return undefined
+        parts += `${key.length}:${key}=${part}|`
+      }
+      // An array's header is its length, so a hole differs from undefined.
+      return `${head}:${parts.length}:${parts}`
+    }
+    default:
+      return undefined
+  }
 }

package/src/hashing/murmur.ts

@@ -63,11 +63,8 @@ export interface Hasher {
   digest: () => number
 }
 
-// Allocate these once, outside the hot path
-const buf = new ArrayBuffer(8)
-// dv and u8 are 2 different views on the same buffer `buf`
-const dv = new DataView(buf)
-const u8 = new Uint8Array(buf)
+// Allocate this once, outside the hot path
+const dv = new DataView(new ArrayBuffer(8))
 
 /**
  * This implementation of Murmur hash uses a random initial seed and random markers.
@@ -106,6 +103,20 @@ export class MurmurHashStream implements Hasher {
     }
   }
 
+  /** Write four bytes, low byte first, as four `writeByte` calls would. */
+  private writeWord(word: number): void {
+    this.length += 4
+    if (this.carryBytes === 0) {
+      this._mix(word >>> 0)
+      return
+    }
+    // The carry's bytes come first. The word's low bytes complete that block,
+    // and its high bytes become the new carry, which keeps its byte count.
+    const shift = 8 * this.carryBytes
+    this._mix((this.carry | (word << shift)) >>> 0)
+    this.carry = word >>> (32 - shift)
+  }
+
   update(chunk: symbol | string | number | bigint): void {
     switch (typeof chunk) {
       case `symbol`: {
@@ -115,22 +126,25 @@ export class MurmurHashStream implements Hasher {
       }
       case `string`:
         this.update(STRING_MARKER)
-        for (let i = 0; i < chunk.length; i++) {
-          const code = chunk.charCodeAt(i)
-          this.writeByte(code & 0xff)
-          this.writeByte((code >>> 8) & 0xff)
+        {
+          // Each code unit is two bytes, so a pair of them is one word.
+          const pairs = chunk.length - (chunk.length % 2)
+          for (let i = 0; i < pairs; i += 2) {
+            this.writeWord(
+              chunk.charCodeAt(i) | (chunk.charCodeAt(i + 1) << 16),
+            )
+          }
+          if (pairs < chunk.length) {
+            const code = chunk.charCodeAt(pairs)
+            this.writeByte(code & 0xff)
+            this.writeByte((code >>> 8) & 0xff)
+          }
         }
         return
       case `number`:
         dv.setFloat64(0, chunk, true) // fixed little-endian
-        this.writeByte(u8[0]!)
-        this.writeByte(u8[1]!)
-        this.writeByte(u8[2]!)
-        this.writeByte(u8[3]!)
-        this.writeByte(u8[4]!)
-        this.writeByte(u8[5]!)
-        this.writeByte(u8[6]!)
-        this.writeByte(u8[7]!)
+        this.writeWord(dv.getUint32(0, true))
+        this.writeWord(dv.getUint32(4, true))
         return
       case `bigint`: {
         let value = chunk

package/src/indexes.ts

@@ -58,8 +58,13 @@ class PrefixMap<TValue, TPrefix> extends Map<
 > {
   /**
    * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.
+   * With `prefixIdentity`, values that share a prefix are the same value.
    */
-  addValue(value: TValue, multiplicity: number): boolean {
+  addValue(
+    value: TValue,
+    multiplicity: number,
+    prefixIdentity = false,
+  ): boolean {
     if (multiplicity === 0) return this.size === 0
 
     const prefix = getPrefix<TValue, TPrefix>(value)
@@ -73,7 +78,11 @@ class PrefixMap<TValue, TPrefix> extends Map<
         throw new Error(`Mismatching prefixes, this should never happen`)
       }
 
-      if (currentValue === value || hash(currentValue) === hash(value)) {
+      if (
+        currentValue === value ||
+        (prefixIdentity && prefix !== NO_PREFIX) ||
+        hash(currentValue) === hash(value)
+      ) {
         // Same value, update multiplicity
         const newMultiplicity = currentMultiplicity + multiplicity
         if (newMultiplicity === 0) {
@@ -150,10 +159,21 @@ export class Index<TKey, TValue, TPrefix = any> {
    *   hash to identify identical values, storing them in a third level value map.
    */
   #inner: IndexMap<TKey, TValue, TPrefix>
-  #consolidatedMultiplicity: Map<TKey, number> = new Map() // sum of multiplicities per key
+  // Sum of multiplicities per key, for presence. Only joins read presence; a
+  // reduce index is built without it.
+  #consolidatedMultiplicity: Map<TKey, number> | undefined
+  /**
+   * When true, the producer guarantees that a value's string or number
+   * prefix determines the value: two values under one key with an equal
+   * prefix are equal, so they merge without a structural hash.
+   */
+  #prefixIdentity: boolean
 
-  constructor() {
+  constructor(options?: { prefixIdentity?: boolean; trackPresence?: boolean }) {
     this.#inner = new Map()
+    if (options?.trackPresence ?? true)
+      this.#consolidatedMultiplicity = new Map()
+    this.#prefixIdentity = options?.prefixIdentity ?? false
   }
 
   /**
@@ -194,6 +214,12 @@ export class Index<TKey, TValue, TPrefix = any> {
     return this.#inner.size
   }
 
+  #presence(): Map<TKey, number> {
+    if (this.#consolidatedMultiplicity === undefined)
+      throw new Error(`This Index does not track presence`)
+    return this.#consolidatedMultiplicity
+  }
+
   /**
    * This method checks if the index has a given key.
    * @param key - The key to check.
@@ -209,7 +235,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns True if the key has non-zero consolidated multiplicity, false otherwise.
    */
   hasPresence(key: TKey): boolean {
-    return (this.#consolidatedMultiplicity.get(key) || 0) !== 0
+    return (this.#presence().get(key) || 0) !== 0
   }
 
   /**
@@ -218,7 +244,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns The consolidated multiplicity for the key.
    */
   getConsolidatedMultiplicity(key: TKey): number {
-    return this.#consolidatedMultiplicity.get(key) || 0
+    return this.#presence().get(key) || 0
   }
 
   /**
@@ -226,7 +252,7 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns An iterator of keys with non-zero consolidated multiplicity.
    */
   getPresenceKeys(): Iterable<TKey> {
-    return this.#consolidatedMultiplicity.keys()
+    return this.#presence().keys()
   }
 
   /**
@@ -235,7 +261,22 @@ export class Index<TKey, TValue, TPrefix = any> {
    * @returns An array of value tuples [value, multiplicity].
    */
   get(key: TKey): Array<[TValue, number]> {
-    return [...this.getIterator(key)]
+    // Built directly, in getIterator's order: join and reduce call this for
+    // every changed key.
+    const mapOrSingleValue = this.#inner.get(key)
+    if (mapOrSingleValue === undefined) return []
+    if (isSingleValue(mapOrSingleValue)) return [mapOrSingleValue]
+    if (mapOrSingleValue instanceof ValueMap)
+      return [...mapOrSingleValue.values()]
+    const result: Array<[TValue, number]> = []
+    for (const singleValueOrValueMap of mapOrSingleValue.values()) {
+      if (isSingleValue(singleValueOrValueMap))
+        result.push(singleValueOrValueMap)
+      else
+        for (const valueTuple of singleValueOrValueMap.values())
+          result.push(valueTuple)
+    }
+    return result
   }
 
   /**
@@ -303,12 +344,15 @@ export class Index<TKey, TValue, TPrefix = any> {
     if (multiplicity === 0) return
 
     // Update consolidated multiplicity tracking
-    const newConsolidatedMultiplicity =
-      (this.#consolidatedMultiplicity.get(key) || 0) + multiplicity
-    if (newConsolidatedMultiplicity === 0) {
-      this.#consolidatedMultiplicity.delete(key)
-    } else {
-      this.#consolidatedMultiplicity.set(key, newConsolidatedMultiplicity)
+    const presence = this.#consolidatedMultiplicity
+    if (presence) {
+      const newConsolidatedMultiplicity =
+        (presence.get(key) || 0) + multiplicity
+      if (newConsolidatedMultiplicity === 0) {
+        presence.delete(key)
+      } else {
+        presence.set(key, newConsolidatedMultiplicity)
+      }
     }
 
     const mapOrSingleValue = this.#inner.get(key)
@@ -348,7 +392,11 @@ export class Index<TKey, TValue, TPrefix = any> {
       }
     } else {
       // Handle existing PrefixMap
-      const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
+      const isEmpty = mapOrSingleValue.addValue(
+        value,
+        multiplicity,
+        this.#prefixIdentity,
+      )
       if (isEmpty) {
         this.#inner.delete(key)
       }
@@ -384,7 +432,8 @@ export class Index<TKey, TValue, TPrefix = any> {
     // Check if they're the same value by prefix/suffix comparison
     if (
       isSamePrefix(currentPrefix, newPrefix) &&
-      (currentValue === newValue || hash(currentValue) === hash(newValue))
+      ((this.#prefixIdentity && newPrefix !== NO_PREFIX) ||
+        hash(currentValue) === hash(newValue))
     ) {
       const newMultiplicity = currentMultiplicity + multiplicity
       if (newMultiplicity === 0) {

package/src/operators/groupBy.ts

@@ -1,3 +1,4 @@
+import { structuralIdentity } from '../hashing/hash.js'
 import { serializeValue } from '../utils.js'
 import { map } from './map.js'
 import { reduce } from './reduce.js'
@@ -5,19 +6,12 @@ import type { IStreamBuilder, KeyValue } from '../types.js'
 
 type GroupKey = Record<string, unknown>
 
-type BasicAggregateFunction<T, R, V = unknown> = {
+type AggregateFunction<T, R, V = unknown> = {
   preMap: (data: T) => V
   reduce: (values: Array<[V, number]>) => V
   postMap?: (result: V) => R
 }
 
-type PipedAggregateFunction<T, R> = {
-  pipe: (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<string, R>>
-}
-
-type AggregateFunction<T, R, V = unknown> =
-  BasicAggregateFunction<T, R, V> | PipedAggregateFunction<T, R>
-
 type ExtractAggregateReturnType<T, A> =
   A extends AggregateFunction<T, infer R, any> ? R : never
 
@@ -25,12 +19,6 @@ type AggregatesReturnType<T, A> = {
   [K in keyof A]: ExtractAggregateReturnType<T, A[K]>
 }
 
-function isPipedAggregateFunction<T, R>(
-  aggregate: AggregateFunction<T, R>,
-): aggregate is PipedAggregateFunction<T, R> {
-  return `pipe` in aggregate
-}
-
 /**
  * Groups data by key and applies multiple aggregate operations
  * @param keyExtractor Function to extract grouping key from data
@@ -43,27 +31,19 @@ export function groupBy<
 >(keyExtractor: (data: T) => K, aggregates: A = {} as A) {
   type ResultType = K & AggregatesReturnType<T, A>
 
-  const basicAggregates = Object.fromEntries(
-    Object.entries(aggregates).filter(
-      ([_, aggregate]) => !isPipedAggregateFunction(aggregate),
-    ),
-  ) as Record<string, BasicAggregateFunction<T, any, any>>
-
-  // @ts-expect-error - TODO: we don't use this yet, but we will
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  const pipedAggregates = Object.fromEntries(
-    Object.entries(aggregates).filter(([_, aggregate]) =>
-      isPipedAggregateFunction(aggregate),
-    ),
-  ) as Record<string, PipedAggregateFunction<T, any>>
-
   return (
     stream: IStreamBuilder<T>,
   ): IStreamBuilder<KeyValue<string, ResultType>> => {
     // Special key to store the original key object
     const KEY_SENTINEL = Symbol(`original_group_key`)
 
-    // First map to extract keys and pre-aggregate values
+    const aggregateEntries = Object.entries(aggregates)
+
+    // First map to extract keys and pre-aggregate values. Each contribution
+    // leads with an identity built from its group key and pre-aggregate
+    // values when every one of them has one. Within one group the reduce
+    // index then merges equal contributions by that identity instead of
+    // hashing them. A null identity falls back to the structural hash.
     const withKeysAndValues = stream.pipe(
       map((data) => {
         const key = keyExtractor(data)
@@ -72,51 +52,61 @@ export function groupBy<
         // Create values object with pre-aggregated values
         const values: Record<string | symbol, unknown> = {}
 
-        // Store the original key object
+        // Store the original key object. Contributions whose keys serialize
+        // alike but hash apart must not merge, so the key is part of the
+        // identity.
         values[KEY_SENTINEL] = key
 
         // Add pre-aggregated values
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          values[name] = aggregate.preMap(data)
+        let identity: string | null = structuralIdentity(key) ?? null
+        for (const [name, aggregate] of aggregateEntries) {
+          const value = aggregate.preMap(data)
+          values[name] = value
+          if (identity === null) continue
+          const part = structuralIdentity(value)
+          identity = part === undefined ? null : `${identity}|${part}`
         }
 
-        return [keyString, values] as KeyValue<
+        return [keyString, [identity, values]] as KeyValue<
           string,
-          Record<string | symbol, unknown>
+          [string | null, Record<string | symbol, unknown>]
         >
       }),
     )
 
     // Then reduce to compute aggregates
     const reduced = withKeysAndValues.pipe(
-      reduce((values) => {
-        // Calculate total multiplicity to check if the group should exist
-        let totalMultiplicity = 0
-        for (const [_, multiplicity] of values) {
-          totalMultiplicity += multiplicity
-        }
+      reduce(
+        (contributions) => {
+          // Calculate total multiplicity to check if the group should exist
+          let totalMultiplicity = 0
+          for (const [_, multiplicity] of contributions) {
+            totalMultiplicity += multiplicity
+          }
 
-        // If total multiplicity is 0 or negative, the group should be removed completely
-        if (totalMultiplicity <= 0) {
-          return []
-        }
+          // If total multiplicity is 0 or negative, the group should be removed completely
+          if (totalMultiplicity <= 0) {
+            return []
+          }
 
-        const result: Record<PropertyKey, unknown> = {}
+          const result: Record<PropertyKey, unknown> = {}
 
-        // Get the original key from first value in group
-        const originalKey = values[0]?.[0]?.[KEY_SENTINEL]
-        result[KEY_SENTINEL] = originalKey
+          // Get the original key from first value in group
+          const originalKey = contributions[0]?.[0][1][KEY_SENTINEL]
+          result[KEY_SENTINEL] = originalKey
 
-        // Apply each aggregate function
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          const preValues = values.map(
-            ([v, m]) => [v[name], m] as [any, number],
-          )
-          result[name] = aggregate.reduce(preValues)
-        }
+          // Apply each aggregate function
+          for (const [name, aggregate] of aggregateEntries) {
+            const preValues = contributions.map(
+              ([[, v], m]) => [v[name], m] as [any, number],
+            )
+            result[name] = aggregate.reduce(preValues)
+          }
 
-        return [[result, 1]]
-      }),
+          return [[result, 1]]
+        },
+        { prefixIdentity: true },
+      ),
     )
 
     // Finally map to extract the key and include all values
@@ -132,7 +122,7 @@ export function groupBy<
         Object.assign(result, key)
 
         // Apply postMap if provided
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
+        for (const [name, aggregate] of aggregateEntries) {
           if (aggregate.postMap) {
             result[name] = aggregate.postMap(values[name])
           } else {

package/src/operators/groupedTopKWithFractionalIndex.ts

@@ -8,7 +8,7 @@ import {
   topKBatch,
 } from './topKState.js'
 import { TopKArray, createKeyedComparator } from './topKArray.js'
-import type { IndexedValue, TopK } from './topKArray.js'
+import type { IndexedValue } from './topKArray.js'
 import type { DifferenceStreamReader } from '../graph.js'
 import type { IStreamBuilder, PipedOperator } from '../types.js'
 
@@ -61,18 +61,6 @@ export class GroupedTopKWithFractionalIndexOperator<
     options.setWindowFn?.(this.#moveTopK.bind(this))
   }
 
-  /**
-   * Creates a new TopK data structure for a group.
-   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).
-   */
-  protected createTopK(
-    offset: number,
-    limit: number,
-    comparator: (a: [K, T], b: [K, T]) => number,
-  ): TopK<[K, T]> {
-    return new TopKArray(offset, limit, comparator)
-  }
-
   #getTotalSize(): number {
     let size = 0
     for (const state of this.#groupStates.values()) {
@@ -84,8 +72,9 @@ export class GroupedTopKWithFractionalIndexOperator<
   #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {
     let state = this.#groupStates.get(groupKey)
     if (!state) {
-      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)
-      state = new TopKState(topK)
+      state = new TopKState(
+        new TopKArray(this.#offset, this.#limit, this.#comparator),
+      )
       this.#groupStates.set(groupKey, state)
     }
     return state

package/src/operators/index.ts

@@ -2,20 +2,14 @@ export * from './pipe.js'
 export * from './map.js'
 export * from './tap.js'
 export * from './filter.js'
-export * from './negate.js'
 export * from './concat.js'
-export * from './debug.js'
 export * from './output.js'
 export * from './consolidate.js'
 export * from './join.js'
 export * from './reduce.js'
-export * from './count.js'
 export * from './distinct.js'
-export * from './keying.js'
-export * from './topK.js'
 export * from './topKWithFractionalIndex.js'
 export * from './groupedTopKWithFractionalIndex.js'
 export * from './orderBy.js'
 export * from './groupedOrderBy.js'
-export * from './filterBy.js'
 export { groupBy, groupByOperators } from './groupBy.js'