@tanstack/db-ivm 0.1.7 → 0.1.8

package/dist/esm/utils.js

@@ -83,23 +83,11 @@ class ObjectIdGenerator {
   }
 }
 const globalObjectIdGenerator = new ObjectIdGenerator();
-function* concatIterable(...iterables) {
-  for (const iterable of iterables) {
-    yield* iterable;
-  }
-}
-function* mapIterable(it, fn) {
-  for (const t of it) {
-    yield fn(t);
-  }
-}
 export {
   DefaultMap,
   ObjectIdGenerator,
   binarySearch,
   chunkedArrayPush,
-  concatIterable,
-  globalObjectIdGenerator,
-  mapIterable
+  globalObjectIdGenerator
 };
 //# sourceMappingURL=utils.js.map

package/dist/esm/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sources":["../../src/utils.ts"],"sourcesContent":["/**\n * Simple assertion function for runtime checks.\n * Throws an error if the condition is false.\n */\nexport function assert(\n  condition: unknown,\n  message?: string\n): asserts condition {\n  if (!condition) {\n    throw new Error(message || `Assertion failed`)\n  }\n}\n\n/**\n * A map that returns a default value for keys that are not present.\n */\nexport class DefaultMap<K, V> extends Map<K, V> {\n  constructor(\n    private defaultValue: () => V,\n    entries?: Iterable<[K, V]>\n  ) {\n    super(entries)\n  }\n\n  get(key: K): V {\n    if (!this.has(key)) {\n      // this.set(key, this.defaultValue())\n      return this.defaultValue()\n    }\n    return super.get(key)!\n  }\n\n  /**\n   * Update the value for a key using a function.\n   */\n  update(key: K, updater: (value: V) => V): V {\n    const value = this.get(key)\n    const newValue = updater(value)\n    this.set(key, newValue)\n    return newValue\n  }\n}\n\n// JS engines have various limits on how many args can be passed to a function\n// with a spread operator, so we need to split the operation into chunks\n// 32767 is the max for Chrome 14, all others are higher\n// TODO: investigate the performance of this and other approaches\nconst chunkSize = 30000\nexport function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {\n  if (other.length <= chunkSize) {\n    array.push(...other)\n  } else {\n    for (let i = 0; i < other.length; i += chunkSize) {\n      const chunk = other.slice(i, i + chunkSize)\n      array.push(...chunk)\n    }\n  }\n}\n\nexport function binarySearch<T>(\n  array: Array<T>,\n  value: T,\n  comparator: (a: T, b: T) => number\n): number {\n  let low = 0\n  let high = array.length\n  while (low < high) {\n    const mid = Math.floor((low + high) / 2)\n    const comparison = comparator(array[mid]!, value)\n    if (comparison < 0) {\n      low = mid + 1\n    } else if (comparison > 0) {\n      high = mid\n    } else {\n      return mid\n    }\n  }\n  return low\n}\n\n/**\n * Utility for generating unique IDs for objects and values.\n * Uses WeakMap for object reference tracking and consistent hashing for primitives.\n */\nexport class ObjectIdGenerator {\n  private objectIds = new WeakMap<object, number>()\n  private nextId = 0\n\n  /**\n   * Get a unique identifier for any value.\n   * - Objects: Uses WeakMap for reference-based identity\n   * - Primitives: Uses consistent string-based hashing\n   */\n  getId(value: any): number {\n    // For primitives, use a simple hash of their string representation\n    if (typeof value !== `object` || value === null) {\n      const str = String(value)\n      let hashValue = 0\n      for (let i = 0; i < str.length; i++) {\n        const char = str.charCodeAt(i)\n        hashValue = (hashValue << 5) - hashValue + char\n        hashValue = hashValue & hashValue // Convert to 32-bit integer\n      }\n      return hashValue\n    }\n\n    // For objects, use WeakMap to assign unique IDs\n    if (!this.objectIds.has(value)) {\n      this.objectIds.set(value, this.nextId++)\n    }\n    return this.objectIds.get(value)!\n  }\n\n  /**\n   * Get a string representation of the ID for use in composite keys.\n   */\n  getStringId(value: any): string {\n    if (value === null) return `null`\n    if (value === undefined) return `undefined`\n    if (typeof value !== `object`) return `str_${String(value)}`\n\n    return `obj_${this.getId(value)}`\n  }\n}\n\n/**\n * Global instance for cases where a shared object ID space is needed.\n */\nexport const globalObjectIdGenerator = new ObjectIdGenerator()\n\nexport function* concatIterable<T>(\n  ...iterables: Array<Iterable<T>>\n): Iterable<T> {\n  for (const iterable of iterables) {\n    yield* iterable\n  }\n}\n\nexport function* mapIterable<T, U>(\n  it: Iterable<T>,\n  fn: (t: T) => U\n): Iterable<U> {\n  for (const t of it) {\n    yield fn(t)\n  }\n}\n"],"names":[],"mappings":"AAgBO,MAAM,mBAAyB,IAAU;AAAA,EAC9C,YACU,cACR,SACA;AACA,UAAM,OAAO;AAHL,SAAA,eAAA;AAAA,EAIV;AAAA,EAEA,IAAI,KAAW;AACb,QAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAElB,aAAO,KAAK,aAAA;AAAA,IACd;AACA,WAAO,MAAM,IAAI,GAAG;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,KAAQ,SAA6B;AAC1C,UAAM,QAAQ,KAAK,IAAI,GAAG;AAC1B,UAAM,WAAW,QAAQ,KAAK;AAC9B,SAAK,IAAI,KAAK,QAAQ;AACtB,WAAO;AAAA,EACT;AACF;AAMA,MAAM,YAAY;AACX,SAAS,iBAAiB,OAAuB,OAAuB;AAC7E,MAAI,MAAM,UAAU,WAAW;AAC7B,UAAM,KAAK,GAAG,KAAK;AAAA,EACrB,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,SAAS;AAC1C,YAAM,KAAK,GAAG,KAAK;AAAA,IACrB;AAAA,EACF;AACF;AAEO,SAAS,aACd,OACA,OACA,YACQ;AACR,MAAI,MAAM;AACV,MAAI,OAAO,MAAM;AACjB,SAAO,MAAM,MAAM;AACjB,UAAM,MAAM,KAAK,OAAO,MAAM,QAAQ,CAAC;AACvC,UAAM,aAAa,WAAW,MAAM,GAAG,GAAI,KAAK;AAChD,QAAI,aAAa,GAAG;AAClB,YAAM,MAAM;AAAA,IACd,WAAW,aAAa,GAAG;AACzB,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAMO,MAAM,kBAAkB;AAAA,EAAxB,cAAA;AACL,SAAQ,gCAAgB,QAAA;AACxB,SAAQ,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOjB,MAAM,OAAoB;AAExB,QAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,YAAM,MAAM,OAAO,KAAK;AACxB,UAAI,YAAY;AAChB,eAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,cAAM,OAAO,IAAI,WAAW,CAAC;AAC7B,qBAAa,aAAa,KAAK,YAAY;AAC3C,oBAAY,YAAY;AAAA,MAC1B;AACA,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,UAAU,IAAI,KAAK,GAAG;AAC9B,WAAK,UAAU,IAAI,OAAO,KAAK,QAAQ;AAAA,IACzC;AACA,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,OAAoB;AAC9B,QAAI,UAAU,KAAM,QAAO;AAC3B,QAAI,UAAU,OAAW,QAAO;AAChC,QAAI,OAAO,UAAU,iBAAiB,OAAO,OAAO,KAAK,CAAC;AAE1D,WAAO,OAAO,KAAK,MAAM,KAAK,CAAC;AAAA,EACjC;AACF;AAKO,MAAM,0BAA0B,IAAI,kBAAA;AAEpC,UAAU,kBACZ,WACU;AACb,aAAW,YAAY,WAAW;AAChC,WAAO;AAAA,EACT;AACF;AAEO,UAAU,YACf,IACA,IACa;AACb,aAAW,KAAK,IAAI;AAClB,UAAM,GAAG,CAAC;AAAA,EACZ;AACF;"}
+{"version":3,"file":"utils.js","sources":["../../src/utils.ts"],"sourcesContent":["/**\n * Simple assertion function for runtime checks.\n * Throws an error if the condition is false.\n */\nexport function assert(\n  condition: unknown,\n  message?: string\n): asserts condition {\n  if (!condition) {\n    throw new Error(message || `Assertion failed`)\n  }\n}\n\n/**\n * A map that returns a default value for keys that are not present.\n */\nexport class DefaultMap<K, V> extends Map<K, V> {\n  constructor(\n    private defaultValue: () => V,\n    entries?: Iterable<[K, V]>\n  ) {\n    super(entries)\n  }\n\n  get(key: K): V {\n    if (!this.has(key)) {\n      // this.set(key, this.defaultValue())\n      return this.defaultValue()\n    }\n    return super.get(key)!\n  }\n\n  /**\n   * Update the value for a key using a function.\n   */\n  update(key: K, updater: (value: V) => V): V {\n    const value = this.get(key)\n    const newValue = updater(value)\n    this.set(key, newValue)\n    return newValue\n  }\n}\n\n// JS engines have various limits on how many args can be passed to a function\n// with a spread operator, so we need to split the operation into chunks\n// 32767 is the max for Chrome 14, all others are higher\n// TODO: investigate the performance of this and other approaches\nconst chunkSize = 30000\nexport function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {\n  if (other.length <= chunkSize) {\n    array.push(...other)\n  } else {\n    for (let i = 0; i < other.length; i += chunkSize) {\n      const chunk = other.slice(i, i + chunkSize)\n      array.push(...chunk)\n    }\n  }\n}\n\nexport function binarySearch<T>(\n  array: Array<T>,\n  value: T,\n  comparator: (a: T, b: T) => number\n): number {\n  let low = 0\n  let high = array.length\n  while (low < high) {\n    const mid = Math.floor((low + high) / 2)\n    const comparison = comparator(array[mid]!, value)\n    if (comparison < 0) {\n      low = mid + 1\n    } else if (comparison > 0) {\n      high = mid\n    } else {\n      return mid\n    }\n  }\n  return low\n}\n\n/**\n * Utility for generating unique IDs for objects and values.\n * Uses WeakMap for object reference tracking and consistent hashing for primitives.\n */\nexport class ObjectIdGenerator {\n  private objectIds = new WeakMap<object, number>()\n  private nextId = 0\n\n  /**\n   * Get a unique identifier for any value.\n   * - Objects: Uses WeakMap for reference-based identity\n   * - Primitives: Uses consistent string-based hashing\n   */\n  getId(value: any): number {\n    // For primitives, use a simple hash of their string representation\n    if (typeof value !== `object` || value === null) {\n      const str = String(value)\n      let hashValue = 0\n      for (let i = 0; i < str.length; i++) {\n        const char = str.charCodeAt(i)\n        hashValue = (hashValue << 5) - hashValue + char\n        hashValue = hashValue & hashValue // Convert to 32-bit integer\n      }\n      return hashValue\n    }\n\n    // For objects, use WeakMap to assign unique IDs\n    if (!this.objectIds.has(value)) {\n      this.objectIds.set(value, this.nextId++)\n    }\n    return this.objectIds.get(value)!\n  }\n\n  /**\n   * Get a string representation of the ID for use in composite keys.\n   */\n  getStringId(value: any): string {\n    if (value === null) return `null`\n    if (value === undefined) return `undefined`\n    if (typeof value !== `object`) return `str_${String(value)}`\n\n    return `obj_${this.getId(value)}`\n  }\n}\n\n/**\n * Global instance for cases where a shared object ID space is needed.\n */\nexport const globalObjectIdGenerator = new ObjectIdGenerator()\n\nexport function* concatIterable<T>(\n  ...iterables: Array<Iterable<T>>\n): Iterable<T> {\n  for (const iterable of iterables) {\n    yield* iterable\n  }\n}\n\nexport function* mapIterable<T, U>(\n  it: Iterable<T>,\n  fn: (t: T) => U\n): Iterable<U> {\n  for (const t of it) {\n    yield fn(t)\n  }\n}\n"],"names":[],"mappings":"AAgBO,MAAM,mBAAyB,IAAU;AAAA,EAC9C,YACU,cACR,SACA;AACA,UAAM,OAAO;AAHL,SAAA,eAAA;AAAA,EAIV;AAAA,EAEA,IAAI,KAAW;AACb,QAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAElB,aAAO,KAAK,aAAA;AAAA,IACd;AACA,WAAO,MAAM,IAAI,GAAG;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,KAAQ,SAA6B;AAC1C,UAAM,QAAQ,KAAK,IAAI,GAAG;AAC1B,UAAM,WAAW,QAAQ,KAAK;AAC9B,SAAK,IAAI,KAAK,QAAQ;AACtB,WAAO;AAAA,EACT;AACF;AAMA,MAAM,YAAY;AACX,SAAS,iBAAiB,OAAuB,OAAuB;AAC7E,MAAI,MAAM,UAAU,WAAW;AAC7B,UAAM,KAAK,GAAG,KAAK;AAAA,EACrB,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,SAAS;AAC1C,YAAM,KAAK,GAAG,KAAK;AAAA,IACrB;AAAA,EACF;AACF;AAEO,SAAS,aACd,OACA,OACA,YACQ;AACR,MAAI,MAAM;AACV,MAAI,OAAO,MAAM;AACjB,SAAO,MAAM,MAAM;AACjB,UAAM,MAAM,KAAK,OAAO,MAAM,QAAQ,CAAC;AACvC,UAAM,aAAa,WAAW,MAAM,GAAG,GAAI,KAAK;AAChD,QAAI,aAAa,GAAG;AAClB,YAAM,MAAM;AAAA,IACd,WAAW,aAAa,GAAG;AACzB,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAMO,MAAM,kBAAkB;AAAA,EAAxB,cAAA;AACL,SAAQ,gCAAgB,QAAA;AACxB,SAAQ,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOjB,MAAM,OAAoB;AAExB,QAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,YAAM,MAAM,OAAO,KAAK;AACxB,UAAI,YAAY;AAChB,eAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,cAAM,OAAO,IAAI,WAAW,CAAC;AAC7B,qBAAa,aAAa,KAAK,YAAY;AAC3C,oBAAY,YAAY;AAAA,MAC1B;AACA,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,UAAU,IAAI,KAAK,GAAG;AAC9B,WAAK,UAAU,IAAI,OAAO,KAAK,QAAQ;AAAA,IACzC;AACA,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,OAAoB;AAC9B,QAAI,UAAU,KAAM,QAAO;AAC3B,QAAI,UAAU,OAAW,QAAO;AAChC,QAAI,OAAO,UAAU,iBAAiB,OAAO,OAAO,KAAK,CAAC;AAE1D,WAAO,OAAO,KAAK,MAAM,KAAK,CAAC;AAAA,EACjC;AACF;AAKO,MAAM,0BAA0B,IAAI,kBAAA;"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tanstack/db-ivm",
   "description": "Incremental View Maintenance for TanStack DB based on Differential Dataflow",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "dependencies": {
     "fractional-indexing": "^3.2.0",
     "sorted-btree": "^1.8.1"

package/src/indexes.ts

@@ -1,60 +1,238 @@
+/**
+ * # Optimized Index Data Structure
+ *
+ * Multi-level index that adapts storage strategy based on data patterns to minimize memory
+ * usage, eliminate wasteful lookups, and avoid hashing whenever possible.
+ *
+ * ## Storage Strategy
+ *
+ * **Single value**: `IndexMap['key'] → [value, multiplicity]` (no hashing needed)
+ *
+ * **Multiple unprefixed values**: Direct ValueMap (avoids NO_PREFIX lookup)
+ * ```
+ * IndexMap['key'] → ValueMap { hash(value1) → [value1, mult1], ... }
+ * ```
+ *
+ * **Values with prefixes**: PrefixMap uses prefix keys directly (no hashing)
+ * ```
+ * IndexMap['key'] → PrefixMap { 'prefix1' → [value1, mult1], NO_PREFIX → ValueMap{...} }
+ * ```
+ *
+ * **Multiple values per prefix**: ValueMap within PrefixMap (hash only suffixes)
+ * ```
+ * PrefixMap['prefix'] → ValueMap { hash(suffix1) → [full_value1, mult1], ... }
+ * ```
+ *
+ * ## Dynamic Evolution
+ *
+ * Structure automatically evolves as data is added:
+ * - Single → ValueMap (when both values unprefixed)
+ * - Single → PrefixMap (when at least one prefixed)
+ * - ValueMap → PrefixMap (adding prefixed value to unprefixed)
+ *
+ * Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'
+ */
+
 import { MultiSet } from "./multiset.js"
-import { HashIndex } from "./hashIndex.js"
-import { ValueIndex } from "./valueIndex.js"
-import { concatIterable, mapIterable } from "./utils.js"
+import { hash } from "./hashing/index.js"
+import type { Hash } from "./hashing/index.js"
+
+// We use a symbol to represent the absence of a prefix, unprefixed values a stored
+// against this key.
+const NO_PREFIX = Symbol(`NO_PREFIX`)
+type NO_PREFIX = typeof NO_PREFIX
+
+// A single value is a tuple of the value and the multiplicity.
+type SingleValue<TValue> = [TValue, number]
+
+// Base map type for the index. Stores single values, prefix maps, or value maps against a key.
+type IndexMap<TKey, TValue, TPrefix> = Map<
+  TKey,
+  SingleValue<TValue> | PrefixMap<TValue, TPrefix> | ValueMap<TValue>
+>
+
+// Second level map type for the index, stores single values or value maps against a prefix.
+class PrefixMap<TValue, TPrefix> extends Map<
+  TPrefix | NO_PREFIX,
+  SingleValue<TValue> | ValueMap<TValue>
+> {
+  /**
+   * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.
+   */
+  addValue(value: TValue, multiplicity: number): boolean {
+    if (multiplicity === 0) return this.size === 0
+
+    const prefix = getPrefix<TValue, TPrefix>(value)
+    const valueMapOrSingleValue = this.get(prefix)
+
+    if (isSingleValue(valueMapOrSingleValue)) {
+      const [currentValue, currentMultiplicity] = valueMapOrSingleValue
+      const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
+
+      if (currentPrefix !== prefix) {
+        throw new Error(`Mismatching prefixes, this should never happen`)
+      }
+
+      if (currentValue === value || hash(currentValue) === hash(value)) {
+        // Same value, update multiplicity
+        const newMultiplicity = currentMultiplicity + multiplicity
+        if (newMultiplicity === 0) {
+          this.delete(prefix)
+        } else {
+          this.set(prefix, [value, newMultiplicity])
+        }
+      } else {
+        // Different suffixes, need to create ValueMap
+        const valueMap = new ValueMap<TValue>()
+        valueMap.set(hash(currentValue), valueMapOrSingleValue)
+        valueMap.set(hash(value), [value, multiplicity])
+        this.set(prefix, valueMap)
+      }
+    } else if (valueMapOrSingleValue === undefined) {
+      // No existing value for this prefix
+      this.set(prefix, [value, multiplicity])
+    } else {
+      // Existing ValueMap
+      const isEmpty = valueMapOrSingleValue.addValue(value, multiplicity)
+      if (isEmpty) {
+        this.delete(prefix)
+      }
+    }
+
+    return this.size === 0
+  }
+}
+
+// Third level map type for the index, stores single values or value maps against a hash.
+class ValueMap<TValue> extends Map<Hash, [TValue, number]> {
+  /**
+   * Add a value to the ValueMap. Returns true if the map becomes empty after the operation.
+   * @param value - The full value to store
+   * @param multiplicity - The multiplicity to add
+   * @param hashKey - Optional hash key to use instead of hashing the full value (used when in PrefixMap context)
+   */
+  addValue(value: TValue, multiplicity: number): boolean {
+    if (multiplicity === 0) return this.size === 0
+
+    const key = hash(value)
+    const currentValue = this.get(key)
+
+    if (currentValue) {
+      const [, currentMultiplicity] = currentValue
+      const newMultiplicity = currentMultiplicity + multiplicity
+      if (newMultiplicity === 0) {
+        this.delete(key)
+      } else {
+        this.set(key, [value, newMultiplicity])
+      }
+    } else {
+      this.set(key, [value, multiplicity])
+    }
+
+    return this.size === 0
+  }
+}
 
 /**
  * A map from a difference collection trace's keys -> (value, multiplicities) that changed.
  * Used in operations like join and reduce where the operation needs to
  * exploit the key-value structure of the data to run efficiently.
  */
-export class Index<K, V> {
+export class Index<TKey, TValue, TPrefix = any> {
   /*
-   * This is a hybrid Index that composes a ValueIndex and a HashIndex.
-   * Keys that have only one value are stored in the ValueIndex.
-   * Keys that have multiple values are stored in the HashIndex, the hash distinguishes between the values.
-   * This reduces the amount of hashes we need to compute since often times only a small portion of the keys are updated
-   * so we don't have to hash the keys that are never updated.
-   *
-   * Note: The `valueIndex` and `hashIndex` have disjoint keys.
-   *       When a key that has only one value gets a new distinct value,
-   *       it is added to the `hashIndex` and removed from the `valueIndex` and vice versa.
+   * This index maintains a nested map of keys -> (value, multiplicities), where:
+   * - initially the values are stored against the key as a single value tuple
+   * - when a key gets additional values, the values are stored against the key in a
+   *   prefix map
+   * - the prefix is extract where possible from values that are structured as
+   *   [rowPrimaryKey, rowValue], as they are in the Tanstack DB query pipeline.
+   * - only when there are multiple values for a given prefix do we fall back to a
+   *   hash to identify identical values, storing them in a third level value map.
    */
-  #valueIndex: ValueIndex<K, V>
-  #hashIndex: HashIndex<K, V>
+  #inner: IndexMap<TKey, TValue, TPrefix>
 
   constructor() {
-    this.#valueIndex = new ValueIndex<K, V>()
-    this.#hashIndex = new HashIndex<K, V>()
+    this.#inner = new Map()
   }
 
+  /**
+   * This method returns a string representation of the index.
+   * @param indent - Whether to indent the string representation.
+   * @returns A string representation of the index.
+   */
   toString(indent = false): string {
-    return `Index(\n  ${this.#valueIndex.toString(indent)},\n  ${this.#hashIndex.toString(indent)}\n)`
+    return `Index(${JSON.stringify(
+      [...this.entries()],
+      undefined,
+      indent ? 2 : undefined
+    )})`
   }
 
-  get(key: K): Array<[V, number]> {
-    if (this.#valueIndex.has(key)) {
-      return [this.#valueIndex.get(key)!]
-    }
-    return this.#hashIndex.get(key)
+  /**
+   * The size of the index.
+   */
+  get size(): number {
+    return this.#inner.size
+  }
+
+  /**
+   * This method checks if the index has a given key.
+   * @param key - The key to check.
+   * @returns True if the index has the key, false otherwise.
+   */
+  has(key: TKey): boolean {
+    return this.#inner.has(key)
   }
 
-  getMultiplicity(key: K, value: V): number {
-    if (this.#valueIndex.has(key)) {
-      return this.#valueIndex.getMultiplicity(key)
+  /**
+   * This method returns all values for a given key.
+   * @param key - The key to get the values for.
+   * @returns An array of value tuples [value, multiplicity].
+   */
+  get(key: TKey): Array<[TValue, number]> {
+    return [...this.getIterator(key)]
+  }
+
+  /**
+   * This method returns an iterator over all values for a given key.
+   * @param key - The key to get the values for.
+   * @returns An iterator of value tuples [value, multiplicity].
+   */
+  *getIterator(key: TKey): Iterable<[TValue, number]> {
+    const mapOrSingleValue = this.#inner.get(key)
+    if (isSingleValue(mapOrSingleValue)) {
+      yield mapOrSingleValue
+    } else if (mapOrSingleValue === undefined) {
+      return
+    } else if (mapOrSingleValue instanceof ValueMap) {
+      // Direct ValueMap - all values have NO_PREFIX
+      for (const valueTuple of mapOrSingleValue.values()) {
+        yield valueTuple
+      }
+    } else {
+      // PrefixMap - iterate through all prefixes
+      for (const singleValueOrValueMap of mapOrSingleValue.values()) {
+        if (isSingleValue(singleValueOrValueMap)) {
+          yield singleValueOrValueMap
+        } else {
+          for (const valueTuple of singleValueOrValueMap.values()) {
+            yield valueTuple
+          }
+        }
+      }
     }
-    return this.#hashIndex.getMultiplicity(key, value)
   }
 
   /**
    * This returns an iterator that iterates over all key-value pairs.
    * @returns An iterable of all key-value pairs (and their multiplicities) in the index.
    */
-  #entries(): Iterable<[K, [V, number]]> {
-    return concatIterable(
-      this.#valueIndex.entries(),
-      this.#hashIndex.entriesIterator()
-    )
+  *entries(): Iterable<[TKey, [TValue, number]]> {
+    for (const key of this.#inner.keys()) {
+      for (const valueTuple of this.getIterator(key)) {
+        yield [key, valueTuple]
+      }
+    }
   }
 
   /**
@@ -63,90 +241,156 @@ export class Index<K, V> {
    * It returns an iterator that you can use if you need to iterate over the values for a given key.
    * @returns An iterator of all *keys* in the index and their corresponding value iterator.
    */
-  *#entriesIterators(): Iterable<[K, Iterable<[V, number]>]> {
-    for (const [key, [value, multiplicity]] of this.#valueIndex.entries()) {
-      yield [key, new Map<V, number>([[value, multiplicity]])]
-    }
-    for (const [key, valueMap] of this.#hashIndex.entries()) {
-      yield [
-        key,
-        mapIterable(valueMap, ([_hash, [value, multiplicity]]) => [
-          value,
-          multiplicity,
-        ]),
-      ]
+  *entriesIterators(): Iterable<[TKey, Iterable<[TValue, number]>]> {
+    for (const key of this.#inner.keys()) {
+      yield [key, this.getIterator(key)]
     }
   }
 
-  has(key: K): boolean {
-    return this.#valueIndex.has(key) || this.#hashIndex.has(key)
-  }
+  /**
+   * This method adds a value to the index.
+   * @param key - The key to add the value to.
+   * @param valueTuple - The value tuple [value, multiplicity] to add to the index.
+   */
+  addValue(key: TKey, valueTuple: SingleValue<TValue>) {
+    const [value, multiplicity] = valueTuple
+    // If the multiplicity is 0, do nothing
+    if (multiplicity === 0) return
 
-  get size(): number {
-    return this.#valueIndex.size + this.#hashIndex.size
-  }
+    const mapOrSingleValue = this.#inner.get(key)
 
-  addValue(key: K, value: [V, number]): void {
-    const containedInValueIndex = this.#valueIndex.has(key)
-    const containedInHashIndex = this.#hashIndex.has(key)
+    if (mapOrSingleValue === undefined) {
+      // First value for this key
+      this.#inner.set(key, valueTuple)
+      return
+    }
 
-    if (containedInHashIndex && containedInValueIndex) {
-      throw new Error(
-        `Key ${key} is contained in both the value index and the hash index. This should never happen because they should have disjoint keysets.`
+    if (isSingleValue(mapOrSingleValue)) {
+      // Handle transition from single value to map
+      this.#handleSingleValueTransition(
+        key,
+        mapOrSingleValue,
+        value,
+        multiplicity
       )
+      return
     }
 
-    if (!containedInValueIndex && !containedInHashIndex) {
-      // This is the first time we see the key
-      // Add it to the value index
-      this.#valueIndex.addValue(key, value)
-      return
+    if (mapOrSingleValue instanceof ValueMap) {
+      // Handle existing ValueMap
+      const prefix = getPrefix<TValue, TPrefix>(value)
+      if (prefix !== NO_PREFIX) {
+        // Convert ValueMap to PrefixMap since we have a prefixed value
+        const prefixMap = new PrefixMap<TValue, TPrefix>()
+        prefixMap.set(NO_PREFIX, mapOrSingleValue)
+        prefixMap.set(prefix, valueTuple)
+        this.#inner.set(key, prefixMap)
+      } else {
+        // Add to existing ValueMap
+        const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
+        if (isEmpty) {
+          this.#inner.delete(key)
+        }
+      }
+    } else {
+      // Handle existing PrefixMap
+      const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
+      if (isEmpty) {
+        this.#inner.delete(key)
+      }
     }
+  }
 
-    if (containedInValueIndex) {
-      // This key is already in the value index
-      // It could be that it's the same value or a different one
-      // If it's a different value we will need to remove the key from the value index
-      // and add the key and its two values to the hash index
-      try {
-        this.#valueIndex.addValue(key, value)
-      } catch {
-        // This is a different value, need to move the key to the hash index
-        const existingValue = this.#valueIndex.get(key)!
-        this.#valueIndex.delete(key)
-        this.#hashIndex.addValue(key, existingValue)
-        this.#hashIndex.addValue(key, value)
+  /**
+   * Handle the transition from a single value to either a ValueMap or PrefixMap
+   */
+  #handleSingleValueTransition(
+    key: TKey,
+    currentSingleValue: SingleValue<TValue>,
+    newValue: TValue,
+    multiplicity: number
+  ) {
+    const [currentValue, currentMultiplicity] = currentSingleValue
+
+    // Check for exact same value (reference equality)
+    if (currentValue === newValue) {
+      const newMultiplicity = currentMultiplicity + multiplicity
+      if (newMultiplicity === 0) {
+        this.#inner.delete(key)
+      } else {
+        this.#inner.set(key, [newValue, newMultiplicity])
       }
       return
     }
 
-    if (containedInHashIndex) {
-      // This key is already in the hash index so it already has two or more values.
-      // However, this new value and multiplicity could cause an existing value to be removed
-      // and lead to the key having only a single value in which case we need to move it back to the value index
-      const singleRemainingValue = this.#hashIndex.addValue(key, value)
-      if (singleRemainingValue) {
-        // The key only has a single remaining value so we need to move it back to the value index
-        this.#hashIndex.delete(key)
-        this.#valueIndex.addValue(key, singleRemainingValue)
+    // Get prefixes for both values
+    const newPrefix = getPrefix<TValue, TPrefix>(newValue)
+    const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
+
+    // Check if they're the same value by prefix/suffix comparison
+    if (
+      currentPrefix === newPrefix &&
+      (currentValue === newValue || hash(currentValue) === hash(newValue))
+    ) {
+      const newMultiplicity = currentMultiplicity + multiplicity
+      if (newMultiplicity === 0) {
+        this.#inner.delete(key)
+      } else {
+        this.#inner.set(key, [newValue, newMultiplicity])
       }
       return
     }
+
+    // Different values - choose appropriate map type
+    if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) {
+      // Both have NO_PREFIX, use ValueMap directly
+      const valueMap = new ValueMap<TValue>()
+      valueMap.set(hash(currentValue), currentSingleValue)
+      valueMap.set(hash(newValue), [newValue, multiplicity])
+      this.#inner.set(key, valueMap)
+    } else {
+      // At least one has a prefix, use PrefixMap
+      const prefixMap = new PrefixMap<TValue, TPrefix>()
+
+      if (currentPrefix === newPrefix) {
+        // Same prefix, different suffixes - need ValueMap within PrefixMap
+        const valueMap = new ValueMap<TValue>()
+        valueMap.set(hash(currentValue), currentSingleValue)
+        valueMap.set(hash(newValue), [newValue, multiplicity])
+        prefixMap.set(currentPrefix, valueMap)
+      } else {
+        // Different prefixes - store as separate single values
+        prefixMap.set(currentPrefix, currentSingleValue)
+        prefixMap.set(newPrefix, [newValue, multiplicity])
+      }
+
+      this.#inner.set(key, prefixMap)
+    }
   }
 
-  append(other: Index<K, V>): void {
-    for (const [key, value] of other.#entries()) {
+  /**
+   * This method appends another index to the current index.
+   * @param other - The index to append to the current index.
+   */
+  append(other: Index<TKey, TValue>): void {
+    for (const [key, value] of other.entries()) {
       this.addValue(key, value)
     }
   }
 
-  join<V2>(other: Index<K, V2>): MultiSet<[K, [V, V2]]> {
-    const result: Array<[[K, [V, V2]], number]> = []
-
+  /**
+   * This method joins two indexes.
+   * @param other - The index to join with the current index.
+   * @returns A multiset of the joined values.
+   */
+  join<TValue2>(
+    other: Index<TKey, TValue2>
+  ): MultiSet<[TKey, [TValue, TValue2]]> {
+    const result: Array<[[TKey, [TValue, TValue2]], number]> = []
     // We want to iterate over the smaller of the two indexes to reduce the
     // number of operations we need to do.
     if (this.size <= other.size) {
-      for (const [key, valueIt] of this.#entriesIterators()) {
+      for (const [key, valueIt] of this.entriesIterators()) {
         if (!other.has(key)) continue
         const otherValues = other.get(key)
         for (const [val1, mul1] of valueIt) {
@@ -158,7 +402,7 @@ export class Index<K, V> {
         }
       }
     } else {
-      for (const [key, otherValueIt] of other.#entriesIterators()) {
+      for (const [key, otherValueIt] of other.entriesIterators()) {
         if (!this.has(key)) continue
         const values = this.get(key)
         for (const [val2, mul2] of otherValueIt) {
@@ -174,3 +418,34 @@ export class Index<K, V> {
     return new MultiSet(result)
   }
 }
+
+/**
+ * This function extracts the prefix from a value.
+ * @param value - The value to extract the prefix from.
+ * @returns The prefix and the suffix.
+ */
+function getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {
+  // If the value is an array and the first element is a string or number, then the
+  // first element is the prefix. This is used to distinguish between values without
+  // the need for hashing unless there are multiple values for the same prefix.
+  if (
+    Array.isArray(value) &&
+    (typeof value[0] === `string` ||
+      typeof value[0] === `number` ||
+      typeof value[0] === `bigint`)
+  ) {
+    return value[0] as TPrefix
+  }
+  return NO_PREFIX
+}
+
+/**
+ * This function checks if a value is a single value.
+ * @param value - The value to check.
+ * @returns True if the value is a single value, false otherwise.
+ */
+function isSingleValue<TValue>(
+  value: SingleValue<TValue> | unknown
+): value is SingleValue<TValue> {
+  return Array.isArray(value)
+}

package/src/operators/distinct.ts

@@ -4,21 +4,25 @@ import { hash } from "../hashing/index.js"
 import { MultiSet } from "../multiset.js"
 import type { Hash } from "../hashing/index.js"
 import type { DifferenceStreamReader } from "../graph.js"
-import type { IStreamBuilder } from "../types.js"
+import type { IStreamBuilder, KeyValue } from "../types.js"
 
 type Multiplicity = number
 
+type GetValue<T> = T extends KeyValue<any, infer V> ? V : never
+
 /**
  * Operator that removes duplicates
  */
-export class DistinctOperator<T> extends UnaryOperator<T> {
+export class DistinctOperator<
+  T extends KeyValue<any, any>,
+> extends UnaryOperator<T, KeyValue<number, GetValue<T>>> {
   #by: (value: T) => any
   #values: Map<Hash, Multiplicity> // keeps track of the number of times each value has been seen
 
   constructor(
     id: number,
     input: DifferenceStreamReader<T>,
-    output: DifferenceStreamWriter<T>,
+    output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,
     by: (value: T) => any = (value: T) => value
   ) {
     super(id, input, output)
@@ -39,12 +43,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
           this.#values.get(hashedValue) ??
           0
         const newMultiplicity = oldMultiplicity + diff
-
         updatedValues.set(hashedValue, [newMultiplicity, value])
       }
     }
 
-    const result: Array<[T, number]> = []
+    const result: Array<[KeyValue<number, GetValue<T>>, number]> = []
 
     // Check which values became visible or disappeared
     for (const [
@@ -62,11 +65,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
       if (oldMultiplicity <= 0 && newMultiplicity > 0) {
         // The value wasn't present in the stream
         // but with this change it is now present in the stream
-        result.push([value, 1])
+        result.push([[hash(this.#by(value)), value[1]], 1])
       } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
         // The value was present in the stream
         // but with this change it is no longer present in the stream
-        result.push([value, -1])
+        result.push([[hash(this.#by(value)), value[1]], -1])
       }
     }
 
@@ -79,7 +82,9 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
 /**
  * Removes duplicate values
  */
-export function distinct<T>(by: (value: T) => any = (value: T) => value) {
+export function distinct<T extends KeyValue<any, any>>(
+  by: (value: T) => any = (value: T) => value
+) {
   return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
     const output = new StreamBuilder<T>(
       stream.graph,