@tanstack/db-ivm 0.1.26 → 0.2.0

package/dist/cjs/indexes.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"indexes.cjs","sources":["../../src/indexes.ts"],"sourcesContent":["/**\n * # Optimized Index Data Structure\n *\n * Multi-level index that adapts storage strategy based on data patterns to minimize memory\n * usage, eliminate wasteful lookups, and avoid hashing whenever possible.\n *\n * ## Storage Strategy\n *\n * **Single value**: `IndexMap['key'] → [value, multiplicity]` (no hashing needed)\n *\n * **Multiple unprefixed values**: Direct ValueMap (avoids NO_PREFIX lookup)\n * ```\n * IndexMap['key'] → ValueMap { hash(value1) → [value1, mult1], ... }\n * ```\n *\n * **Values with prefixes**: PrefixMap uses prefix keys directly (no hashing)\n * ```\n * IndexMap['key'] → PrefixMap { 'prefix1' → [value1, mult1], NO_PREFIX → ValueMap{...} }\n * ```\n *\n * **Multiple values per prefix**: ValueMap within PrefixMap (hash only suffixes)\n * ```\n * PrefixMap['prefix'] → ValueMap { hash(suffix1) → [full_value1, mult1], ... }\n * ```\n *\n * ## Dynamic Evolution\n *\n * Structure automatically evolves as data is added:\n * - Single → ValueMap (when both values unprefixed)\n * - Single → PrefixMap (when at least one prefixed)\n * - ValueMap → PrefixMap (adding prefixed value to unprefixed)\n *\n * Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'\n */\n\nimport { MultiSet } from './multiset.js'\nimport { hash } from './hashing/index.js'\nimport type { Hash } from './hashing/index.js'\n\n// We use a symbol to represent the absence of a prefix, unprefixed values a stored\n// against this key.\nconst NO_PREFIX = Symbol(`NO_PREFIX`)\ntype NO_PREFIX = typeof NO_PREFIX\n\n// A single value is a tuple of the value and the multiplicity.\ntype SingleValue<TValue> = [TValue, number]\n\n// Base map type for the index. Stores single values, prefix maps, or value maps against a key.\ntype IndexMap<TKey, TValue, TPrefix> = Map<\n  TKey,\n  SingleValue<TValue> | PrefixMap<TValue, TPrefix> | ValueMap<TValue>\n>\n\n// Second level map type for the index, stores single values or value maps against a prefix.\nclass PrefixMap<TValue, TPrefix> extends Map<\n  TPrefix | NO_PREFIX,\n  SingleValue<TValue> | ValueMap<TValue>\n> {\n  /**\n   * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.\n   */\n  addValue(value: TValue, multiplicity: number): boolean {\n    if (multiplicity === 0) return this.size === 0\n\n    const prefix = getPrefix<TValue, TPrefix>(value)\n    const valueMapOrSingleValue = this.get(prefix)\n\n    if (isSingleValue(valueMapOrSingleValue)) {\n      const [currentValue, currentMultiplicity] = valueMapOrSingleValue\n      const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)\n\n      if (!isSamePrefix(currentPrefix, prefix)) {\n        throw new Error(`Mismatching prefixes, this should never happen`)\n      }\n\n      if (currentValue === value || hash(currentValue) === hash(value)) {\n        // Same value, update multiplicity\n        const newMultiplicity = currentMultiplicity + multiplicity\n        if (newMultiplicity === 0) {\n          this.delete(prefix)\n        } else {\n          this.set(prefix, [value, newMultiplicity])\n        }\n      } else {\n        // Different suffixes, need to create ValueMap\n        const valueMap = new ValueMap<TValue>()\n        valueMap.set(hash(currentValue), valueMapOrSingleValue)\n        valueMap.set(hash(value), [value, multiplicity])\n        this.set(prefix, valueMap)\n      }\n    } else if (valueMapOrSingleValue === undefined) {\n      // No existing value for this prefix\n      this.set(prefix, [value, multiplicity])\n    } else {\n      // Existing ValueMap\n      const isEmpty = valueMapOrSingleValue.addValue(value, multiplicity)\n      if (isEmpty) {\n        this.delete(prefix)\n      }\n    }\n\n    return this.size === 0\n  }\n}\n\n// Third level map type for the index, stores single values or value maps against a hash.\nclass ValueMap<TValue> extends Map<Hash, [TValue, number]> {\n  /**\n   * Add a value to the ValueMap. Returns true if the map becomes empty after the operation.\n   * @param value - The full value to store\n   * @param multiplicity - The multiplicity to add\n   * @param hashKey - Optional hash key to use instead of hashing the full value (used when in PrefixMap context)\n   */\n  addValue(value: TValue, multiplicity: number): boolean {\n    if (multiplicity === 0) return this.size === 0\n\n    const key = hash(value)\n    const currentValue = this.get(key)\n\n    if (currentValue) {\n      const [, currentMultiplicity] = currentValue\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.delete(key)\n      } else {\n        this.set(key, [value, newMultiplicity])\n      }\n    } else {\n      this.set(key, [value, multiplicity])\n    }\n\n    return this.size === 0\n  }\n}\n\n/**\n * A map from a difference collection trace's keys -> (value, multiplicities) that changed.\n * Used in operations like join and reduce where the operation needs to\n * exploit the key-value structure of the data to run efficiently.\n */\nexport class Index<TKey, TValue, TPrefix = any> {\n  /*\n   * This index maintains a nested map of keys -> (value, multiplicities), where:\n   * - initially the values are stored against the key as a single value tuple\n   * - when a key gets additional values, the values are stored against the key in a\n   *   prefix map\n   * - the prefix is extract where possible from values that are structured as\n   *   [rowPrimaryKey, rowValue], as they are in the Tanstack DB query pipeline.\n   * - only when there are multiple values for a given prefix do we fall back to a\n   *   hash to identify identical values, storing them in a third level value map.\n   */\n  #inner: IndexMap<TKey, TValue, TPrefix>\n  #consolidatedMultiplicity: Map<TKey, number> = new Map() // sum of multiplicities per key\n\n  constructor() {\n    this.#inner = new Map()\n  }\n\n  /**\n   * Create an Index from multiple MultiSet messages.\n   * @param messages - Array of MultiSet messages to build the index from.\n   * @returns A new Index containing all the data from the messages.\n   */\n  static fromMultiSets<K, V>(messages: Array<MultiSet<[K, V]>>): Index<K, V> {\n    const index = new Index<K, V>()\n\n    for (const message of messages) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        index.addValue(key, [value, multiplicity])\n      }\n    }\n\n    return index\n  }\n\n  /**\n   * This method returns a string representation of the index.\n   * @param indent - Whether to indent the string representation.\n   * @returns A string representation of the index.\n   */\n  toString(indent = false): string {\n    return `Index(${JSON.stringify(\n      [...this.entries()],\n      undefined,\n      indent ? 2 : undefined,\n    )})`\n  }\n\n  /**\n   * The size of the index.\n   */\n  get size(): number {\n    return this.#inner.size\n  }\n\n  /**\n   * This method checks if the index has a given key.\n   * @param key - The key to check.\n   * @returns True if the index has the key, false otherwise.\n   */\n  has(key: TKey): boolean {\n    return this.#inner.has(key)\n  }\n\n  /**\n   * Check if a key has presence (non-zero consolidated multiplicity).\n   * @param key - The key to check.\n   * @returns True if the key has non-zero consolidated multiplicity, false otherwise.\n   */\n  hasPresence(key: TKey): boolean {\n    return (this.#consolidatedMultiplicity.get(key) || 0) !== 0\n  }\n\n  /**\n   * Get the consolidated multiplicity (sum of multiplicities) for a key.\n   * @param key - The key to get the consolidated multiplicity for.\n   * @returns The consolidated multiplicity for the key.\n   */\n  getConsolidatedMultiplicity(key: TKey): number {\n    return this.#consolidatedMultiplicity.get(key) || 0\n  }\n\n  /**\n   * Get all keys that have presence (non-zero consolidated multiplicity).\n   * @returns An iterator of keys with non-zero consolidated multiplicity.\n   */\n  getPresenceKeys(): Iterable<TKey> {\n    return this.#consolidatedMultiplicity.keys()\n  }\n\n  /**\n   * This method returns all values for a given key.\n   * @param key - The key to get the values for.\n   * @returns An array of value tuples [value, multiplicity].\n   */\n  get(key: TKey): Array<[TValue, number]> {\n    return [...this.getIterator(key)]\n  }\n\n  /**\n   * This method returns an iterator over all values for a given key.\n   * @param key - The key to get the values for.\n   * @returns An iterator of value tuples [value, multiplicity].\n   */\n  *getIterator(key: TKey): Iterable<[TValue, number]> {\n    const mapOrSingleValue = this.#inner.get(key)\n    if (isSingleValue(mapOrSingleValue)) {\n      yield mapOrSingleValue\n    } else if (mapOrSingleValue === undefined) {\n      return\n    } else if (mapOrSingleValue instanceof ValueMap) {\n      // Direct ValueMap - all values have NO_PREFIX\n      for (const valueTuple of mapOrSingleValue.values()) {\n        yield valueTuple\n      }\n    } else {\n      // PrefixMap - iterate through all prefixes\n      for (const singleValueOrValueMap of mapOrSingleValue.values()) {\n        if (isSingleValue(singleValueOrValueMap)) {\n          yield singleValueOrValueMap\n        } else {\n          for (const valueTuple of singleValueOrValueMap.values()) {\n            yield valueTuple\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * This returns an iterator that iterates over all key-value pairs.\n   * @returns An iterable of all key-value pairs (and their multiplicities) in the index.\n   */\n  *entries(): Iterable<[TKey, [TValue, number]]> {\n    for (const key of this.#inner.keys()) {\n      for (const valueTuple of this.getIterator(key)) {\n        yield [key, valueTuple]\n      }\n    }\n  }\n\n  /**\n   * This method only iterates over the keys and not over the values.\n   * Hence, it is more efficient than the `#entries` method.\n   * It returns an iterator that you can use if you need to iterate over the values for a given key.\n   * @returns An iterator of all *keys* in the index and their corresponding value iterator.\n   */\n  *entriesIterators(): Iterable<[TKey, Iterable<[TValue, number]>]> {\n    for (const key of this.#inner.keys()) {\n      yield [key, this.getIterator(key)]\n    }\n  }\n\n  /**\n   * This method adds a value to the index.\n   * @param key - The key to add the value to.\n   * @param valueTuple - The value tuple [value, multiplicity] to add to the index.\n   */\n  addValue(key: TKey, valueTuple: SingleValue<TValue>) {\n    const [value, multiplicity] = valueTuple\n    // If the multiplicity is 0, do nothing\n    if (multiplicity === 0) return\n\n    // Update consolidated multiplicity tracking\n    const newConsolidatedMultiplicity =\n      (this.#consolidatedMultiplicity.get(key) || 0) + multiplicity\n    if (newConsolidatedMultiplicity === 0) {\n      this.#consolidatedMultiplicity.delete(key)\n    } else {\n      this.#consolidatedMultiplicity.set(key, newConsolidatedMultiplicity)\n    }\n\n    const mapOrSingleValue = this.#inner.get(key)\n\n    if (mapOrSingleValue === undefined) {\n      // First value for this key\n      this.#inner.set(key, valueTuple)\n      return\n    }\n\n    if (isSingleValue(mapOrSingleValue)) {\n      // Handle transition from single value to map\n      this.#handleSingleValueTransition(\n        key,\n        mapOrSingleValue,\n        value,\n        multiplicity,\n      )\n      return\n    }\n\n    if (mapOrSingleValue instanceof ValueMap) {\n      // Handle existing ValueMap\n      const prefix = getPrefix<TValue, TPrefix>(value)\n      if (prefix !== NO_PREFIX) {\n        // Convert ValueMap to PrefixMap since we have a prefixed value\n        const prefixMap = new PrefixMap<TValue, TPrefix>()\n        prefixMap.set(NO_PREFIX, mapOrSingleValue)\n        prefixMap.set(prefix, valueTuple)\n        this.#inner.set(key, prefixMap)\n      } else {\n        // Add to existing ValueMap\n        const isEmpty = mapOrSingleValue.addValue(value, multiplicity)\n        if (isEmpty) {\n          this.#inner.delete(key)\n        }\n      }\n    } else {\n      // Handle existing PrefixMap\n      const isEmpty = mapOrSingleValue.addValue(value, multiplicity)\n      if (isEmpty) {\n        this.#inner.delete(key)\n      }\n    }\n  }\n\n  /**\n   * Handle the transition from a single value to either a ValueMap or PrefixMap\n   */\n  #handleSingleValueTransition(\n    key: TKey,\n    currentSingleValue: SingleValue<TValue>,\n    newValue: TValue,\n    multiplicity: number,\n  ) {\n    const [currentValue, currentMultiplicity] = currentSingleValue\n\n    // Check for exact same value (reference equality)\n    if (currentValue === newValue) {\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.#inner.delete(key)\n      } else {\n        this.#inner.set(key, [newValue, newMultiplicity])\n      }\n      return\n    }\n\n    // Get prefixes for both values\n    const newPrefix = getPrefix<TValue, TPrefix>(newValue)\n    const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)\n\n    // Check if they're the same value by prefix/suffix comparison\n    if (\n      isSamePrefix(currentPrefix, newPrefix) &&\n      (currentValue === newValue || hash(currentValue) === hash(newValue))\n    ) {\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.#inner.delete(key)\n      } else {\n        this.#inner.set(key, [newValue, newMultiplicity])\n      }\n      return\n    }\n\n    // Different values - choose appropriate map type\n    if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) {\n      // Both have NO_PREFIX, use ValueMap directly\n      const valueMap = new ValueMap<TValue>()\n      valueMap.set(hash(currentValue), currentSingleValue)\n      valueMap.set(hash(newValue), [newValue, multiplicity])\n      this.#inner.set(key, valueMap)\n    } else {\n      // At least one has a prefix, use PrefixMap\n      const prefixMap = new PrefixMap<TValue, TPrefix>()\n\n      if (isSamePrefix(currentPrefix, newPrefix)) {\n        // Same prefix, different suffixes - need ValueMap within PrefixMap\n        const valueMap = new ValueMap<TValue>()\n        valueMap.set(hash(currentValue), currentSingleValue)\n        valueMap.set(hash(newValue), [newValue, multiplicity])\n        prefixMap.set(currentPrefix, valueMap)\n      } else {\n        // Different prefixes - store as separate single values\n        prefixMap.set(currentPrefix, currentSingleValue)\n        prefixMap.set(newPrefix, [newValue, multiplicity])\n      }\n\n      this.#inner.set(key, prefixMap)\n    }\n  }\n\n  /**\n   * This method appends another index to the current index.\n   * @param other - The index to append to the current index.\n   */\n  append(other: Index<TKey, TValue>): void {\n    for (const [key, value] of other.entries()) {\n      this.addValue(key, value)\n    }\n  }\n\n  /**\n   * This method joins two indexes.\n   * @param other - The index to join with the current index.\n   * @returns A multiset of the joined values.\n   */\n  join<TValue2>(\n    other: Index<TKey, TValue2>,\n  ): MultiSet<[TKey, [TValue, TValue2]]> {\n    const result: Array<[[TKey, [TValue, TValue2]], number]> = []\n    // We want to iterate over the smaller of the two indexes to reduce the\n    // number of operations we need to do.\n    if (this.size <= other.size) {\n      for (const [key, valueIt] of this.entriesIterators()) {\n        if (!other.has(key)) continue\n        const otherValues = other.get(key)\n        for (const [val1, mul1] of valueIt) {\n          for (const [val2, mul2] of otherValues) {\n            if (mul1 !== 0 && mul2 !== 0) {\n              result.push([[key, [val1, val2]], mul1 * mul2])\n            }\n          }\n        }\n      }\n    } else {\n      for (const [key, otherValueIt] of other.entriesIterators()) {\n        if (!this.has(key)) continue\n        const values = this.get(key)\n        for (const [val2, mul2] of otherValueIt) {\n          for (const [val1, mul1] of values) {\n            if (mul1 !== 0 && mul2 !== 0) {\n              result.push([[key, [val1, val2]], mul1 * mul2])\n            }\n          }\n        }\n      }\n    }\n\n    return new MultiSet(result)\n  }\n}\n\n/**\n * This function extracts the prefix from a value.\n * @param value - The value to extract the prefix from.\n * @returns The prefix and the suffix.\n */\n// Prefixes are Map keys, so they compare as a Map does: NaN equals NaN.\nfunction isSamePrefix(a: unknown, b: unknown): boolean {\n  return a === b || (Number.isNaN(a) && Number.isNaN(b))\n}\n\nfunction getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {\n  // If the value is an array and the first element is a string or number, then the\n  // first element is the prefix. This is used to distinguish between values without\n  // the need for hashing unless there are multiple values for the same prefix.\n  if (\n    Array.isArray(value) &&\n    (typeof value[0] === `string` ||\n      typeof value[0] === `number` ||\n      typeof value[0] === `bigint`)\n  ) {\n    return value[0] as TPrefix\n  }\n  return NO_PREFIX\n}\n\n/**\n * This function checks if a value is a single value.\n * @param value - The value to check.\n * @returns True if the value is a single value, false otherwise.\n */\nfunction isSingleValue<TValue>(\n  value: SingleValue<TValue> | unknown,\n): value is SingleValue<TValue> {\n  return Array.isArray(value)\n}\n"],"names":["hash","MultiSet"],"mappings":";;;;AAyCA,MAAM,mCAAmB,WAAW;AAapC,MAAM,kBAAmC,IAGvC;AAAA;AAAA;AAAA;AAAA,EAIA,SAAS,OAAe,cAA+B;AACrD,QAAI,iBAAiB,EAAG,QAAO,KAAK,SAAS;AAE7C,UAAM,SAAS,UAA2B,KAAK;AAC/C,UAAM,wBAAwB,KAAK,IAAI,MAAM;AAE7C,QAAI,cAAc,qBAAqB,GAAG;AACxC,YAAM,CAAC,cAAc,mBAAmB,IAAI;AAC5C,YAAM,gBAAgB,UAA2B,YAAY;AAE7D,UAAI,CAAC,aAAa,eAAe,MAAM,GAAG;AACxC,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAEA,UAAI,iBAAiB,SAASA,KAAAA,KAAK,YAAY,MAAMA,KAAAA,KAAK,KAAK,GAAG;AAEhE,cAAM,kBAAkB,sBAAsB;AAC9C,YAAI,oBAAoB,GAAG;AACzB,eAAK,OAAO,MAAM;AAAA,QACpB,OAAO;AACL,eAAK,IAAI,QAAQ,CAAC,OAAO,eAAe,CAAC;AAAA,QAC3C;AAAA,MACF,OAAO;AAEL,cAAM,WAAW,IAAI,SAAA;AACrB,iBAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,qBAAqB;AACtD,iBAAS,IAAIA,UAAK,KAAK,GAAG,CAAC,OAAO,YAAY,CAAC;AAC/C,aAAK,IAAI,QAAQ,QAAQ;AAAA,MAC3B;AAAA,IACF,WAAW,0BAA0B,QAAW;AAE9C,WAAK,IAAI,QAAQ,CAAC,OAAO,YAAY,CAAC;AAAA,IACxC,OAAO;AAEL,YAAM,UAAU,sBAAsB,SAAS,OAAO,YAAY;AAClE,UAAI,SAAS;AACX,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAEA,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;AAGA,MAAM,iBAAyB,IAA4B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOzD,SAAS,OAAe,cAA+B;AACrD,QAAI,iBAAiB,EAAG,QAAO,KAAK,SAAS;AAE7C,UAAM,MAAMA,KAAAA,KAAK,KAAK;AACtB,UAAM,eAAe,KAAK,IAAI,GAAG;AAEjC,QAAI,cAAc;AAChB,YAAM,CAAA,EAAG,mBAAmB,IAAI;AAChC,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,GAAG;AAAA,MACjB,OAAO;AACL,aAAK,IAAI,KAAK,CAAC,OAAO,eAAe,CAAC;AAAA,MACxC;AAAA,IACF,OAAO;AACL,WAAK,IAAI,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,IACrC;AAEA,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;AAOO,MAAM,MAAmC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAW9C;AAAA,EACA,gDAAmD,IAAA;AAAA;AAAA,EAEnD,cAAc;AACZ,SAAK,6BAAa,IAAA;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,cAAoB,UAAgD;AACzE,UAAM,QAAQ,IAAI,MAAA;AAElB,eAAW,WAAW,UAAU;AAC9B,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,cAAM,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,MAC3C;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,OAAe;AAC/B,WAAO,SAAS,KAAK;AAAA,MACnB,CAAC,GAAG,KAAK,SAAS;AAAA,MAClB;AAAA,MACA,SAAS,IAAI;AAAA,IAAA,CACd;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,OAAe;AACjB,WAAO,KAAK,OAAO;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,KAAoB;AACtB,WAAO,KAAK,OAAO,IAAI,GAAG;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,YAAY,KAAoB;AAC9B,YAAQ,KAAK,0BAA0B,IAAI,GAAG,KAAK,OAAO;AAAA,EAC5D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,4BAA4B,KAAmB;AAC7C,WAAO,KAAK,0BAA0B,IAAI,GAAG,KAAK;AAAA,EACpD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkC;AAChC,WAAO,KAAK,0BAA0B,KAAA;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,KAAoC;AACtC,WAAO,CAAC,GAAG,KAAK,YAAY,GAAG,CAAC;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,CAAC,YAAY,KAAuC;AAClD,UAAM,mBAAmB,KAAK,OAAO,IAAI,GAAG;AAC5C,QAAI,cAAc,gBAAgB,GAAG;AACnC,YAAM;AAAA,IACR,WAAW,qBAAqB,QAAW;AACzC;AAAA,IACF,WAAW,4BAA4B,UAAU;AAE/C,iBAAW,cAAc,iBAAiB,UAAU;AAClD,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AAEL,iBAAW,yBAAyB,iBAAiB,UAAU;AAC7D,YAAI,cAAc,qBAAqB,GAAG;AACxC,gBAAM;AAAA,QACR,OAAO;AACL,qBAAW,cAAc,sBAAsB,UAAU;AACvD,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,UAA8C;AAC7C,eAAW,OAAO,KAAK,OAAO,KAAA,GAAQ;AACpC,iBAAW,cAAc,KAAK,YAAY,GAAG,GAAG;AAC9C,cAAM,CAAC,KAAK,UAAU;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,CAAC,mBAAiE;AAChE,eAAW,OAAO,KAAK,OAAO,KAAA,GAAQ;AACpC,YAAM,CAAC,KAAK,KAAK,YAAY,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,KAAW,YAAiC;AACnD,UAAM,CAAC,OAAO,YAAY,IAAI;AAE9B,QAAI,iBAAiB,EAAG;AAGxB,UAAM,+BACH,KAAK,0BAA0B,IAAI,GAAG,KAAK,KAAK;AACnD,QAAI,gCAAgC,GAAG;AACrC,WAAK,0BAA0B,OAAO,GAAG;AAAA,IAC3C,OAAO;AACL,WAAK,0BAA0B,IAAI,KAAK,2BAA2B;AAAA,IACrE;AAEA,UAAM,mBAAmB,KAAK,OAAO,IAAI,GAAG;AAE5C,QAAI,qBAAqB,QAAW;AAElC,WAAK,OAAO,IAAI,KAAK,UAAU;AAC/B;AAAA,IACF;AAEA,QAAI,cAAc,gBAAgB,GAAG;AAEnC,WAAK;AAAA,QACH;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF;AAAA,IACF;AAEA,QAAI,4BAA4B,UAAU;AAExC,YAAM,SAAS,UAA2B,KAAK;AAC/C,UAAI,WAAW,WAAW;AAExB,cAAM,YAAY,IAAI,UAAA;AACtB,kBAAU,IAAI,WAAW,gBAAgB;AACzC,kBAAU,IAAI,QAAQ,UAAU;AAChC,aAAK,OAAO,IAAI,KAAK,SAAS;AAAA,MAChC,OAAO;AAEL,cAAM,UAAU,iBAAiB,SAAS,OAAO,YAAY;AAC7D,YAAI,SAAS;AACX,eAAK,OAAO,OAAO,GAAG;AAAA,QACxB;AAAA,MACF;AAAA,IACF,OAAO;AAEL,YAAM,UAAU,iBAAiB,SAAS,OAAO,YAAY;AAC7D,UAAI,SAAS;AACX,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,6BACE,KACA,oBACA,UACA,cACA;AACA,UAAM,CAAC,cAAc,mBAAmB,IAAI;AAG5C,QAAI,iBAAiB,UAAU;AAC7B,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB,OAAO;AACL,aAAK,OAAO,IAAI,KAAK,CAAC,UAAU,eAAe,CAAC;AAAA,MAClD;AACA;AAAA,IACF;AAGA,UAAM,YAAY,UAA2B,QAAQ;AACrD,UAAM,gBAAgB,UAA2B,YAAY;AAG7D,QACE,aAAa,eAAe,SAAS,MACpC,iBAAiB,YAAYA,KAAAA,KAAK,YAAY,MAAMA,UAAK,QAAQ,IAClE;AACA,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB,OAAO;AACL,aAAK,OAAO,IAAI,KAAK,CAAC,UAAU,eAAe,CAAC;AAAA,MAClD;AACA;AAAA,IACF;AAGA,QAAI,kBAAkB,aAAa,cAAc,WAAW;AAE1D,YAAM,WAAW,IAAI,SAAA;AACrB,eAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,kBAAkB;AACnD,eAAS,IAAIA,UAAK,QAAQ,GAAG,CAAC,UAAU,YAAY,CAAC;AACrD,WAAK,OAAO,IAAI,KAAK,QAAQ;AAAA,IAC/B,OAAO;AAEL,YAAM,YAAY,IAAI,UAAA;AAEtB,UAAI,aAAa,eAAe,SAAS,GAAG;AAE1C,cAAM,WAAW,IAAI,SAAA;AACrB,iBAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,kBAAkB;AACnD,iBAAS,IAAIA,UAAK,QAAQ,GAAG,CAAC,UAAU,YAAY,CAAC;AACrD,kBAAU,IAAI,eAAe,QAAQ;AAAA,MACvC,OAAO;AAEL,kBAAU,IAAI,eAAe,kBAAkB;AAC/C,kBAAU,IAAI,WAAW,CAAC,UAAU,YAAY,CAAC;AAAA,MACnD;AAEA,WAAK,OAAO,IAAI,KAAK,SAAS;AAAA,IAChC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,OAAkC;AACvC,eAAW,CAAC,KAAK,KAAK,KAAK,MAAM,WAAW;AAC1C,WAAK,SAAS,KAAK,KAAK;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KACE,OACqC;AACrC,UAAM,SAAqD,CAAA;AAG3D,QAAI,KAAK,QAAQ,MAAM,MAAM;AAC3B,iBAAW,CAAC,KAAK,OAAO,KAAK,KAAK,oBAAoB;AACpD,YAAI,CAAC,MAAM,IAAI,GAAG,EAAG;AACrB,cAAM,cAAc,MAAM,IAAI,GAAG;AACjC,mBAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,qBAAW,CAAC,MAAM,IAAI,KAAK,aAAa;AACtC,gBAAI,SAAS,KAAK,SAAS,GAAG;AAC5B,qBAAO,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,OAAO;AACL,iBAAW,CAAC,KAAK,YAAY,KAAK,MAAM,oBAAoB;AAC1D,YAAI,CAAC,KAAK,IAAI,GAAG,EAAG;AACpB,cAAM,SAAS,KAAK,IAAI,GAAG;AAC3B,mBAAW,CAAC,MAAM,IAAI,KAAK,cAAc;AACvC,qBAAW,CAAC,MAAM,IAAI,KAAK,QAAQ;AACjC,gBAAI,SAAS,KAAK,SAAS,GAAG;AAC5B,qBAAO,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAIC,SAAAA,SAAS,MAAM;AAAA,EAC5B;AACF;AAQA,SAAS,aAAa,GAAY,GAAqB;AACrD,SAAO,MAAM,KAAM,OAAO,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC;AACtD;AAEA,SAAS,UAA2B,OAAoC;AAItE,MACE,MAAM,QAAQ,KAAK,MAClB,OAAO,MAAM,CAAC,MAAM,YACnB,OAAO,MAAM,CAAC,MAAM,YACpB,OAAO,MAAM,CAAC,MAAM,WACtB;AACA,WAAO,MAAM,CAAC;AAAA,EAChB;AACA,SAAO;AACT;AAOA,SAAS,cACP,OAC8B;AAC9B,SAAO,MAAM,QAAQ,KAAK;AAC5B;;"}
+{"version":3,"file":"indexes.cjs","sources":["../../src/indexes.ts"],"sourcesContent":["/**\n * # Optimized Index Data Structure\n *\n * Multi-level index that adapts storage strategy based on data patterns to minimize memory\n * usage, eliminate wasteful lookups, and avoid hashing whenever possible.\n *\n * ## Storage Strategy\n *\n * **Single value**: `IndexMap['key'] → [value, multiplicity]` (no hashing needed)\n *\n * **Multiple unprefixed values**: Direct ValueMap (avoids NO_PREFIX lookup)\n * ```\n * IndexMap['key'] → ValueMap { hash(value1) → [value1, mult1], ... }\n * ```\n *\n * **Values with prefixes**: PrefixMap uses prefix keys directly (no hashing)\n * ```\n * IndexMap['key'] → PrefixMap { 'prefix1' → [value1, mult1], NO_PREFIX → ValueMap{...} }\n * ```\n *\n * **Multiple values per prefix**: ValueMap within PrefixMap (hash only suffixes)\n * ```\n * PrefixMap['prefix'] → ValueMap { hash(suffix1) → [full_value1, mult1], ... }\n * ```\n *\n * ## Dynamic Evolution\n *\n * Structure automatically evolves as data is added:\n * - Single → ValueMap (when both values unprefixed)\n * - Single → PrefixMap (when at least one prefixed)\n * - ValueMap → PrefixMap (adding prefixed value to unprefixed)\n *\n * Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'\n */\n\nimport { MultiSet } from './multiset.js'\nimport { hash } from './hashing/index.js'\nimport type { Hash } from './hashing/index.js'\n\n// We use a symbol to represent the absence of a prefix, unprefixed values a stored\n// against this key.\nconst NO_PREFIX = Symbol(`NO_PREFIX`)\ntype NO_PREFIX = typeof NO_PREFIX\n\n// A single value is a tuple of the value and the multiplicity.\ntype SingleValue<TValue> = [TValue, number]\n\n// Base map type for the index. Stores single values, prefix maps, or value maps against a key.\ntype IndexMap<TKey, TValue, TPrefix> = Map<\n  TKey,\n  SingleValue<TValue> | PrefixMap<TValue, TPrefix> | ValueMap<TValue>\n>\n\n// Second level map type for the index, stores single values or value maps against a prefix.\nclass PrefixMap<TValue, TPrefix> extends Map<\n  TPrefix | NO_PREFIX,\n  SingleValue<TValue> | ValueMap<TValue>\n> {\n  /**\n   * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.\n   * With `prefixIdentity`, values that share a prefix are the same value.\n   */\n  addValue(\n    value: TValue,\n    multiplicity: number,\n    prefixIdentity = false,\n  ): boolean {\n    if (multiplicity === 0) return this.size === 0\n\n    const prefix = getPrefix<TValue, TPrefix>(value)\n    const valueMapOrSingleValue = this.get(prefix)\n\n    if (isSingleValue(valueMapOrSingleValue)) {\n      const [currentValue, currentMultiplicity] = valueMapOrSingleValue\n      const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)\n\n      if (!isSamePrefix(currentPrefix, prefix)) {\n        throw new Error(`Mismatching prefixes, this should never happen`)\n      }\n\n      if (\n        currentValue === value ||\n        (prefixIdentity && prefix !== NO_PREFIX) ||\n        hash(currentValue) === hash(value)\n      ) {\n        // Same value, update multiplicity\n        const newMultiplicity = currentMultiplicity + multiplicity\n        if (newMultiplicity === 0) {\n          this.delete(prefix)\n        } else {\n          this.set(prefix, [value, newMultiplicity])\n        }\n      } else {\n        // Different suffixes, need to create ValueMap\n        const valueMap = new ValueMap<TValue>()\n        valueMap.set(hash(currentValue), valueMapOrSingleValue)\n        valueMap.set(hash(value), [value, multiplicity])\n        this.set(prefix, valueMap)\n      }\n    } else if (valueMapOrSingleValue === undefined) {\n      // No existing value for this prefix\n      this.set(prefix, [value, multiplicity])\n    } else {\n      // Existing ValueMap\n      const isEmpty = valueMapOrSingleValue.addValue(value, multiplicity)\n      if (isEmpty) {\n        this.delete(prefix)\n      }\n    }\n\n    return this.size === 0\n  }\n}\n\n// Third level map type for the index, stores single values or value maps against a hash.\nclass ValueMap<TValue> extends Map<Hash, [TValue, number]> {\n  /**\n   * Add a value to the ValueMap. Returns true if the map becomes empty after the operation.\n   * @param value - The full value to store\n   * @param multiplicity - The multiplicity to add\n   * @param hashKey - Optional hash key to use instead of hashing the full value (used when in PrefixMap context)\n   */\n  addValue(value: TValue, multiplicity: number): boolean {\n    if (multiplicity === 0) return this.size === 0\n\n    const key = hash(value)\n    const currentValue = this.get(key)\n\n    if (currentValue) {\n      const [, currentMultiplicity] = currentValue\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.delete(key)\n      } else {\n        this.set(key, [value, newMultiplicity])\n      }\n    } else {\n      this.set(key, [value, multiplicity])\n    }\n\n    return this.size === 0\n  }\n}\n\n/**\n * A map from a difference collection trace's keys -> (value, multiplicities) that changed.\n * Used in operations like join and reduce where the operation needs to\n * exploit the key-value structure of the data to run efficiently.\n */\nexport class Index<TKey, TValue, TPrefix = any> {\n  /*\n   * This index maintains a nested map of keys -> (value, multiplicities), where:\n   * - initially the values are stored against the key as a single value tuple\n   * - when a key gets additional values, the values are stored against the key in a\n   *   prefix map\n   * - the prefix is extract where possible from values that are structured as\n   *   [rowPrimaryKey, rowValue], as they are in the Tanstack DB query pipeline.\n   * - only when there are multiple values for a given prefix do we fall back to a\n   *   hash to identify identical values, storing them in a third level value map.\n   */\n  #inner: IndexMap<TKey, TValue, TPrefix>\n  // Sum of multiplicities per key, for presence. Only joins read presence; a\n  // reduce index is built without it.\n  #consolidatedMultiplicity: Map<TKey, number> | undefined\n  /**\n   * When true, the producer guarantees that a value's string or number\n   * prefix determines the value: two values under one key with an equal\n   * prefix are equal, so they merge without a structural hash.\n   */\n  #prefixIdentity: boolean\n\n  constructor(options?: { prefixIdentity?: boolean; trackPresence?: boolean }) {\n    this.#inner = new Map()\n    if (options?.trackPresence ?? true)\n      this.#consolidatedMultiplicity = new Map()\n    this.#prefixIdentity = options?.prefixIdentity ?? false\n  }\n\n  /**\n   * Create an Index from multiple MultiSet messages.\n   * @param messages - Array of MultiSet messages to build the index from.\n   * @returns A new Index containing all the data from the messages.\n   */\n  static fromMultiSets<K, V>(messages: Array<MultiSet<[K, V]>>): Index<K, V> {\n    const index = new Index<K, V>()\n\n    for (const message of messages) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        index.addValue(key, [value, multiplicity])\n      }\n    }\n\n    return index\n  }\n\n  /**\n   * This method returns a string representation of the index.\n   * @param indent - Whether to indent the string representation.\n   * @returns A string representation of the index.\n   */\n  toString(indent = false): string {\n    return `Index(${JSON.stringify(\n      [...this.entries()],\n      undefined,\n      indent ? 2 : undefined,\n    )})`\n  }\n\n  /**\n   * The size of the index.\n   */\n  get size(): number {\n    return this.#inner.size\n  }\n\n  #presence(): Map<TKey, number> {\n    if (this.#consolidatedMultiplicity === undefined)\n      throw new Error(`This Index does not track presence`)\n    return this.#consolidatedMultiplicity\n  }\n\n  /**\n   * This method checks if the index has a given key.\n   * @param key - The key to check.\n   * @returns True if the index has the key, false otherwise.\n   */\n  has(key: TKey): boolean {\n    return this.#inner.has(key)\n  }\n\n  /**\n   * Check if a key has presence (non-zero consolidated multiplicity).\n   * @param key - The key to check.\n   * @returns True if the key has non-zero consolidated multiplicity, false otherwise.\n   */\n  hasPresence(key: TKey): boolean {\n    return (this.#presence().get(key) || 0) !== 0\n  }\n\n  /**\n   * Get the consolidated multiplicity (sum of multiplicities) for a key.\n   * @param key - The key to get the consolidated multiplicity for.\n   * @returns The consolidated multiplicity for the key.\n   */\n  getConsolidatedMultiplicity(key: TKey): number {\n    return this.#presence().get(key) || 0\n  }\n\n  /**\n   * Get all keys that have presence (non-zero consolidated multiplicity).\n   * @returns An iterator of keys with non-zero consolidated multiplicity.\n   */\n  getPresenceKeys(): Iterable<TKey> {\n    return this.#presence().keys()\n  }\n\n  /**\n   * This method returns all values for a given key.\n   * @param key - The key to get the values for.\n   * @returns An array of value tuples [value, multiplicity].\n   */\n  get(key: TKey): Array<[TValue, number]> {\n    // Built directly, in getIterator's order: join and reduce call this for\n    // every changed key.\n    const mapOrSingleValue = this.#inner.get(key)\n    if (mapOrSingleValue === undefined) return []\n    if (isSingleValue(mapOrSingleValue)) return [mapOrSingleValue]\n    if (mapOrSingleValue instanceof ValueMap)\n      return [...mapOrSingleValue.values()]\n    const result: Array<[TValue, number]> = []\n    for (const singleValueOrValueMap of mapOrSingleValue.values()) {\n      if (isSingleValue(singleValueOrValueMap))\n        result.push(singleValueOrValueMap)\n      else\n        for (const valueTuple of singleValueOrValueMap.values())\n          result.push(valueTuple)\n    }\n    return result\n  }\n\n  /**\n   * This method returns an iterator over all values for a given key.\n   * @param key - The key to get the values for.\n   * @returns An iterator of value tuples [value, multiplicity].\n   */\n  *getIterator(key: TKey): Iterable<[TValue, number]> {\n    const mapOrSingleValue = this.#inner.get(key)\n    if (isSingleValue(mapOrSingleValue)) {\n      yield mapOrSingleValue\n    } else if (mapOrSingleValue === undefined) {\n      return\n    } else if (mapOrSingleValue instanceof ValueMap) {\n      // Direct ValueMap - all values have NO_PREFIX\n      for (const valueTuple of mapOrSingleValue.values()) {\n        yield valueTuple\n      }\n    } else {\n      // PrefixMap - iterate through all prefixes\n      for (const singleValueOrValueMap of mapOrSingleValue.values()) {\n        if (isSingleValue(singleValueOrValueMap)) {\n          yield singleValueOrValueMap\n        } else {\n          for (const valueTuple of singleValueOrValueMap.values()) {\n            yield valueTuple\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * This returns an iterator that iterates over all key-value pairs.\n   * @returns An iterable of all key-value pairs (and their multiplicities) in the index.\n   */\n  *entries(): Iterable<[TKey, [TValue, number]]> {\n    for (const key of this.#inner.keys()) {\n      for (const valueTuple of this.getIterator(key)) {\n        yield [key, valueTuple]\n      }\n    }\n  }\n\n  /**\n   * This method only iterates over the keys and not over the values.\n   * Hence, it is more efficient than the `#entries` method.\n   * It returns an iterator that you can use if you need to iterate over the values for a given key.\n   * @returns An iterator of all *keys* in the index and their corresponding value iterator.\n   */\n  *entriesIterators(): Iterable<[TKey, Iterable<[TValue, number]>]> {\n    for (const key of this.#inner.keys()) {\n      yield [key, this.getIterator(key)]\n    }\n  }\n\n  /**\n   * This method adds a value to the index.\n   * @param key - The key to add the value to.\n   * @param valueTuple - The value tuple [value, multiplicity] to add to the index.\n   */\n  addValue(key: TKey, valueTuple: SingleValue<TValue>) {\n    const [value, multiplicity] = valueTuple\n    // If the multiplicity is 0, do nothing\n    if (multiplicity === 0) return\n\n    // Update consolidated multiplicity tracking\n    const presence = this.#consolidatedMultiplicity\n    if (presence) {\n      const newConsolidatedMultiplicity =\n        (presence.get(key) || 0) + multiplicity\n      if (newConsolidatedMultiplicity === 0) {\n        presence.delete(key)\n      } else {\n        presence.set(key, newConsolidatedMultiplicity)\n      }\n    }\n\n    const mapOrSingleValue = this.#inner.get(key)\n\n    if (mapOrSingleValue === undefined) {\n      // First value for this key\n      this.#inner.set(key, valueTuple)\n      return\n    }\n\n    if (isSingleValue(mapOrSingleValue)) {\n      // Handle transition from single value to map\n      this.#handleSingleValueTransition(\n        key,\n        mapOrSingleValue,\n        value,\n        multiplicity,\n      )\n      return\n    }\n\n    if (mapOrSingleValue instanceof ValueMap) {\n      // Handle existing ValueMap\n      const prefix = getPrefix<TValue, TPrefix>(value)\n      if (prefix !== NO_PREFIX) {\n        // Convert ValueMap to PrefixMap since we have a prefixed value\n        const prefixMap = new PrefixMap<TValue, TPrefix>()\n        prefixMap.set(NO_PREFIX, mapOrSingleValue)\n        prefixMap.set(prefix, valueTuple)\n        this.#inner.set(key, prefixMap)\n      } else {\n        // Add to existing ValueMap\n        const isEmpty = mapOrSingleValue.addValue(value, multiplicity)\n        if (isEmpty) {\n          this.#inner.delete(key)\n        }\n      }\n    } else {\n      // Handle existing PrefixMap\n      const isEmpty = mapOrSingleValue.addValue(\n        value,\n        multiplicity,\n        this.#prefixIdentity,\n      )\n      if (isEmpty) {\n        this.#inner.delete(key)\n      }\n    }\n  }\n\n  /**\n   * Handle the transition from a single value to either a ValueMap or PrefixMap\n   */\n  #handleSingleValueTransition(\n    key: TKey,\n    currentSingleValue: SingleValue<TValue>,\n    newValue: TValue,\n    multiplicity: number,\n  ) {\n    const [currentValue, currentMultiplicity] = currentSingleValue\n\n    // Check for exact same value (reference equality)\n    if (currentValue === newValue) {\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.#inner.delete(key)\n      } else {\n        this.#inner.set(key, [newValue, newMultiplicity])\n      }\n      return\n    }\n\n    // Get prefixes for both values\n    const newPrefix = getPrefix<TValue, TPrefix>(newValue)\n    const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)\n\n    // Check if they're the same value by prefix/suffix comparison\n    if (\n      isSamePrefix(currentPrefix, newPrefix) &&\n      ((this.#prefixIdentity && newPrefix !== NO_PREFIX) ||\n        hash(currentValue) === hash(newValue))\n    ) {\n      const newMultiplicity = currentMultiplicity + multiplicity\n      if (newMultiplicity === 0) {\n        this.#inner.delete(key)\n      } else {\n        this.#inner.set(key, [newValue, newMultiplicity])\n      }\n      return\n    }\n\n    // Different values - choose appropriate map type\n    if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) {\n      // Both have NO_PREFIX, use ValueMap directly\n      const valueMap = new ValueMap<TValue>()\n      valueMap.set(hash(currentValue), currentSingleValue)\n      valueMap.set(hash(newValue), [newValue, multiplicity])\n      this.#inner.set(key, valueMap)\n    } else {\n      // At least one has a prefix, use PrefixMap\n      const prefixMap = new PrefixMap<TValue, TPrefix>()\n\n      if (isSamePrefix(currentPrefix, newPrefix)) {\n        // Same prefix, different suffixes - need ValueMap within PrefixMap\n        const valueMap = new ValueMap<TValue>()\n        valueMap.set(hash(currentValue), currentSingleValue)\n        valueMap.set(hash(newValue), [newValue, multiplicity])\n        prefixMap.set(currentPrefix, valueMap)\n      } else {\n        // Different prefixes - store as separate single values\n        prefixMap.set(currentPrefix, currentSingleValue)\n        prefixMap.set(newPrefix, [newValue, multiplicity])\n      }\n\n      this.#inner.set(key, prefixMap)\n    }\n  }\n\n  /**\n   * This method appends another index to the current index.\n   * @param other - The index to append to the current index.\n   */\n  append(other: Index<TKey, TValue>): void {\n    for (const [key, value] of other.entries()) {\n      this.addValue(key, value)\n    }\n  }\n\n  /**\n   * This method joins two indexes.\n   * @param other - The index to join with the current index.\n   * @returns A multiset of the joined values.\n   */\n  join<TValue2>(\n    other: Index<TKey, TValue2>,\n  ): MultiSet<[TKey, [TValue, TValue2]]> {\n    const result: Array<[[TKey, [TValue, TValue2]], number]> = []\n    // We want to iterate over the smaller of the two indexes to reduce the\n    // number of operations we need to do.\n    if (this.size <= other.size) {\n      for (const [key, valueIt] of this.entriesIterators()) {\n        if (!other.has(key)) continue\n        const otherValues = other.get(key)\n        for (const [val1, mul1] of valueIt) {\n          for (const [val2, mul2] of otherValues) {\n            if (mul1 !== 0 && mul2 !== 0) {\n              result.push([[key, [val1, val2]], mul1 * mul2])\n            }\n          }\n        }\n      }\n    } else {\n      for (const [key, otherValueIt] of other.entriesIterators()) {\n        if (!this.has(key)) continue\n        const values = this.get(key)\n        for (const [val2, mul2] of otherValueIt) {\n          for (const [val1, mul1] of values) {\n            if (mul1 !== 0 && mul2 !== 0) {\n              result.push([[key, [val1, val2]], mul1 * mul2])\n            }\n          }\n        }\n      }\n    }\n\n    return new MultiSet(result)\n  }\n}\n\n/**\n * This function extracts the prefix from a value.\n * @param value - The value to extract the prefix from.\n * @returns The prefix and the suffix.\n */\n// Prefixes are Map keys, so they compare as a Map does: NaN equals NaN.\nfunction isSamePrefix(a: unknown, b: unknown): boolean {\n  return a === b || (Number.isNaN(a) && Number.isNaN(b))\n}\n\nfunction getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {\n  // If the value is an array and the first element is a string or number, then the\n  // first element is the prefix. This is used to distinguish between values without\n  // the need for hashing unless there are multiple values for the same prefix.\n  if (\n    Array.isArray(value) &&\n    (typeof value[0] === `string` ||\n      typeof value[0] === `number` ||\n      typeof value[0] === `bigint`)\n  ) {\n    return value[0] as TPrefix\n  }\n  return NO_PREFIX\n}\n\n/**\n * This function checks if a value is a single value.\n * @param value - The value to check.\n * @returns True if the value is a single value, false otherwise.\n */\nfunction isSingleValue<TValue>(\n  value: SingleValue<TValue> | unknown,\n): value is SingleValue<TValue> {\n  return Array.isArray(value)\n}\n"],"names":["hash","MultiSet"],"mappings":";;;;AAyCA,MAAM,mCAAmB,WAAW;AAapC,MAAM,kBAAmC,IAGvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA,SACE,OACA,cACA,iBAAiB,OACR;AACT,QAAI,iBAAiB,EAAG,QAAO,KAAK,SAAS;AAE7C,UAAM,SAAS,UAA2B,KAAK;AAC/C,UAAM,wBAAwB,KAAK,IAAI,MAAM;AAE7C,QAAI,cAAc,qBAAqB,GAAG;AACxC,YAAM,CAAC,cAAc,mBAAmB,IAAI;AAC5C,YAAM,gBAAgB,UAA2B,YAAY;AAE7D,UAAI,CAAC,aAAa,eAAe,MAAM,GAAG;AACxC,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAEA,UACE,iBAAiB,SAChB,kBAAkB,WAAW,aAC9BA,KAAAA,KAAK,YAAY,MAAMA,UAAK,KAAK,GACjC;AAEA,cAAM,kBAAkB,sBAAsB;AAC9C,YAAI,oBAAoB,GAAG;AACzB,eAAK,OAAO,MAAM;AAAA,QACpB,OAAO;AACL,eAAK,IAAI,QAAQ,CAAC,OAAO,eAAe,CAAC;AAAA,QAC3C;AAAA,MACF,OAAO;AAEL,cAAM,WAAW,IAAI,SAAA;AACrB,iBAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,qBAAqB;AACtD,iBAAS,IAAIA,UAAK,KAAK,GAAG,CAAC,OAAO,YAAY,CAAC;AAC/C,aAAK,IAAI,QAAQ,QAAQ;AAAA,MAC3B;AAAA,IACF,WAAW,0BAA0B,QAAW;AAE9C,WAAK,IAAI,QAAQ,CAAC,OAAO,YAAY,CAAC;AAAA,IACxC,OAAO;AAEL,YAAM,UAAU,sBAAsB,SAAS,OAAO,YAAY;AAClE,UAAI,SAAS;AACX,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAEA,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;AAGA,MAAM,iBAAyB,IAA4B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOzD,SAAS,OAAe,cAA+B;AACrD,QAAI,iBAAiB,EAAG,QAAO,KAAK,SAAS;AAE7C,UAAM,MAAMA,KAAAA,KAAK,KAAK;AACtB,UAAM,eAAe,KAAK,IAAI,GAAG;AAEjC,QAAI,cAAc;AAChB,YAAM,CAAA,EAAG,mBAAmB,IAAI;AAChC,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,GAAG;AAAA,MACjB,OAAO;AACL,aAAK,IAAI,KAAK,CAAC,OAAO,eAAe,CAAC;AAAA,MACxC;AAAA,IACF,OAAO;AACL,WAAK,IAAI,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,IACrC;AAEA,WAAO,KAAK,SAAS;AAAA,EACvB;AACF;AAOO,MAAM,MAAmC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAW9C;AAAA;AAAA;AAAA,EAGA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA,EAEA,YAAY,SAAiE;AAC3E,SAAK,6BAAa,IAAA;AAClB,QAAI,SAAS,iBAAiB;AAC5B,WAAK,gDAAgC,IAAA;AACvC,SAAK,kBAAkB,SAAS,kBAAkB;AAAA,EACpD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,cAAoB,UAAgD;AACzE,UAAM,QAAQ,IAAI,MAAA;AAElB,eAAW,WAAW,UAAU;AAC9B,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,cAAM,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,MAC3C;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,SAAS,OAAe;AAC/B,WAAO,SAAS,KAAK;AAAA,MACnB,CAAC,GAAG,KAAK,SAAS;AAAA,MAClB;AAAA,MACA,SAAS,IAAI;AAAA,IAAA,CACd;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,OAAe;AACjB,WAAO,KAAK,OAAO;AAAA,EACrB;AAAA,EAEA,YAA+B;AAC7B,QAAI,KAAK,8BAA8B;AACrC,YAAM,IAAI,MAAM,oCAAoC;AACtD,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,KAAoB;AACtB,WAAO,KAAK,OAAO,IAAI,GAAG;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,YAAY,KAAoB;AAC9B,YAAQ,KAAK,UAAA,EAAY,IAAI,GAAG,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,4BAA4B,KAAmB;AAC7C,WAAO,KAAK,UAAA,EAAY,IAAI,GAAG,KAAK;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkC;AAChC,WAAO,KAAK,UAAA,EAAY,KAAA;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,KAAoC;AAGtC,UAAM,mBAAmB,KAAK,OAAO,IAAI,GAAG;AAC5C,QAAI,qBAAqB,OAAW,QAAO,CAAA;AAC3C,QAAI,cAAc,gBAAgB,EAAG,QAAO,CAAC,gBAAgB;AAC7D,QAAI,4BAA4B;AAC9B,aAAO,CAAC,GAAG,iBAAiB,QAAQ;AACtC,UAAM,SAAkC,CAAA;AACxC,eAAW,yBAAyB,iBAAiB,UAAU;AAC7D,UAAI,cAAc,qBAAqB;AACrC,eAAO,KAAK,qBAAqB;AAAA;AAEjC,mBAAW,cAAc,sBAAsB,OAAA;AAC7C,iBAAO,KAAK,UAAU;AAAA,IAC5B;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,CAAC,YAAY,KAAuC;AAClD,UAAM,mBAAmB,KAAK,OAAO,IAAI,GAAG;AAC5C,QAAI,cAAc,gBAAgB,GAAG;AACnC,YAAM;AAAA,IACR,WAAW,qBAAqB,QAAW;AACzC;AAAA,IACF,WAAW,4BAA4B,UAAU;AAE/C,iBAAW,cAAc,iBAAiB,UAAU;AAClD,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AAEL,iBAAW,yBAAyB,iBAAiB,UAAU;AAC7D,YAAI,cAAc,qBAAqB,GAAG;AACxC,gBAAM;AAAA,QACR,OAAO;AACL,qBAAW,cAAc,sBAAsB,UAAU;AACvD,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,CAAC,UAA8C;AAC7C,eAAW,OAAO,KAAK,OAAO,KAAA,GAAQ;AACpC,iBAAW,cAAc,KAAK,YAAY,GAAG,GAAG;AAC9C,cAAM,CAAC,KAAK,UAAU;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,CAAC,mBAAiE;AAChE,eAAW,OAAO,KAAK,OAAO,KAAA,GAAQ;AACpC,YAAM,CAAC,KAAK,KAAK,YAAY,GAAG,CAAC;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,SAAS,KAAW,YAAiC;AACnD,UAAM,CAAC,OAAO,YAAY,IAAI;AAE9B,QAAI,iBAAiB,EAAG;AAGxB,UAAM,WAAW,KAAK;AACtB,QAAI,UAAU;AACZ,YAAM,+BACH,SAAS,IAAI,GAAG,KAAK,KAAK;AAC7B,UAAI,gCAAgC,GAAG;AACrC,iBAAS,OAAO,GAAG;AAAA,MACrB,OAAO;AACL,iBAAS,IAAI,KAAK,2BAA2B;AAAA,MAC/C;AAAA,IACF;AAEA,UAAM,mBAAmB,KAAK,OAAO,IAAI,GAAG;AAE5C,QAAI,qBAAqB,QAAW;AAElC,WAAK,OAAO,IAAI,KAAK,UAAU;AAC/B;AAAA,IACF;AAEA,QAAI,cAAc,gBAAgB,GAAG;AAEnC,WAAK;AAAA,QACH;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAEF;AAAA,IACF;AAEA,QAAI,4BAA4B,UAAU;AAExC,YAAM,SAAS,UAA2B,KAAK;AAC/C,UAAI,WAAW,WAAW;AAExB,cAAM,YAAY,IAAI,UAAA;AACtB,kBAAU,IAAI,WAAW,gBAAgB;AACzC,kBAAU,IAAI,QAAQ,UAAU;AAChC,aAAK,OAAO,IAAI,KAAK,SAAS;AAAA,MAChC,OAAO;AAEL,cAAM,UAAU,iBAAiB,SAAS,OAAO,YAAY;AAC7D,YAAI,SAAS;AACX,eAAK,OAAO,OAAO,GAAG;AAAA,QACxB;AAAA,MACF;AAAA,IACF,OAAO;AAEL,YAAM,UAAU,iBAAiB;AAAA,QAC/B;AAAA,QACA;AAAA,QACA,KAAK;AAAA,MAAA;AAEP,UAAI,SAAS;AACX,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,6BACE,KACA,oBACA,UACA,cACA;AACA,UAAM,CAAC,cAAc,mBAAmB,IAAI;AAG5C,QAAI,iBAAiB,UAAU;AAC7B,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB,OAAO;AACL,aAAK,OAAO,IAAI,KAAK,CAAC,UAAU,eAAe,CAAC;AAAA,MAClD;AACA;AAAA,IACF;AAGA,UAAM,YAAY,UAA2B,QAAQ;AACrD,UAAM,gBAAgB,UAA2B,YAAY;AAG7D,QACE,aAAa,eAAe,SAAS,MACnC,KAAK,mBAAmB,cAAc,aACtCA,KAAAA,KAAK,YAAY,MAAMA,KAAAA,KAAK,QAAQ,IACtC;AACA,YAAM,kBAAkB,sBAAsB;AAC9C,UAAI,oBAAoB,GAAG;AACzB,aAAK,OAAO,OAAO,GAAG;AAAA,MACxB,OAAO;AACL,aAAK,OAAO,IAAI,KAAK,CAAC,UAAU,eAAe,CAAC;AAAA,MAClD;AACA;AAAA,IACF;AAGA,QAAI,kBAAkB,aAAa,cAAc,WAAW;AAE1D,YAAM,WAAW,IAAI,SAAA;AACrB,eAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,kBAAkB;AACnD,eAAS,IAAIA,UAAK,QAAQ,GAAG,CAAC,UAAU,YAAY,CAAC;AACrD,WAAK,OAAO,IAAI,KAAK,QAAQ;AAAA,IAC/B,OAAO;AAEL,YAAM,YAAY,IAAI,UAAA;AAEtB,UAAI,aAAa,eAAe,SAAS,GAAG;AAE1C,cAAM,WAAW,IAAI,SAAA;AACrB,iBAAS,IAAIA,KAAAA,KAAK,YAAY,GAAG,kBAAkB;AACnD,iBAAS,IAAIA,UAAK,QAAQ,GAAG,CAAC,UAAU,YAAY,CAAC;AACrD,kBAAU,IAAI,eAAe,QAAQ;AAAA,MACvC,OAAO;AAEL,kBAAU,IAAI,eAAe,kBAAkB;AAC/C,kBAAU,IAAI,WAAW,CAAC,UAAU,YAAY,CAAC;AAAA,MACnD;AAEA,WAAK,OAAO,IAAI,KAAK,SAAS;AAAA,IAChC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,OAAkC;AACvC,eAAW,CAAC,KAAK,KAAK,KAAK,MAAM,WAAW;AAC1C,WAAK,SAAS,KAAK,KAAK;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KACE,OACqC;AACrC,UAAM,SAAqD,CAAA;AAG3D,QAAI,KAAK,QAAQ,MAAM,MAAM;AAC3B,iBAAW,CAAC,KAAK,OAAO,KAAK,KAAK,oBAAoB;AACpD,YAAI,CAAC,MAAM,IAAI,GAAG,EAAG;AACrB,cAAM,cAAc,MAAM,IAAI,GAAG;AACjC,mBAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,qBAAW,CAAC,MAAM,IAAI,KAAK,aAAa;AACtC,gBAAI,SAAS,KAAK,SAAS,GAAG;AAC5B,qBAAO,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,OAAO;AACL,iBAAW,CAAC,KAAK,YAAY,KAAK,MAAM,oBAAoB;AAC1D,YAAI,CAAC,KAAK,IAAI,GAAG,EAAG;AACpB,cAAM,SAAS,KAAK,IAAI,GAAG;AAC3B,mBAAW,CAAC,MAAM,IAAI,KAAK,cAAc;AACvC,qBAAW,CAAC,MAAM,IAAI,KAAK,QAAQ;AACjC,gBAAI,SAAS,KAAK,SAAS,GAAG;AAC5B,qBAAO,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAIC,SAAAA,SAAS,MAAM;AAAA,EAC5B;AACF;AAQA,SAAS,aAAa,GAAY,GAAqB;AACrD,SAAO,MAAM,KAAM,OAAO,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC;AACtD;AAEA,SAAS,UAA2B,OAAoC;AAItE,MACE,MAAM,QAAQ,KAAK,MAClB,OAAO,MAAM,CAAC,MAAM,YACnB,OAAO,MAAM,CAAC,MAAM,YACpB,OAAO,MAAM,CAAC,MAAM,WACtB;AACA,WAAO,MAAM,CAAC;AAAA,EAChB;AACA,SAAO;AACT;AAOA,SAAS,cACP,OAC8B;AAC9B,SAAO,MAAM,QAAQ,KAAK;AAC5B;;"}

package/dist/cjs/indexes.d.cts

@@ -10,7 +10,10 @@ type SingleValue<TValue> = [
  */
 export declare class Index<TKey, TValue, TPrefix = any> {
     #private;
-    constructor();
+    constructor(options?: {
+        prefixIdentity?: boolean;
+        trackPresence?: boolean;
+    });
     /**
      * Create an Index from multiple MultiSet messages.
      * @param messages - Array of MultiSet messages to build the index from.

package/dist/cjs/operators/groupBy.cjs

@@ -1,63 +1,60 @@
 "use strict";
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const hash = require("../hashing/hash.cjs");
 const utils = require("../utils.cjs");
 const map = require("./map.cjs");
 const reduce = require("./reduce.cjs");
-function isPipedAggregateFunction(aggregate) {
-  return `pipe` in aggregate;
-}
 function groupBy(keyExtractor, aggregates = {}) {
-  const basicAggregates = Object.fromEntries(
-    Object.entries(aggregates).filter(
-      ([_, aggregate]) => !isPipedAggregateFunction(aggregate)
-    )
-  );
-  Object.fromEntries(
-    Object.entries(aggregates).filter(
-      ([_, aggregate]) => isPipedAggregateFunction(aggregate)
-    )
-  );
   return (stream) => {
     const KEY_SENTINEL = /* @__PURE__ */ Symbol(`original_group_key`);
+    const aggregateEntries = Object.entries(aggregates);
     const withKeysAndValues = stream.pipe(
       map.map((data) => {
         const key = keyExtractor(data);
         const keyString = utils.serializeValue(key);
         const values = {};
         values[KEY_SENTINEL] = key;
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          values[name] = aggregate.preMap(data);
+        let identity = hash.structuralIdentity(key) ?? null;
+        for (const [name, aggregate] of aggregateEntries) {
+          const value = aggregate.preMap(data);
+          values[name] = value;
+          if (identity === null) continue;
+          const part = hash.structuralIdentity(value);
+          identity = part === void 0 ? null : `${identity}|${part}`;
         }
-        return [keyString, values];
+        return [keyString, [identity, values]];
       })
     );
     const reduced = withKeysAndValues.pipe(
-      reduce.reduce((values) => {
-        let totalMultiplicity = 0;
-        for (const [_, multiplicity] of values) {
-          totalMultiplicity += multiplicity;
-        }
-        if (totalMultiplicity <= 0) {
-          return [];
-        }
-        const result = {};
-        const originalKey = values[0]?.[0]?.[KEY_SENTINEL];
-        result[KEY_SENTINEL] = originalKey;
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
-          const preValues = values.map(
-            ([v, m]) => [v[name], m]
-          );
-          result[name] = aggregate.reduce(preValues);
-        }
-        return [[result, 1]];
-      })
+      reduce.reduce(
+        (contributions) => {
+          let totalMultiplicity = 0;
+          for (const [_, multiplicity] of contributions) {
+            totalMultiplicity += multiplicity;
+          }
+          if (totalMultiplicity <= 0) {
+            return [];
+          }
+          const result = {};
+          const originalKey = contributions[0]?.[0][1][KEY_SENTINEL];
+          result[KEY_SENTINEL] = originalKey;
+          for (const [name, aggregate] of aggregateEntries) {
+            const preValues = contributions.map(
+              ([[, v], m]) => [v[name], m]
+            );
+            result[name] = aggregate.reduce(preValues);
+          }
+          return [[result, 1]];
+        },
+        { prefixIdentity: true }
+      )
     );
     return reduced.pipe(
       map.map(([keyString, values]) => {
         const key = values[KEY_SENTINEL];
         const result = {};
         Object.assign(result, key);
-        for (const [name, aggregate] of Object.entries(basicAggregates)) {
+        for (const [name, aggregate] of aggregateEntries) {
           if (aggregate.postMap) {
             result[name] = aggregate.postMap(values[name]);
           } else {

package/dist/cjs/operators/groupBy.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"groupBy.cjs","sources":["../../../src/operators/groupBy.ts"],"sourcesContent":["import { serializeValue } from '../utils.js'\nimport { map } from './map.js'\nimport { reduce } from './reduce.js'\nimport type { IStreamBuilder, KeyValue } from '../types.js'\n\ntype GroupKey = Record<string, unknown>\n\ntype BasicAggregateFunction<T, R, V = unknown> = {\n  preMap: (data: T) => V\n  reduce: (values: Array<[V, number]>) => V\n  postMap?: (result: V) => R\n}\n\ntype PipedAggregateFunction<T, R> = {\n  pipe: (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<string, R>>\n}\n\ntype AggregateFunction<T, R, V = unknown> =\n  BasicAggregateFunction<T, R, V> | PipedAggregateFunction<T, R>\n\ntype ExtractAggregateReturnType<T, A> =\n  A extends AggregateFunction<T, infer R, any> ? R : never\n\ntype AggregatesReturnType<T, A> = {\n  [K in keyof A]: ExtractAggregateReturnType<T, A[K]>\n}\n\nfunction isPipedAggregateFunction<T, R>(\n  aggregate: AggregateFunction<T, R>,\n): aggregate is PipedAggregateFunction<T, R> {\n  return `pipe` in aggregate\n}\n\n/**\n * Groups data by key and applies multiple aggregate operations\n * @param keyExtractor Function to extract grouping key from data\n * @param aggregates Object mapping aggregate names to aggregate functions\n */\nexport function groupBy<\n  T,\n  K extends GroupKey,\n  A extends Record<string, AggregateFunction<T, any, any>>,\n>(keyExtractor: (data: T) => K, aggregates: A = {} as A) {\n  type ResultType = K & AggregatesReturnType<T, A>\n\n  const basicAggregates = Object.fromEntries(\n    Object.entries(aggregates).filter(\n      ([_, aggregate]) => !isPipedAggregateFunction(aggregate),\n    ),\n  ) as Record<string, BasicAggregateFunction<T, any, any>>\n\n  // @ts-expect-error - TODO: we don't use this yet, but we will\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  const pipedAggregates = Object.fromEntries(\n    Object.entries(aggregates).filter(([_, aggregate]) =>\n      isPipedAggregateFunction(aggregate),\n    ),\n  ) as Record<string, PipedAggregateFunction<T, any>>\n\n  return (\n    stream: IStreamBuilder<T>,\n  ): IStreamBuilder<KeyValue<string, ResultType>> => {\n    // Special key to store the original key object\n    const KEY_SENTINEL = Symbol(`original_group_key`)\n\n    // First map to extract keys and pre-aggregate values\n    const withKeysAndValues = stream.pipe(\n      map((data) => {\n        const key = keyExtractor(data)\n        const keyString = serializeValue(key)\n\n        // Create values object with pre-aggregated values\n        const values: Record<string | symbol, unknown> = {}\n\n        // Store the original key object\n        values[KEY_SENTINEL] = key\n\n        // Add pre-aggregated values\n        for (const [name, aggregate] of Object.entries(basicAggregates)) {\n          values[name] = aggregate.preMap(data)\n        }\n\n        return [keyString, values] as KeyValue<\n          string,\n          Record<string | symbol, unknown>\n        >\n      }),\n    )\n\n    // Then reduce to compute aggregates\n    const reduced = withKeysAndValues.pipe(\n      reduce((values) => {\n        // Calculate total multiplicity to check if the group should exist\n        let totalMultiplicity = 0\n        for (const [_, multiplicity] of values) {\n          totalMultiplicity += multiplicity\n        }\n\n        // If total multiplicity is 0 or negative, the group should be removed completely\n        if (totalMultiplicity <= 0) {\n          return []\n        }\n\n        const result: Record<PropertyKey, unknown> = {}\n\n        // Get the original key from first value in group\n        const originalKey = values[0]?.[0]?.[KEY_SENTINEL]\n        result[KEY_SENTINEL] = originalKey\n\n        // Apply each aggregate function\n        for (const [name, aggregate] of Object.entries(basicAggregates)) {\n          const preValues = values.map(\n            ([v, m]) => [v[name], m] as [any, number],\n          )\n          result[name] = aggregate.reduce(preValues)\n        }\n\n        return [[result, 1]]\n      }),\n    )\n\n    // Finally map to extract the key and include all values\n    return reduced.pipe(\n      map(([keyString, values]) => {\n        // Extract the original key\n        const key = values[KEY_SENTINEL] as K\n\n        // Create intermediate result with key values and aggregate results\n        const result: Record<string, unknown> = {}\n\n        // Add key properties to result\n        Object.assign(result, key)\n\n        // Apply postMap if provided\n        for (const [name, aggregate] of Object.entries(basicAggregates)) {\n          if (aggregate.postMap) {\n            result[name] = aggregate.postMap(values[name])\n          } else {\n            result[name] = values[name]\n          }\n        }\n\n        // Return with the string key instead of the object\n        return [keyString, result] as KeyValue<string, ResultType>\n      }),\n    )\n  }\n}\n\n/**\n * Creates a sum aggregate function\n */\nexport function sum<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, number> {\n  return {\n    preMap: (data: T) => valueExtractor(data),\n    reduce: (values: Array<[number, number]>) => {\n      let total = 0\n      for (const [value, multiplicity] of values) {\n        total += value * multiplicity\n      }\n      return total\n    },\n  }\n}\n\n/**\n * Creates a count aggregate function\n */\nexport function count<T>(\n  valueExtractor: (value: T) => any = (v) => v,\n): AggregateFunction<T, number, number> {\n  return {\n    // Count only not-null values (the `== null` comparison gives true for both null and undefined)\n    preMap: (data: T) => (valueExtractor(data) == null ? 0 : 1),\n    reduce: (values: Array<[number, number]>) => {\n      let totalCount = 0\n      for (const [nullMultiplier, multiplicity] of values) {\n        totalCount += nullMultiplier * multiplicity\n      }\n      return totalCount\n    },\n  }\n}\n\n/**\n * Creates an average aggregate function\n */\nexport function avg<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, { sum: number; count: number }> {\n  return {\n    preMap: (data: T) => ({\n      sum: valueExtractor(data),\n      count: 0,\n    }),\n    reduce: (values: Array<[{ sum: number; count: number }, number]>) => {\n      let totalSum = 0\n      let totalCount = 0\n      for (const [value, multiplicity] of values) {\n        totalSum += value.sum * multiplicity\n        totalCount += multiplicity\n      }\n      return {\n        sum: totalSum,\n        count: totalCount,\n      }\n    },\n    postMap: (result: { sum: number; count: number }) => {\n      return result.sum / result.count\n    },\n  }\n}\n\ntype CanMinMax = number | Date | bigint | string\n\n/**\n * Creates a min aggregate function that computes the minimum value in a group\n * @param valueExtractor Function to extract a comparable value from each data entry\n */\nexport function min<T extends CanMinMax>(): AggregateFunction<\n  T,\n  T | undefined,\n  T | undefined\n>\nexport function min<T, V extends CanMinMax>(\n  valueExtractor: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined>\nexport function min<T, V extends CanMinMax>(\n  valueExtractor?: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined> {\n  const extractor = valueExtractor ?? ((v: T) => v as unknown as V)\n  return {\n    preMap: (data: T) => extractor(data),\n    reduce: (values) => {\n      let minValue: V | undefined\n      for (const [value, _multiplicity] of values) {\n        if (\n          value !== undefined &&\n          (minValue === undefined || value < minValue)\n        ) {\n          minValue = value\n        }\n      }\n      return minValue\n    },\n  }\n}\n\n/**\n * Creates a max aggregate function that computes the maximum value in a group\n * @param valueExtractor Function to extract a comparable value from each data entry\n */\nexport function max<T extends CanMinMax>(): AggregateFunction<\n  T,\n  T | undefined,\n  T | undefined\n>\nexport function max<T, V extends CanMinMax>(\n  valueExtractor: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined>\nexport function max<T, V extends CanMinMax>(\n  valueExtractor?: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined> {\n  const extractor = valueExtractor ?? ((v: T) => v as unknown as V)\n  return {\n    preMap: (data: T) => extractor(data),\n    reduce: (values) => {\n      let maxValue: V | undefined\n      for (const [value, _multiplicity] of values) {\n        if (\n          value !== undefined &&\n          (maxValue === undefined || value > maxValue)\n        ) {\n          maxValue = value\n        }\n      }\n      return maxValue\n    },\n  }\n}\n\n/**\n * Creates a median aggregate function that computes the middle value in a sorted group\n * If there's an even number of values, returns the average of the two middle values\n * @param valueExtractor Function to extract a numeric value from each data entry\n */\nexport function median<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, Array<number>> {\n  return {\n    preMap: (data: T) => [valueExtractor(data)],\n    reduce: (values: Array<[Array<number>, number]>) => {\n      // Flatten all values, taking multiplicity into account\n      const allValues: Array<number> = []\n      for (const [valueArray, multiplicity] of values) {\n        for (const value of valueArray) {\n          // Add each value multiple times based on multiplicity\n          for (let i = 0; i < multiplicity; i++) {\n            allValues.push(value)\n          }\n        }\n      }\n\n      // Return empty array if no values\n      if (allValues.length === 0) {\n        return []\n      }\n\n      // Sort values\n      allValues.sort((a, b) => a - b)\n\n      return allValues\n    },\n    postMap: (result: Array<number>) => {\n      if (result.length === 0) return 0\n\n      const mid = Math.floor(result.length / 2)\n\n      // If even number of values, average the two middle values\n      if (result.length % 2 === 0) {\n        return (result[mid - 1]! + result[mid]!) / 2\n      }\n\n      // If odd number of values, return the middle value\n      return result[mid]!\n    },\n  }\n}\n\n/**\n * Creates a mode aggregate function that computes the most frequent value in a group\n * If multiple values have the same highest frequency, returns the first one encountered\n * @param valueExtractor Function to extract a value from each data entry\n */\nexport function mode<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, Map<number, number>> {\n  return {\n    preMap: (data: T) => {\n      const value = valueExtractor(data)\n      const frequencyMap = new Map<number, number>()\n      frequencyMap.set(value, 1)\n      return frequencyMap\n    },\n    reduce: (values: Array<[Map<number, number>, number]>) => {\n      // Combine all frequency maps\n      const combinedMap = new Map<number, number>()\n\n      for (const [frequencyMap, multiplicity] of values) {\n        for (const [value, frequencyCount] of frequencyMap.entries()) {\n          const currentCount = combinedMap.get(value) || 0\n          combinedMap.set(value, currentCount + frequencyCount * multiplicity)\n        }\n      }\n\n      return combinedMap\n    },\n    postMap: (result: Map<number, number>) => {\n      if (result.size === 0) return 0\n\n      let modeValue = 0\n      let maxFrequency = 0\n\n      for (const [value, frequency] of result.entries()) {\n        if (frequency > maxFrequency) {\n          maxFrequency = frequency\n          modeValue = value\n        }\n      }\n\n      return modeValue\n    },\n  }\n}\n\nexport const groupByOperators = {\n  sum,\n  count,\n  avg,\n  min,\n  max,\n  median,\n  mode,\n}\n"],"names":["map","serializeValue","reduce"],"mappings":";;;;;AA2BA,SAAS,yBACP,WAC2C;AAC3C,SAAO,UAAU;AACnB;AAOO,SAAS,QAId,cAA8B,aAAgB,IAAS;AAGvD,QAAM,kBAAkB,OAAO;AAAA,IAC7B,OAAO,QAAQ,UAAU,EAAE;AAAA,MACzB,CAAC,CAAC,GAAG,SAAS,MAAM,CAAC,yBAAyB,SAAS;AAAA,IAAA;AAAA,EACzD;AAKsB,SAAO;AAAA,IAC7B,OAAO,QAAQ,UAAU,EAAE;AAAA,MAAO,CAAC,CAAC,GAAG,SAAS,MAC9C,yBAAyB,SAAS;AAAA,IAAA;AAAA,EACpC;AAGF,SAAO,CACL,WACiD;AAEjD,UAAM,sCAAsB,oBAAoB;AAGhD,UAAM,oBAAoB,OAAO;AAAA,MAC/BA,IAAAA,IAAI,CAAC,SAAS;AACZ,cAAM,MAAM,aAAa,IAAI;AAC7B,cAAM,YAAYC,MAAAA,eAAe,GAAG;AAGpC,cAAM,SAA2C,CAAA;AAGjD,eAAO,YAAY,IAAI;AAGvB,mBAAW,CAAC,MAAM,SAAS,KAAK,OAAO,QAAQ,eAAe,GAAG;AAC/D,iBAAO,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,QACtC;AAEA,eAAO,CAAC,WAAW,MAAM;AAAA,MAI3B,CAAC;AAAA,IAAA;AAIH,UAAM,UAAU,kBAAkB;AAAA,MAChCC,OAAAA,OAAO,CAAC,WAAW;AAEjB,YAAI,oBAAoB;AACxB,mBAAW,CAAC,GAAG,YAAY,KAAK,QAAQ;AACtC,+BAAqB;AAAA,QACvB;AAGA,YAAI,qBAAqB,GAAG;AAC1B,iBAAO,CAAA;AAAA,QACT;AAEA,cAAM,SAAuC,CAAA;AAG7C,cAAM,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,YAAY;AACjD,eAAO,YAAY,IAAI;AAGvB,mBAAW,CAAC,MAAM,SAAS,KAAK,OAAO,QAAQ,eAAe,GAAG;AAC/D,gBAAM,YAAY,OAAO;AAAA,YACvB,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;AAAA,UAAA;AAEzB,iBAAO,IAAI,IAAI,UAAU,OAAO,SAAS;AAAA,QAC3C;AAEA,eAAO,CAAC,CAAC,QAAQ,CAAC,CAAC;AAAA,MACrB,CAAC;AAAA,IAAA;AAIH,WAAO,QAAQ;AAAA,MACbF,IAAAA,IAAI,CAAC,CAAC,WAAW,MAAM,MAAM;AAE3B,cAAM,MAAM,OAAO,YAAY;AAG/B,cAAM,SAAkC,CAAA;AAGxC,eAAO,OAAO,QAAQ,GAAG;AAGzB,mBAAW,CAAC,MAAM,SAAS,KAAK,OAAO,QAAQ,eAAe,GAAG;AAC/D,cAAI,UAAU,SAAS;AACrB,mBAAO,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,CAAC;AAAA,UAC/C,OAAO;AACL,mBAAO,IAAI,IAAI,OAAO,IAAI;AAAA,UAC5B;AAAA,QACF;AAGA,eAAO,CAAC,WAAW,MAAM;AAAA,MAC3B,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAKO,SAAS,IACd,iBAAuC,CAAC,MAAM,GACR;AACtC,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,eAAe,IAAI;AAAA,IACxC,QAAQ,CAAC,WAAoC;AAC3C,UAAI,QAAQ;AACZ,iBAAW,CAAC,OAAO,YAAY,KAAK,QAAQ;AAC1C,iBAAS,QAAQ;AAAA,MACnB;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAKO,SAAS,MACd,iBAAoC,CAAC,MAAM,GACL;AACtC,SAAO;AAAA;AAAA,IAEL,QAAQ,CAAC,SAAa,eAAe,IAAI,KAAK,OAAO,IAAI;AAAA,IACzD,QAAQ,CAAC,WAAoC;AAC3C,UAAI,aAAa;AACjB,iBAAW,CAAC,gBAAgB,YAAY,KAAK,QAAQ;AACnD,sBAAc,iBAAiB;AAAA,MACjC;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAKO,SAAS,IACd,iBAAuC,CAAC,MAAM,GACgB;AAC9D,SAAO;AAAA,IACL,QAAQ,CAAC,UAAa;AAAA,MACpB,KAAK,eAAe,IAAI;AAAA,MACxB,OAAO;AAAA,IAAA;AAAA,IAET,QAAQ,CAAC,WAA4D;AACnE,UAAI,WAAW;AACf,UAAI,aAAa;AACjB,iBAAW,CAAC,OAAO,YAAY,KAAK,QAAQ;AAC1C,oBAAY,MAAM,MAAM;AACxB,sBAAc;AAAA,MAChB;AACA,aAAO;AAAA,QACL,KAAK;AAAA,QACL,OAAO;AAAA,MAAA;AAAA,IAEX;AAAA,IACA,SAAS,CAAC,WAA2C;AACnD,aAAO,OAAO,MAAM,OAAO;AAAA,IAC7B;AAAA,EAAA;AAEJ;AAgBO,SAAS,IACd,gBACoD;AACpD,QAAM,YAAY,mBAAmB,CAAC,MAAS;AAC/C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,UAAU,IAAI;AAAA,IACnC,QAAQ,CAAC,WAAW;AAClB,UAAI;AACJ,iBAAW,CAAC,OAAO,aAAa,KAAK,QAAQ;AAC3C,YACE,UAAU,WACT,aAAa,UAAa,QAAQ,WACnC;AACA,qBAAW;AAAA,QACb;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAcO,SAAS,IACd,gBACoD;AACpD,QAAM,YAAY,mBAAmB,CAAC,MAAS;AAC/C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,UAAU,IAAI;AAAA,IACnC,QAAQ,CAAC,WAAW;AAClB,UAAI;AACJ,iBAAW,CAAC,OAAO,aAAa,KAAK,QAAQ;AAC3C,YACE,UAAU,WACT,aAAa,UAAa,QAAQ,WACnC;AACA,qBAAW;AAAA,QACb;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAOO,SAAS,OACd,iBAAuC,CAAC,MAAM,GACD;AAC7C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,CAAC,eAAe,IAAI,CAAC;AAAA,IAC1C,QAAQ,CAAC,WAA2C;AAElD,YAAM,YAA2B,CAAA;AACjC,iBAAW,CAAC,YAAY,YAAY,KAAK,QAAQ;AAC/C,mBAAW,SAAS,YAAY;AAE9B,mBAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,sBAAU,KAAK,KAAK;AAAA,UACtB;AAAA,QACF;AAAA,MACF;AAGA,UAAI,UAAU,WAAW,GAAG;AAC1B,eAAO,CAAA;AAAA,MACT;AAGA,gBAAU,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAE9B,aAAO;AAAA,IACT;AAAA,IACA,SAAS,CAAC,WAA0B;AAClC,UAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,YAAM,MAAM,KAAK,MAAM,OAAO,SAAS,CAAC;AAGxC,UAAI,OAAO,SAAS,MAAM,GAAG;AAC3B,gBAAQ,OAAO,MAAM,CAAC,IAAK,OAAO,GAAG,KAAM;AAAA,MAC7C;AAGA,aAAO,OAAO,GAAG;AAAA,IACnB;AAAA,EAAA;AAEJ;AAOO,SAAS,KACd,iBAAuC,CAAC,MAAM,GACK;AACnD,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY;AACnB,YAAM,QAAQ,eAAe,IAAI;AACjC,YAAM,mCAAmB,IAAA;AACzB,mBAAa,IAAI,OAAO,CAAC;AACzB,aAAO;AAAA,IACT;AAAA,IACA,QAAQ,CAAC,WAAiD;AAExD,YAAM,kCAAkB,IAAA;AAExB,iBAAW,CAAC,cAAc,YAAY,KAAK,QAAQ;AACjD,mBAAW,CAAC,OAAO,cAAc,KAAK,aAAa,WAAW;AAC5D,gBAAM,eAAe,YAAY,IAAI,KAAK,KAAK;AAC/C,sBAAY,IAAI,OAAO,eAAe,iBAAiB,YAAY;AAAA,QACrE;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAAA,IACA,SAAS,CAAC,WAAgC;AACxC,UAAI,OAAO,SAAS,EAAG,QAAO;AAE9B,UAAI,YAAY;AAChB,UAAI,eAAe;AAEnB,iBAAW,CAAC,OAAO,SAAS,KAAK,OAAO,WAAW;AACjD,YAAI,YAAY,cAAc;AAC5B,yBAAe;AACf,sBAAY;AAAA,QACd;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAEO,MAAM,mBAAmB;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;;;;;;;;;;"}
+{"version":3,"file":"groupBy.cjs","sources":["../../../src/operators/groupBy.ts"],"sourcesContent":["import { structuralIdentity } from '../hashing/hash.js'\nimport { serializeValue } from '../utils.js'\nimport { map } from './map.js'\nimport { reduce } from './reduce.js'\nimport type { IStreamBuilder, KeyValue } from '../types.js'\n\ntype GroupKey = Record<string, unknown>\n\ntype AggregateFunction<T, R, V = unknown> = {\n  preMap: (data: T) => V\n  reduce: (values: Array<[V, number]>) => V\n  postMap?: (result: V) => R\n}\n\ntype ExtractAggregateReturnType<T, A> =\n  A extends AggregateFunction<T, infer R, any> ? R : never\n\ntype AggregatesReturnType<T, A> = {\n  [K in keyof A]: ExtractAggregateReturnType<T, A[K]>\n}\n\n/**\n * Groups data by key and applies multiple aggregate operations\n * @param keyExtractor Function to extract grouping key from data\n * @param aggregates Object mapping aggregate names to aggregate functions\n */\nexport function groupBy<\n  T,\n  K extends GroupKey,\n  A extends Record<string, AggregateFunction<T, any, any>>,\n>(keyExtractor: (data: T) => K, aggregates: A = {} as A) {\n  type ResultType = K & AggregatesReturnType<T, A>\n\n  return (\n    stream: IStreamBuilder<T>,\n  ): IStreamBuilder<KeyValue<string, ResultType>> => {\n    // Special key to store the original key object\n    const KEY_SENTINEL = Symbol(`original_group_key`)\n\n    const aggregateEntries = Object.entries(aggregates)\n\n    // First map to extract keys and pre-aggregate values. Each contribution\n    // leads with an identity built from its group key and pre-aggregate\n    // values when every one of them has one. Within one group the reduce\n    // index then merges equal contributions by that identity instead of\n    // hashing them. A null identity falls back to the structural hash.\n    const withKeysAndValues = stream.pipe(\n      map((data) => {\n        const key = keyExtractor(data)\n        const keyString = serializeValue(key)\n\n        // Create values object with pre-aggregated values\n        const values: Record<string | symbol, unknown> = {}\n\n        // Store the original key object. Contributions whose keys serialize\n        // alike but hash apart must not merge, so the key is part of the\n        // identity.\n        values[KEY_SENTINEL] = key\n\n        // Add pre-aggregated values\n        let identity: string | null = structuralIdentity(key) ?? null\n        for (const [name, aggregate] of aggregateEntries) {\n          const value = aggregate.preMap(data)\n          values[name] = value\n          if (identity === null) continue\n          const part = structuralIdentity(value)\n          identity = part === undefined ? null : `${identity}|${part}`\n        }\n\n        return [keyString, [identity, values]] as KeyValue<\n          string,\n          [string | null, Record<string | symbol, unknown>]\n        >\n      }),\n    )\n\n    // Then reduce to compute aggregates\n    const reduced = withKeysAndValues.pipe(\n      reduce(\n        (contributions) => {\n          // Calculate total multiplicity to check if the group should exist\n          let totalMultiplicity = 0\n          for (const [_, multiplicity] of contributions) {\n            totalMultiplicity += multiplicity\n          }\n\n          // If total multiplicity is 0 or negative, the group should be removed completely\n          if (totalMultiplicity <= 0) {\n            return []\n          }\n\n          const result: Record<PropertyKey, unknown> = {}\n\n          // Get the original key from first value in group\n          const originalKey = contributions[0]?.[0][1][KEY_SENTINEL]\n          result[KEY_SENTINEL] = originalKey\n\n          // Apply each aggregate function\n          for (const [name, aggregate] of aggregateEntries) {\n            const preValues = contributions.map(\n              ([[, v], m]) => [v[name], m] as [any, number],\n            )\n            result[name] = aggregate.reduce(preValues)\n          }\n\n          return [[result, 1]]\n        },\n        { prefixIdentity: true },\n      ),\n    )\n\n    // Finally map to extract the key and include all values\n    return reduced.pipe(\n      map(([keyString, values]) => {\n        // Extract the original key\n        const key = values[KEY_SENTINEL] as K\n\n        // Create intermediate result with key values and aggregate results\n        const result: Record<string, unknown> = {}\n\n        // Add key properties to result\n        Object.assign(result, key)\n\n        // Apply postMap if provided\n        for (const [name, aggregate] of aggregateEntries) {\n          if (aggregate.postMap) {\n            result[name] = aggregate.postMap(values[name])\n          } else {\n            result[name] = values[name]\n          }\n        }\n\n        // Return with the string key instead of the object\n        return [keyString, result] as KeyValue<string, ResultType>\n      }),\n    )\n  }\n}\n\n/**\n * Creates a sum aggregate function\n */\nexport function sum<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, number> {\n  return {\n    preMap: (data: T) => valueExtractor(data),\n    reduce: (values: Array<[number, number]>) => {\n      let total = 0\n      for (const [value, multiplicity] of values) {\n        total += value * multiplicity\n      }\n      return total\n    },\n  }\n}\n\n/**\n * Creates a count aggregate function\n */\nexport function count<T>(\n  valueExtractor: (value: T) => any = (v) => v,\n): AggregateFunction<T, number, number> {\n  return {\n    // Count only not-null values (the `== null` comparison gives true for both null and undefined)\n    preMap: (data: T) => (valueExtractor(data) == null ? 0 : 1),\n    reduce: (values: Array<[number, number]>) => {\n      let totalCount = 0\n      for (const [nullMultiplier, multiplicity] of values) {\n        totalCount += nullMultiplier * multiplicity\n      }\n      return totalCount\n    },\n  }\n}\n\n/**\n * Creates an average aggregate function\n */\nexport function avg<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, { sum: number; count: number }> {\n  return {\n    preMap: (data: T) => ({\n      sum: valueExtractor(data),\n      count: 0,\n    }),\n    reduce: (values: Array<[{ sum: number; count: number }, number]>) => {\n      let totalSum = 0\n      let totalCount = 0\n      for (const [value, multiplicity] of values) {\n        totalSum += value.sum * multiplicity\n        totalCount += multiplicity\n      }\n      return {\n        sum: totalSum,\n        count: totalCount,\n      }\n    },\n    postMap: (result: { sum: number; count: number }) => {\n      return result.sum / result.count\n    },\n  }\n}\n\ntype CanMinMax = number | Date | bigint | string\n\n/**\n * Creates a min aggregate function that computes the minimum value in a group\n * @param valueExtractor Function to extract a comparable value from each data entry\n */\nexport function min<T extends CanMinMax>(): AggregateFunction<\n  T,\n  T | undefined,\n  T | undefined\n>\nexport function min<T, V extends CanMinMax>(\n  valueExtractor: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined>\nexport function min<T, V extends CanMinMax>(\n  valueExtractor?: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined> {\n  const extractor = valueExtractor ?? ((v: T) => v as unknown as V)\n  return {\n    preMap: (data: T) => extractor(data),\n    reduce: (values) => {\n      let minValue: V | undefined\n      for (const [value, _multiplicity] of values) {\n        if (\n          value !== undefined &&\n          (minValue === undefined || value < minValue)\n        ) {\n          minValue = value\n        }\n      }\n      return minValue\n    },\n  }\n}\n\n/**\n * Creates a max aggregate function that computes the maximum value in a group\n * @param valueExtractor Function to extract a comparable value from each data entry\n */\nexport function max<T extends CanMinMax>(): AggregateFunction<\n  T,\n  T | undefined,\n  T | undefined\n>\nexport function max<T, V extends CanMinMax>(\n  valueExtractor: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined>\nexport function max<T, V extends CanMinMax>(\n  valueExtractor?: (value: T) => V,\n): AggregateFunction<T, V | undefined, V | undefined> {\n  const extractor = valueExtractor ?? ((v: T) => v as unknown as V)\n  return {\n    preMap: (data: T) => extractor(data),\n    reduce: (values) => {\n      let maxValue: V | undefined\n      for (const [value, _multiplicity] of values) {\n        if (\n          value !== undefined &&\n          (maxValue === undefined || value > maxValue)\n        ) {\n          maxValue = value\n        }\n      }\n      return maxValue\n    },\n  }\n}\n\n/**\n * Creates a median aggregate function that computes the middle value in a sorted group\n * If there's an even number of values, returns the average of the two middle values\n * @param valueExtractor Function to extract a numeric value from each data entry\n */\nexport function median<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, Array<number>> {\n  return {\n    preMap: (data: T) => [valueExtractor(data)],\n    reduce: (values: Array<[Array<number>, number]>) => {\n      // Flatten all values, taking multiplicity into account\n      const allValues: Array<number> = []\n      for (const [valueArray, multiplicity] of values) {\n        for (const value of valueArray) {\n          // Add each value multiple times based on multiplicity\n          for (let i = 0; i < multiplicity; i++) {\n            allValues.push(value)\n          }\n        }\n      }\n\n      // Return empty array if no values\n      if (allValues.length === 0) {\n        return []\n      }\n\n      // Sort values\n      allValues.sort((a, b) => a - b)\n\n      return allValues\n    },\n    postMap: (result: Array<number>) => {\n      if (result.length === 0) return 0\n\n      const mid = Math.floor(result.length / 2)\n\n      // If even number of values, average the two middle values\n      if (result.length % 2 === 0) {\n        return (result[mid - 1]! + result[mid]!) / 2\n      }\n\n      // If odd number of values, return the middle value\n      return result[mid]!\n    },\n  }\n}\n\n/**\n * Creates a mode aggregate function that computes the most frequent value in a group\n * If multiple values have the same highest frequency, returns the first one encountered\n * @param valueExtractor Function to extract a value from each data entry\n */\nexport function mode<T>(\n  valueExtractor: (value: T) => number = (v) => v as unknown as number,\n): AggregateFunction<T, number, Map<number, number>> {\n  return {\n    preMap: (data: T) => {\n      const value = valueExtractor(data)\n      const frequencyMap = new Map<number, number>()\n      frequencyMap.set(value, 1)\n      return frequencyMap\n    },\n    reduce: (values: Array<[Map<number, number>, number]>) => {\n      // Combine all frequency maps\n      const combinedMap = new Map<number, number>()\n\n      for (const [frequencyMap, multiplicity] of values) {\n        for (const [value, frequencyCount] of frequencyMap.entries()) {\n          const currentCount = combinedMap.get(value) || 0\n          combinedMap.set(value, currentCount + frequencyCount * multiplicity)\n        }\n      }\n\n      return combinedMap\n    },\n    postMap: (result: Map<number, number>) => {\n      if (result.size === 0) return 0\n\n      let modeValue = 0\n      let maxFrequency = 0\n\n      for (const [value, frequency] of result.entries()) {\n        if (frequency > maxFrequency) {\n          maxFrequency = frequency\n          modeValue = value\n        }\n      }\n\n      return modeValue\n    },\n  }\n}\n\nexport const groupByOperators = {\n  sum,\n  count,\n  avg,\n  min,\n  max,\n  median,\n  mode,\n}\n"],"names":["map","serializeValue","structuralIdentity","reduce"],"mappings":";;;;;;AA0BO,SAAS,QAId,cAA8B,aAAgB,IAAS;AAGvD,SAAO,CACL,WACiD;AAEjD,UAAM,sCAAsB,oBAAoB;AAEhD,UAAM,mBAAmB,OAAO,QAAQ,UAAU;AAOlD,UAAM,oBAAoB,OAAO;AAAA,MAC/BA,IAAAA,IAAI,CAAC,SAAS;AACZ,cAAM,MAAM,aAAa,IAAI;AAC7B,cAAM,YAAYC,MAAAA,eAAe,GAAG;AAGpC,cAAM,SAA2C,CAAA;AAKjD,eAAO,YAAY,IAAI;AAGvB,YAAI,WAA0BC,KAAAA,mBAAmB,GAAG,KAAK;AACzD,mBAAW,CAAC,MAAM,SAAS,KAAK,kBAAkB;AAChD,gBAAM,QAAQ,UAAU,OAAO,IAAI;AACnC,iBAAO,IAAI,IAAI;AACf,cAAI,aAAa,KAAM;AACvB,gBAAM,OAAOA,KAAAA,mBAAmB,KAAK;AACrC,qBAAW,SAAS,SAAY,OAAO,GAAG,QAAQ,IAAI,IAAI;AAAA,QAC5D;AAEA,eAAO,CAAC,WAAW,CAAC,UAAU,MAAM,CAAC;AAAA,MAIvC,CAAC;AAAA,IAAA;AAIH,UAAM,UAAU,kBAAkB;AAAA,MAChCC,OAAAA;AAAAA,QACE,CAAC,kBAAkB;AAEjB,cAAI,oBAAoB;AACxB,qBAAW,CAAC,GAAG,YAAY,KAAK,eAAe;AAC7C,iCAAqB;AAAA,UACvB;AAGA,cAAI,qBAAqB,GAAG;AAC1B,mBAAO,CAAA;AAAA,UACT;AAEA,gBAAM,SAAuC,CAAA;AAG7C,gBAAM,cAAc,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,YAAY;AACzD,iBAAO,YAAY,IAAI;AAGvB,qBAAW,CAAC,MAAM,SAAS,KAAK,kBAAkB;AAChD,kBAAM,YAAY,cAAc;AAAA,cAC9B,CAAC,CAAC,CAAA,EAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;AAAA,YAAA;AAE7B,mBAAO,IAAI,IAAI,UAAU,OAAO,SAAS;AAAA,UAC3C;AAEA,iBAAO,CAAC,CAAC,QAAQ,CAAC,CAAC;AAAA,QACrB;AAAA,QACA,EAAE,gBAAgB,KAAA;AAAA,MAAK;AAAA,IACzB;AAIF,WAAO,QAAQ;AAAA,MACbH,IAAAA,IAAI,CAAC,CAAC,WAAW,MAAM,MAAM;AAE3B,cAAM,MAAM,OAAO,YAAY;AAG/B,cAAM,SAAkC,CAAA;AAGxC,eAAO,OAAO,QAAQ,GAAG;AAGzB,mBAAW,CAAC,MAAM,SAAS,KAAK,kBAAkB;AAChD,cAAI,UAAU,SAAS;AACrB,mBAAO,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,CAAC;AAAA,UAC/C,OAAO;AACL,mBAAO,IAAI,IAAI,OAAO,IAAI;AAAA,UAC5B;AAAA,QACF;AAGA,eAAO,CAAC,WAAW,MAAM;AAAA,MAC3B,CAAC;AAAA,IAAA;AAAA,EAEL;AACF;AAKO,SAAS,IACd,iBAAuC,CAAC,MAAM,GACR;AACtC,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,eAAe,IAAI;AAAA,IACxC,QAAQ,CAAC,WAAoC;AAC3C,UAAI,QAAQ;AACZ,iBAAW,CAAC,OAAO,YAAY,KAAK,QAAQ;AAC1C,iBAAS,QAAQ;AAAA,MACnB;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAKO,SAAS,MACd,iBAAoC,CAAC,MAAM,GACL;AACtC,SAAO;AAAA;AAAA,IAEL,QAAQ,CAAC,SAAa,eAAe,IAAI,KAAK,OAAO,IAAI;AAAA,IACzD,QAAQ,CAAC,WAAoC;AAC3C,UAAI,aAAa;AACjB,iBAAW,CAAC,gBAAgB,YAAY,KAAK,QAAQ;AACnD,sBAAc,iBAAiB;AAAA,MACjC;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAKO,SAAS,IACd,iBAAuC,CAAC,MAAM,GACgB;AAC9D,SAAO;AAAA,IACL,QAAQ,CAAC,UAAa;AAAA,MACpB,KAAK,eAAe,IAAI;AAAA,MACxB,OAAO;AAAA,IAAA;AAAA,IAET,QAAQ,CAAC,WAA4D;AACnE,UAAI,WAAW;AACf,UAAI,aAAa;AACjB,iBAAW,CAAC,OAAO,YAAY,KAAK,QAAQ;AAC1C,oBAAY,MAAM,MAAM;AACxB,sBAAc;AAAA,MAChB;AACA,aAAO;AAAA,QACL,KAAK;AAAA,QACL,OAAO;AAAA,MAAA;AAAA,IAEX;AAAA,IACA,SAAS,CAAC,WAA2C;AACnD,aAAO,OAAO,MAAM,OAAO;AAAA,IAC7B;AAAA,EAAA;AAEJ;AAgBO,SAAS,IACd,gBACoD;AACpD,QAAM,YAAY,mBAAmB,CAAC,MAAS;AAC/C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,UAAU,IAAI;AAAA,IACnC,QAAQ,CAAC,WAAW;AAClB,UAAI;AACJ,iBAAW,CAAC,OAAO,aAAa,KAAK,QAAQ;AAC3C,YACE,UAAU,WACT,aAAa,UAAa,QAAQ,WACnC;AACA,qBAAW;AAAA,QACb;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAcO,SAAS,IACd,gBACoD;AACpD,QAAM,YAAY,mBAAmB,CAAC,MAAS;AAC/C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,UAAU,IAAI;AAAA,IACnC,QAAQ,CAAC,WAAW;AAClB,UAAI;AACJ,iBAAW,CAAC,OAAO,aAAa,KAAK,QAAQ;AAC3C,YACE,UAAU,WACT,aAAa,UAAa,QAAQ,WACnC;AACA,qBAAW;AAAA,QACb;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAOO,SAAS,OACd,iBAAuC,CAAC,MAAM,GACD;AAC7C,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY,CAAC,eAAe,IAAI,CAAC;AAAA,IAC1C,QAAQ,CAAC,WAA2C;AAElD,YAAM,YAA2B,CAAA;AACjC,iBAAW,CAAC,YAAY,YAAY,KAAK,QAAQ;AAC/C,mBAAW,SAAS,YAAY;AAE9B,mBAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,sBAAU,KAAK,KAAK;AAAA,UACtB;AAAA,QACF;AAAA,MACF;AAGA,UAAI,UAAU,WAAW,GAAG;AAC1B,eAAO,CAAA;AAAA,MACT;AAGA,gBAAU,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAE9B,aAAO;AAAA,IACT;AAAA,IACA,SAAS,CAAC,WAA0B;AAClC,UAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,YAAM,MAAM,KAAK,MAAM,OAAO,SAAS,CAAC;AAGxC,UAAI,OAAO,SAAS,MAAM,GAAG;AAC3B,gBAAQ,OAAO,MAAM,CAAC,IAAK,OAAO,GAAG,KAAM;AAAA,MAC7C;AAGA,aAAO,OAAO,GAAG;AAAA,IACnB;AAAA,EAAA;AAEJ;AAOO,SAAS,KACd,iBAAuC,CAAC,MAAM,GACK;AACnD,SAAO;AAAA,IACL,QAAQ,CAAC,SAAY;AACnB,YAAM,QAAQ,eAAe,IAAI;AACjC,YAAM,mCAAmB,IAAA;AACzB,mBAAa,IAAI,OAAO,CAAC;AACzB,aAAO;AAAA,IACT;AAAA,IACA,QAAQ,CAAC,WAAiD;AAExD,YAAM,kCAAkB,IAAA;AAExB,iBAAW,CAAC,cAAc,YAAY,KAAK,QAAQ;AACjD,mBAAW,CAAC,OAAO,cAAc,KAAK,aAAa,WAAW;AAC5D,gBAAM,eAAe,YAAY,IAAI,KAAK,KAAK;AAC/C,sBAAY,IAAI,OAAO,eAAe,iBAAiB,YAAY;AAAA,QACrE;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAAA,IACA,SAAS,CAAC,WAAgC;AACxC,UAAI,OAAO,SAAS,EAAG,QAAO;AAE9B,UAAI,YAAY;AAChB,UAAI,eAAe;AAEnB,iBAAW,CAAC,OAAO,SAAS,KAAK,OAAO,WAAW;AACjD,YAAI,YAAY,cAAc;AAC5B,yBAAe;AACf,sBAAY;AAAA,QACd;AAAA,MACF;AAEA,aAAO;AAAA,IACT;AAAA,EAAA;AAEJ;AAEO,MAAM,mBAAmB;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;;;;;;;;;;"}

package/dist/cjs/operators/groupBy.d.cts

@@ -1,6 +1,6 @@
 import { IStreamBuilder, KeyValue } from "../types.cjs";
 type GroupKey = Record<string, unknown>;
-type BasicAggregateFunction<T, R, V = unknown> = {
+type AggregateFunction<T, R, V = unknown> = {
     preMap: (data: T) => V;
     reduce: (values: Array<[
         V,
@@ -8,10 +8,6 @@ type BasicAggregateFunction<T, R, V = unknown> = {
     ]>) => V;
     postMap?: (result: V) => R;
 };
-type PipedAggregateFunction<T, R> = {
-    pipe: (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<string, R>>;
-};
-type AggregateFunction<T, R, V = unknown> = BasicAggregateFunction<T, R, V> | PipedAggregateFunction<T, R>;
 type ExtractAggregateReturnType<T, A> = A extends AggregateFunction<T, infer R, any> ? R : never;
 type AggregatesReturnType<T, A> = {
     [K in keyof A]: ExtractAggregateReturnType<T, A[K]>;

package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs

@@ -20,13 +20,6 @@ class GroupedTopKWithFractionalIndexOperator extends graph.UnaryOperator {
     options.setSizeCallback?.(() => this.#getTotalSize());
     options.setWindowFn?.(this.#moveTopK.bind(this));
   }
-  /**
-   * Creates a new TopK data structure for a group.
-   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).
-   */
-  createTopK(offset, limit, comparator) {
-    return new topKArray.TopKArray(offset, limit, comparator);
-  }
   #getTotalSize() {
     let size = 0;
     for (const state of this.#groupStates.values()) {
@@ -37,8 +30,9 @@ class GroupedTopKWithFractionalIndexOperator extends graph.UnaryOperator {
   #getOrCreateGroupState(groupKey) {
     let state = this.#groupStates.get(groupKey);
     if (!state) {
-      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator);
-      state = new topKState.TopKState(topK);
+      state = new topKState.TopKState(
+        new topKArray.TopKArray(this.#offset, this.#limit, this.#comparator)
+      );
       this.#groupStates.set(groupKey, state);
     }
     return state;

package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"groupedTopKWithFractionalIndex.cjs","sources":["../../../src/operators/groupedTopKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport {\n  TopKState,\n  handleMoveIn,\n  handleMoveOut,\n  topKBatch,\n} from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue, TopK } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface GroupedTopKWithFractionalIndexOptions<K, T> {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n  /**\n   * Function to extract a group key from the element's key and value.\n   * Elements with the same group key will be sorted and limited together.\n   */\n  groupKeyFn: (key: K, value: T) => unknown\n}\n\n/**\n * Operator for grouped fractional indexed topK operations.\n * This operator maintains separate topK windows for each group,\n * allowing per-group limits and ordering.\n *\n * The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].\n * Elements are grouped by the groupKeyFn, and each group maintains\n * its own sorted collection with independent limit/offset.\n */\nexport class GroupedTopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #groupStates: Map<unknown, TopKState<K, T>> = new Map()\n  #groupKeyFn: (key: K, value: T) => unknown\n  #comparator: (a: [K, T], b: [K, T]) => number\n  #offset: number\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: GroupedTopKWithFractionalIndexOptions<K, T>,\n  ) {\n    super(id, inputA, output)\n    this.#groupKeyFn = options.groupKeyFn\n    this.#limit = options.limit ?? Infinity\n    this.#offset = options.offset ?? 0\n    this.#comparator = createKeyedComparator(comparator)\n    options.setSizeCallback?.(() => this.#getTotalSize())\n    options.setWindowFn?.(this.#moveTopK.bind(this))\n  }\n\n  /**\n   * Creates a new TopK data structure for a group.\n   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).\n   */\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: [K, T], b: [K, T]) => number,\n  ): TopK<[K, T]> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  #getTotalSize(): number {\n    let size = 0\n    for (const state of this.#groupStates.values()) {\n      size += state.size\n    }\n    return size\n  }\n\n  #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {\n    let state = this.#groupStates.get(groupKey)\n    if (!state) {\n      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)\n      state = new TopKState(topK)\n      this.#groupStates.set(groupKey, state)\n    }\n    return state\n  }\n\n  #cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {\n    if (state.isEmpty) {\n      this.#groupStates.delete(groupKey)\n    }\n  }\n\n  /**\n   * Moves the topK window for all groups based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {\n    if (offset !== undefined) {\n      this.#offset = offset\n    }\n    if (limit !== undefined) {\n      this.#limit = limit\n    }\n\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    let hasChanges = false\n\n    for (const state of this.#groupStates.values()) {\n      const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit\n\n      diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n      diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n      if (diff.changes) {\n        hasChanges = true\n      }\n    }\n\n    if (hasChanges) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const [[key, value], multiplicity] of topKBatch(\n      this.inputMessages(),\n    )) {\n      this.#processElement(key, value, multiplicity, result)\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  #processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const groupKey = this.#groupKeyFn(key, value)\n    const state = this.#getOrCreateGroupState(groupKey)\n\n    const changes = state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n\n    // Cleanup empty groups to prevent memory leaks\n    this.#cleanupGroupIfEmpty(groupKey, state)\n  }\n}\n\n/**\n * Limits the number of results per group based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * This operator groups elements by the provided groupKeyFn and applies the limit/offset\n * independently to each group.\n *\n * @param comparator - A function that compares two elements for ordering\n * @param options - Configuration including groupKeyFn, limit, and offset\n * @returns A piped operator that orders elements per group and limits results per group\n */\nexport function groupedTopKWithFractionalIndex<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options: GroupedTopKWithFractionalIndexOptions<K, T>,\n): PipedOperator<[K, T], [K, IndexedValue<T>]> {\n  return (\n    stream: IStreamBuilder<[K, T]>,\n  ): IStreamBuilder<[K, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[K, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[K, IndexedValue<T>]>(),\n    )\n    const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      options,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":["UnaryOperator","createKeyedComparator","TopKArray","TopKState","handleMoveIn","handleMoveOut","MultiSet","topKBatch","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;;;;AAqCO,MAAM,+CAGHA,MAAAA,cAA4C;AAAA,EACpD,mCAAkD,IAAA;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,cAAc,QAAQ;AAC3B,SAAK,SAAS,QAAQ,SAAS;AAC/B,SAAK,UAAU,QAAQ,UAAU;AACjC,SAAK,cAAcC,UAAAA,sBAAsB,UAAU;AACnD,YAAQ,kBAAkB,MAAM,KAAK,cAAA,CAAe;AACpD,YAAQ,cAAc,KAAK,UAAU,KAAK,IAAI,CAAC;AAAA,EACjD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,WACR,QACA,OACA,YACc;AACd,WAAO,IAAIC,UAAAA,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,gBAAwB;AACtB,QAAI,OAAO;AACX,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,cAAQ,MAAM;AAAA,IAChB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,UAAoC;AACzD,QAAI,QAAQ,KAAK,aAAa,IAAI,QAAQ;AAC1C,QAAI,CAAC,OAAO;AACV,YAAM,OAAO,KAAK,WAAW,KAAK,SAAS,KAAK,QAAQ,KAAK,WAAW;AACxE,cAAQ,IAAIC,UAAAA,UAAU,IAAI;AAC1B,WAAK,aAAa,IAAI,UAAU,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,qBAAqB,UAAmB,OAA8B;AACpE,QAAI,MAAM,SAAS;AACjB,WAAK,aAAa,OAAO,QAAQ;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU,EAAE,QAAQ,SAAoD;AACtE,QAAI,WAAW,QAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,UAAU,QAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,SAAgD,CAAA;AACtD,QAAI,aAAa;AAEjB,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,YAAM,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,SAAS,OAAO,KAAK,QAAQ;AAEpE,WAAK,QAAQ,QAAQ,CAAC,WAAWC,uBAAa,QAAQ,MAAM,CAAC;AAC7D,WAAK,SAAS,QAAQ,CAAC,YAAYC,wBAAc,SAAS,MAAM,CAAC;AAEjE,UAAI,KAAK,SAAS;AAChB,qBAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,YAAY;AACd,WAAK,OAAO,SAAS,IAAIC,SAAAA,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,KAAKC,UAAAA;AAAAA,MACzC,KAAK,cAAA;AAAA,IAAc,GAClB;AACD,WAAK,gBAAgB,KAAK,OAAO,cAAc,MAAM;AAAA,IACvD;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAID,SAAAA,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,gBACE,KACA,OACA,cACA,QACM;AACN,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK;AAC5C,UAAM,QAAQ,KAAK,uBAAuB,QAAQ;AAElD,UAAM,UAAU,MAAM,eAAe,KAAK,OAAO,YAAY;AAC7DF,2BAAa,QAAQ,QAAQ,MAAM;AACnCC,4BAAc,QAAQ,SAAS,MAAM;AAGrC,SAAK,qBAAqB,UAAU,KAAK;AAAA,EAC3C;AACF;AAeO,SAAS,+BACd,YACA,SAC6C;AAC7C,SAAO,CACL,WACyC;AACzC,UAAM,SAAS,IAAIG,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAA6C;AAEnD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;;;"}
+{"version":3,"file":"groupedTopKWithFractionalIndex.cjs","sources":["../../../src/operators/groupedTopKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport {\n  TopKState,\n  handleMoveIn,\n  handleMoveOut,\n  topKBatch,\n} from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface GroupedTopKWithFractionalIndexOptions<K, T> {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n  /**\n   * Function to extract a group key from the element's key and value.\n   * Elements with the same group key will be sorted and limited together.\n   */\n  groupKeyFn: (key: K, value: T) => unknown\n}\n\n/**\n * Operator for grouped fractional indexed topK operations.\n * This operator maintains separate topK windows for each group,\n * allowing per-group limits and ordering.\n *\n * The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].\n * Elements are grouped by the groupKeyFn, and each group maintains\n * its own sorted collection with independent limit/offset.\n */\nexport class GroupedTopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #groupStates: Map<unknown, TopKState<K, T>> = new Map()\n  #groupKeyFn: (key: K, value: T) => unknown\n  #comparator: (a: [K, T], b: [K, T]) => number\n  #offset: number\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: GroupedTopKWithFractionalIndexOptions<K, T>,\n  ) {\n    super(id, inputA, output)\n    this.#groupKeyFn = options.groupKeyFn\n    this.#limit = options.limit ?? Infinity\n    this.#offset = options.offset ?? 0\n    this.#comparator = createKeyedComparator(comparator)\n    options.setSizeCallback?.(() => this.#getTotalSize())\n    options.setWindowFn?.(this.#moveTopK.bind(this))\n  }\n\n  #getTotalSize(): number {\n    let size = 0\n    for (const state of this.#groupStates.values()) {\n      size += state.size\n    }\n    return size\n  }\n\n  #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {\n    let state = this.#groupStates.get(groupKey)\n    if (!state) {\n      state = new TopKState(\n        new TopKArray(this.#offset, this.#limit, this.#comparator),\n      )\n      this.#groupStates.set(groupKey, state)\n    }\n    return state\n  }\n\n  #cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {\n    if (state.isEmpty) {\n      this.#groupStates.delete(groupKey)\n    }\n  }\n\n  /**\n   * Moves the topK window for all groups based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {\n    if (offset !== undefined) {\n      this.#offset = offset\n    }\n    if (limit !== undefined) {\n      this.#limit = limit\n    }\n\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    let hasChanges = false\n\n    for (const state of this.#groupStates.values()) {\n      const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit\n\n      diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n      diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n      if (diff.changes) {\n        hasChanges = true\n      }\n    }\n\n    if (hasChanges) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const [[key, value], multiplicity] of topKBatch(\n      this.inputMessages(),\n    )) {\n      this.#processElement(key, value, multiplicity, result)\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  #processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const groupKey = this.#groupKeyFn(key, value)\n    const state = this.#getOrCreateGroupState(groupKey)\n\n    const changes = state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n\n    // Cleanup empty groups to prevent memory leaks\n    this.#cleanupGroupIfEmpty(groupKey, state)\n  }\n}\n\n/**\n * Limits the number of results per group based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * This operator groups elements by the provided groupKeyFn and applies the limit/offset\n * independently to each group.\n *\n * @param comparator - A function that compares two elements for ordering\n * @param options - Configuration including groupKeyFn, limit, and offset\n * @returns A piped operator that orders elements per group and limits results per group\n */\nexport function groupedTopKWithFractionalIndex<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options: GroupedTopKWithFractionalIndexOptions<K, T>,\n): PipedOperator<[K, T], [K, IndexedValue<T>]> {\n  return (\n    stream: IStreamBuilder<[K, T]>,\n  ): IStreamBuilder<[K, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[K, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[K, IndexedValue<T>]>(),\n    )\n    const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      options,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":["UnaryOperator","createKeyedComparator","TopKState","TopKArray","handleMoveIn","handleMoveOut","MultiSet","topKBatch","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;;;;AAqCO,MAAM,+CAGHA,MAAAA,cAA4C;AAAA,EACpD,mCAAkD,IAAA;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,cAAc,QAAQ;AAC3B,SAAK,SAAS,QAAQ,SAAS;AAC/B,SAAK,UAAU,QAAQ,UAAU;AACjC,SAAK,cAAcC,UAAAA,sBAAsB,UAAU;AACnD,YAAQ,kBAAkB,MAAM,KAAK,cAAA,CAAe;AACpD,YAAQ,cAAc,KAAK,UAAU,KAAK,IAAI,CAAC;AAAA,EACjD;AAAA,EAEA,gBAAwB;AACtB,QAAI,OAAO;AACX,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,cAAQ,MAAM;AAAA,IAChB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,UAAoC;AACzD,QAAI,QAAQ,KAAK,aAAa,IAAI,QAAQ;AAC1C,QAAI,CAAC,OAAO;AACV,cAAQ,IAAIC,UAAAA;AAAAA,QACV,IAAIC,UAAAA,UAAU,KAAK,SAAS,KAAK,QAAQ,KAAK,WAAW;AAAA,MAAA;AAE3D,WAAK,aAAa,IAAI,UAAU,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,qBAAqB,UAAmB,OAA8B;AACpE,QAAI,MAAM,SAAS;AACjB,WAAK,aAAa,OAAO,QAAQ;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU,EAAE,QAAQ,SAAoD;AACtE,QAAI,WAAW,QAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,UAAU,QAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,SAAgD,CAAA;AACtD,QAAI,aAAa;AAEjB,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,YAAM,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,SAAS,OAAO,KAAK,QAAQ;AAEpE,WAAK,QAAQ,QAAQ,CAAC,WAAWC,uBAAa,QAAQ,MAAM,CAAC;AAC7D,WAAK,SAAS,QAAQ,CAAC,YAAYC,wBAAc,SAAS,MAAM,CAAC;AAEjE,UAAI,KAAK,SAAS;AAChB,qBAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,YAAY;AACd,WAAK,OAAO,SAAS,IAAIC,SAAAA,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,KAAKC,UAAAA;AAAAA,MACzC,KAAK,cAAA;AAAA,IAAc,GAClB;AACD,WAAK,gBAAgB,KAAK,OAAO,cAAc,MAAM;AAAA,IACvD;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAID,SAAAA,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,gBACE,KACA,OACA,cACA,QACM;AACN,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK;AAC5C,UAAM,QAAQ,KAAK,uBAAuB,QAAQ;AAElD,UAAM,UAAU,MAAM,eAAe,KAAK,OAAO,YAAY;AAC7DF,2BAAa,QAAQ,QAAQ,MAAM;AACnCC,4BAAc,QAAQ,SAAS,MAAM;AAGrC,SAAK,qBAAqB,UAAU,KAAK;AAAA,EAC3C;AACF;AAeO,SAAS,+BACd,YACA,SAC6C;AAC7C,SAAO,CACL,WACyC;AACzC,UAAM,SAAS,IAAIG,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAA6C;AAEnD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;;;"}

package/dist/cjs/operators/groupedTopKWithFractionalIndex.d.cts

@@ -1,5 +1,5 @@
 import { DifferenceStreamWriter, UnaryOperator, DifferenceStreamReader } from "../graph.cjs";
-import { IndexedValue, TopK } from "./topKArray.cjs";
+import { IndexedValue } from "./topKArray.cjs";
 import { PipedOperator } from "../types.cjs";
 export interface GroupedTopKWithFractionalIndexOptions<K, T> {
     limit?: number;
@@ -39,20 +39,6 @@ export declare class GroupedTopKWithFractionalIndexOperator<K extends string | n
         K,
         IndexedValue<T>
     ]>, comparator: (a: T, b: T) => number, options: GroupedTopKWithFractionalIndexOptions<K, T>);
-    /**
-     * Creates a new TopK data structure for a group.
-     * Can be overridden in subclasses to use different implementations (e.g., B+ tree).
-     */
-    protected createTopK(offset: number, limit: number, comparator: (a: [
-        K,
-        T
-    ], b: [
-        K,
-        T
-    ]) => number): TopK<[
-        K,
-        T
-    ]>;
     run(): void;
 }
 /**

package/dist/cjs/operators/index.d.cts

@@ -2,20 +2,14 @@ export * from "./pipe.cjs";
 export * from "./map.cjs";
 export * from "./tap.cjs";
 export * from "./filter.cjs";
-export * from "./negate.cjs";
 export * from "./concat.cjs";
-export * from "./debug.cjs";
 export * from "./output.cjs";
 export * from "./consolidate.cjs";
 export * from "./join.cjs";
 export * from "./reduce.cjs";
-export * from "./count.cjs";
 export * from "./distinct.cjs";
-export * from "./keying.cjs";
-export * from "./topK.cjs";
 export * from "./topKWithFractionalIndex.cjs";
 export * from "./groupedTopKWithFractionalIndex.cjs";
 export * from "./orderBy.cjs";
 export * from "./groupedOrderBy.cjs";
-export * from "./filterBy.cjs";
 export { groupBy, groupByOperators } from "./groupBy.cjs";

package/dist/cjs/operators/join.cjs

@@ -130,26 +130,6 @@ function join(other, type = `inner`) {
     return output;
   };
 }
-function innerJoin(other) {
-  return join(other, `inner`);
-}
-function antiJoin(other) {
-  return join(other, `anti`);
-}
-function leftJoin(other) {
-  return join(other, `left`);
-}
-function rightJoin(other) {
-  return join(other, `right`);
-}
-function fullJoin(other) {
-  return join(other, `full`);
-}
 exports.JoinOperator = JoinOperator;
-exports.antiJoin = antiJoin;
-exports.fullJoin = fullJoin;
-exports.innerJoin = innerJoin;
 exports.join = join;
-exports.leftJoin = leftJoin;
-exports.rightJoin = rightJoin;
 //# sourceMappingURL=join.cjs.map

package/dist/cjs/operators/join.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"join.cjs","sources":["../../../src/operators/join.ts"],"sourcesContent":["/**\n * # Direct Join Algorithms for Incremental View Maintenance\n *\n * High-performance join operations implementing all join types (inner, left, right, full, anti)\n * with minimal state and optimized performance.\n *\n * ## Algorithm\n *\n * For each tick, the algorithm processes incoming changes (deltas) and emits join results:\n *\n * 1. **Build deltas**: Create delta indexes from input messages using `Index.fromMultiSet()`\n * 2. **Inner results**: Emit `ΔA⋈B_old + A_old⋈ΔB + ΔA⋈ΔB` (matched pairs)\n * 3. **Outer results**: For unmatched rows, emit null-extended tuples:\n *    - New unmatched rows from deltas (when opposite side empty)\n *    - Presence transitions: when key goes `0→>0` (retract nulls) or `>0→0` (emit nulls)\n * 4. **Update state**: Append deltas to indexes (consolidated multiplicity tracking automatic)\n *\n * **Consolidated multiplicity tracking** enables O(1) presence checks instead of scanning index buckets.\n *\n * ## State\n *\n * **Indexes** store the actual data:\n * - `indexA: Index<K, V1>` - all left-side rows accumulated over time\n * - `indexB: Index<K, V2>` - all right-side rows accumulated over time\n *\n * **Consolidated multiplicity tracking** (built into Index):\n * - Each Index maintains sum of multiplicities per key internally\n * - Provides O(1) presence checks: `index.hasPresence(key)` and `index.getConsolidatedMultiplicity(key)`\n * - Avoids scanning entire index buckets just to check if key has any rows\n *\n * ## Join Types\n *\n * - **Inner**: Standard delta terms only\n * - **Outer**: Inner results + null-extended unmatched rows with transition handling\n * - **Anti**: Unmatched rows only (no inner results)\n *\n * ## Key Optimizations\n *\n * - **No temp copying**: Uses `(A⊎ΔA)⋈ΔB = A⋈ΔB ⊎ ΔA⋈ΔB` distributive property\n * - **Early-out checks**: Skip phases when no deltas present\n * - **Zero-entry pruning**: Keep maps compact, O(distinct keys) memory\n * - **Final presence logic**: Avoid emit→retract churn within same tick\n *\n * ## Correctness\n *\n * - **Ordering**: Pre-append snapshots for emissions, post-emit state updates\n * - **Presence**: Key matched iff mass ≠ 0, transitions trigger null handling\n * - **Bag semantics**: Proper multiplicity handling including negatives\n */\n\nimport { BinaryOperator, DifferenceStreamWriter } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport { Index } from '../indexes.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'\n\n/**\n * Type of join to perform\n */\nexport type JoinType = `inner` | `left` | `right` | `full` | `anti`\n\n/**\n * Operator that joins two input streams using direct join algorithms\n */\nexport class JoinOperator<K, V1, V2> extends BinaryOperator<\n  [K, V1] | [K, V2] | [K, [V1, V2]] | [K, [V1 | null, V2 | null]]\n> {\n  #indexA = new Index<K, V1>()\n  #indexB = new Index<K, V2>()\n  #mode: JoinType\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, V1]>,\n    inputB: DifferenceStreamReader<[K, V2]>,\n    output: DifferenceStreamWriter<any>,\n    mode: JoinType = `inner`,\n  ) {\n    super(id, inputA, inputB, output)\n    this.#mode = mode\n  }\n\n  run(): void {\n    // Build deltas from input messages\n    const deltaA = Index.fromMultiSets<K, V1>(\n      this.inputAMessages() as Array<MultiSet<[K, V1]>>,\n    )\n    const deltaB = Index.fromMultiSets<K, V2>(\n      this.inputBMessages() as Array<MultiSet<[K, V2]>>,\n    )\n\n    // Early-out if nothing changed\n    if (deltaA.size === 0 && deltaB.size === 0) return\n\n    const results = new MultiSet<any>()\n\n    // Emit inner results (all modes except anti)\n    if (this.#mode !== `anti`) {\n      this.emitInnerResults(deltaA, deltaB, results)\n    }\n\n    // Emit left outer/anti results\n    if (\n      this.#mode === `left` ||\n      this.#mode === `full` ||\n      this.#mode === `anti`\n    ) {\n      this.emitLeftOuterResults(deltaA, deltaB, results)\n    }\n\n    // Emit right outer results\n    if (this.#mode === `right` || this.#mode === `full`) {\n      this.emitRightOuterResults(deltaA, deltaB, results)\n    }\n\n    // Update state and send results\n    // IMPORTANT: All emissions use pre-append snapshots of indexA/indexB.\n    // Now append ALL deltas to indices - this happens unconditionally for every key,\n    // regardless of whether presence flipped. Consolidated multiplicity tracking is automatic.\n    this.#indexA.append(deltaA)\n    this.#indexB.append(deltaB)\n\n    // Send results\n    if (results.getInner().length > 0) {\n      this.output.sendData(results)\n    }\n  }\n\n  private emitInnerResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit the three standard delta terms: ΔA⋈B_old, A_old⋈ΔB, ΔA⋈ΔB\n    if (deltaA.size > 0) results.extend(deltaA.join(this.#indexB))\n    if (deltaB.size > 0) results.extend(this.#indexA.join(deltaB))\n    if (deltaA.size > 0 && deltaB.size > 0) results.extend(deltaA.join(deltaB))\n  }\n\n  private emitLeftOuterResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit unmatched left rows from deltaA\n    if (deltaA.size > 0) {\n      for (const [key, valueIterator] of deltaA.entriesIterators()) {\n        const currentMultiplicityB =\n          this.#indexB.getConsolidatedMultiplicity(key)\n        const deltaMultiplicityB = deltaB.getConsolidatedMultiplicity(key)\n        const finalMultiplicityB = currentMultiplicityB + deltaMultiplicityB\n\n        if (finalMultiplicityB === 0) {\n          for (const [value, multiplicity] of valueIterator) {\n            if (multiplicity !== 0) {\n              results.add([key, [value, null]], multiplicity)\n            }\n          }\n        }\n      }\n    }\n\n    // Handle presence transitions from right side changes\n    if (deltaB.size > 0) {\n      for (const key of deltaB.getPresenceKeys()) {\n        const before = this.#indexB.getConsolidatedMultiplicity(key)\n        const deltaMult = deltaB.getConsolidatedMultiplicity(key)\n        if (deltaMult === 0) continue\n        const after = before + deltaMult\n\n        // Skip transition handling if presence doesn't flip (both zero or both non-zero)\n        // Note: Index updates happen later regardless - we're only skipping null-extension emissions here\n        if ((before === 0) === (after === 0)) continue\n\n        // Determine the type of transition:\n        // - 0 → non-zero: Right becomes non-empty, left rows transition from unmatched to matched\n        //   → RETRACT previously emitted null-extended rows (emit with negative multiplicity)\n        // - non-zero → 0: Right becomes empty, left rows transition from matched to unmatched\n        //   → EMIT new null-extended rows (emit with positive multiplicity)\n        const transitioningToMatched = before === 0\n\n        for (const [value, multiplicity] of this.#indexA.getIterator(key)) {\n          if (multiplicity !== 0) {\n            results.add(\n              [key, [value, null]],\n              transitioningToMatched ? -multiplicity : +multiplicity,\n            )\n          }\n        }\n      }\n    }\n  }\n\n  private emitRightOuterResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit unmatched right rows from deltaB\n    if (deltaB.size > 0) {\n      for (const [key, valueIterator] of deltaB.entriesIterators()) {\n        const currentMultiplicityA =\n          this.#indexA.getConsolidatedMultiplicity(key)\n        const deltaMultiplicityA = deltaA.getConsolidatedMultiplicity(key)\n        const finalMultiplicityA = currentMultiplicityA + deltaMultiplicityA\n\n        if (finalMultiplicityA === 0) {\n          for (const [value, multiplicity] of valueIterator) {\n            if (multiplicity !== 0) {\n              results.add([key, [null, value]], multiplicity)\n            }\n          }\n        }\n      }\n    }\n\n    // Handle presence transitions from left side changes\n    if (deltaA.size > 0) {\n      for (const key of deltaA.getPresenceKeys()) {\n        const before = this.#indexA.getConsolidatedMultiplicity(key)\n        const deltaMult = deltaA.getConsolidatedMultiplicity(key)\n        if (deltaMult === 0) continue\n        const after = before + deltaMult\n\n        // Skip transition handling if presence doesn't flip (both zero or both non-zero)\n        // Note: Index updates happen later regardless - we're only skipping null-extension emissions here\n        if ((before === 0) === (after === 0)) continue\n\n        // Determine the type of transition:\n        // - 0 → non-zero: Left becomes non-empty, right rows transition from unmatched to matched\n        //   → RETRACT previously emitted null-extended rows (emit with negative multiplicity)\n        // - non-zero → 0: Left becomes empty, right rows transition from matched to unmatched\n        //   → EMIT new null-extended rows (emit with positive multiplicity)\n        const transitioningToMatched = before === 0\n\n        for (const [value, multiplicity] of this.#indexB.getIterator(key)) {\n          if (multiplicity !== 0) {\n            results.add(\n              [key, [null, value]],\n              transitioningToMatched ? -multiplicity : +multiplicity,\n            )\n          }\n        }\n      }\n    }\n  }\n}\n\n/**\n * Joins two input streams\n * @param other - The other stream to join with\n * @param type - The type of join to perform\n */\nexport function join<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n  type: JoinType = `inner`,\n): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {\n  return (\n    stream: IStreamBuilder<T>,\n  ): IStreamBuilder<KeyValue<K, [V1 | null, V2 | null]>> => {\n    if (stream.graph !== other.graph) {\n      throw new Error(`Cannot join streams from different graphs`)\n    }\n    const output = new StreamBuilder<KeyValue<K, [V1 | null, V2 | null]>>(\n      stream.graph,\n      new DifferenceStreamWriter<KeyValue<K, [V1 | null, V2 | null]>>(),\n    )\n    const operator = new JoinOperator<K, V1, V2>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader() as DifferenceStreamReader<KeyValue<K, V1>>,\n      other.connectReader(),\n      output.writer,\n      type,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n\n/**\n * Joins two input streams (inner join)\n * @param other - The other stream to join with\n */\nexport function innerJoin<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n): PipedOperator<T, KeyValue<K, [V1, V2]>> {\n  return join(other, `inner`) as unknown as PipedOperator<\n    T,\n    KeyValue<K, [V1, V2]>\n  >\n}\n\n/**\n * Joins two input streams (anti join)\n * @param other - The other stream to join with\n */\nexport function antiJoin<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n): PipedOperator<T, KeyValue<K, [V1, null]>> {\n  return join(other, `anti`) as unknown as PipedOperator<\n    T,\n    KeyValue<K, [V1, null]>\n  >\n}\n\n/**\n * Joins two input streams (left join)\n * @param other - The other stream to join with\n */\nexport function leftJoin<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n): PipedOperator<T, KeyValue<K, [V1, V2 | null]>> {\n  return join(other, `left`) as unknown as PipedOperator<\n    T,\n    KeyValue<K, [V1, V2 | null]>\n  >\n}\n\n/**\n * Joins two input streams (right join)\n * @param other - The other stream to join with\n */\nexport function rightJoin<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n): PipedOperator<T, KeyValue<K, [V1 | null, V2]>> {\n  return join(other, `right`) as unknown as PipedOperator<\n    T,\n    KeyValue<K, [V1 | null, V2]>\n  >\n}\n\n/**\n * Joins two input streams (full join)\n * @param other - The other stream to join with\n */\nexport function fullJoin<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {\n  return join(other, `full`) as unknown as PipedOperator<\n    T,\n    KeyValue<K, [V1 | null, V2 | null]>\n  >\n}\n"],"names":["BinaryOperator","Index","MultiSet","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;;;AAiEO,MAAM,qBAAgCA,MAAAA,eAE3C;AAAA,EACA,UAAU,IAAIC,QAAAA,MAAA;AAAA,EACd,UAAU,IAAIA,QAAAA,MAAA;AAAA,EACd;AAAA,EAEA,YACE,IACA,QACA,QACA,QACA,OAAiB,SACjB;AACA,UAAM,IAAI,QAAQ,QAAQ,MAAM;AAChC,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,MAAY;AAEV,UAAM,SAASA,QAAAA,MAAM;AAAA,MACnB,KAAK,eAAA;AAAA,IAAe;AAEtB,UAAM,SAASA,QAAAA,MAAM;AAAA,MACnB,KAAK,eAAA;AAAA,IAAe;AAItB,QAAI,OAAO,SAAS,KAAK,OAAO,SAAS,EAAG;AAE5C,UAAM,UAAU,IAAIC,kBAAA;AAGpB,QAAI,KAAK,UAAU,QAAQ;AACzB,WAAK,iBAAiB,QAAQ,QAAQ,OAAO;AAAA,IAC/C;AAGA,QACE,KAAK,UAAU,UACf,KAAK,UAAU,UACf,KAAK,UAAU,QACf;AACA,WAAK,qBAAqB,QAAQ,QAAQ,OAAO;AAAA,IACnD;AAGA,QAAI,KAAK,UAAU,WAAW,KAAK,UAAU,QAAQ;AACnD,WAAK,sBAAsB,QAAQ,QAAQ,OAAO;AAAA,IACpD;AAMA,SAAK,QAAQ,OAAO,MAAM;AAC1B,SAAK,QAAQ,OAAO,MAAM;AAG1B,QAAI,QAAQ,WAAW,SAAS,GAAG;AACjC,WAAK,OAAO,SAAS,OAAO;AAAA,IAC9B;AAAA,EACF;AAAA,EAEQ,iBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,EAAG,SAAQ,OAAO,OAAO,KAAK,KAAK,OAAO,CAAC;AAC7D,QAAI,OAAO,OAAO,EAAG,SAAQ,OAAO,KAAK,QAAQ,KAAK,MAAM,CAAC;AAC7D,QAAI,OAAO,OAAO,KAAK,OAAO,OAAO,EAAG,SAAQ,OAAO,OAAO,KAAK,MAAM,CAAC;AAAA,EAC5E;AAAA,EAEQ,qBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,CAAC,KAAK,aAAa,KAAK,OAAO,oBAAoB;AAC5D,cAAM,uBACJ,KAAK,QAAQ,4BAA4B,GAAG;AAC9C,cAAM,qBAAqB,OAAO,4BAA4B,GAAG;AACjE,cAAM,qBAAqB,uBAAuB;AAElD,YAAI,uBAAuB,GAAG;AAC5B,qBAAW,CAAC,OAAO,YAAY,KAAK,eAAe;AACjD,gBAAI,iBAAiB,GAAG;AACtB,sBAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,GAAG,YAAY;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,OAAO,OAAO,mBAAmB;AAC1C,cAAM,SAAS,KAAK,QAAQ,4BAA4B,GAAG;AAC3D,cAAM,YAAY,OAAO,4BAA4B,GAAG;AACxD,YAAI,cAAc,EAAG;AACrB,cAAM,QAAQ,SAAS;AAIvB,YAAK,WAAW,OAAQ,UAAU,GAAI;AAOtC,cAAM,yBAAyB,WAAW;AAE1C,mBAAW,CAAC,OAAO,YAAY,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG;AACjE,cAAI,iBAAiB,GAAG;AACtB,oBAAQ;AAAA,cACN,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,cACnB,yBAAyB,CAAC,eAAe,CAAC;AAAA,YAAA;AAAA,UAE9C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,sBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,CAAC,KAAK,aAAa,KAAK,OAAO,oBAAoB;AAC5D,cAAM,uBACJ,KAAK,QAAQ,4BAA4B,GAAG;AAC9C,cAAM,qBAAqB,OAAO,4BAA4B,GAAG;AACjE,cAAM,qBAAqB,uBAAuB;AAElD,YAAI,uBAAuB,GAAG;AAC5B,qBAAW,CAAC,OAAO,YAAY,KAAK,eAAe;AACjD,gBAAI,iBAAiB,GAAG;AACtB,sBAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,GAAG,YAAY;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,OAAO,OAAO,mBAAmB;AAC1C,cAAM,SAAS,KAAK,QAAQ,4BAA4B,GAAG;AAC3D,cAAM,YAAY,OAAO,4BAA4B,GAAG;AACxD,YAAI,cAAc,EAAG;AACrB,cAAM,QAAQ,SAAS;AAIvB,YAAK,WAAW,OAAQ,UAAU,GAAI;AAOtC,cAAM,yBAAyB,WAAW;AAE1C,mBAAW,CAAC,OAAO,YAAY,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG;AACjE,cAAI,iBAAiB,GAAG;AACtB,oBAAQ;AAAA,cACN,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;AAAA,cACnB,yBAAyB,CAAC,eAAe,CAAC;AAAA,YAAA;AAAA,UAE9C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAOO,SAAS,KAMd,OACA,OAAiB,SACsC;AACvD,SAAO,CACL,WACwD;AACxD,QAAI,OAAO,UAAU,MAAM,OAAO;AAChC,YAAM,IAAI,MAAM,2CAA2C;AAAA,IAC7D;AACA,UAAM,SAAS,IAAIC,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAA4D;AAElE,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,MAAM,cAAA;AAAA,MACN,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;AAMO,SAAS,UAMd,OACyC;AACzC,SAAO,KAAK,OAAO,OAAO;AAI5B;AAMO,SAAS,SAMd,OAC2C;AAC3C,SAAO,KAAK,OAAO,MAAM;AAI3B;AAMO,SAAS,SAMd,OACgD;AAChD,SAAO,KAAK,OAAO,MAAM;AAI3B;AAMO,SAAS,UAMd,OACgD;AAChD,SAAO,KAAK,OAAO,OAAO;AAI5B;AAMO,SAAS,SAMd,OACuD;AACvD,SAAO,KAAK,OAAO,MAAM;AAI3B;;;;;;;;"}
+{"version":3,"file":"join.cjs","sources":["../../../src/operators/join.ts"],"sourcesContent":["/**\n * # Direct Join Algorithms for Incremental View Maintenance\n *\n * High-performance join operations implementing all join types (inner, left, right, full, anti)\n * with minimal state and optimized performance.\n *\n * ## Algorithm\n *\n * For each tick, the algorithm processes incoming changes (deltas) and emits join results:\n *\n * 1. **Build deltas**: Create delta indexes from input messages using `Index.fromMultiSet()`\n * 2. **Inner results**: Emit `ΔA⋈B_old + A_old⋈ΔB + ΔA⋈ΔB` (matched pairs)\n * 3. **Outer results**: For unmatched rows, emit null-extended tuples:\n *    - New unmatched rows from deltas (when opposite side empty)\n *    - Presence transitions: when key goes `0→>0` (retract nulls) or `>0→0` (emit nulls)\n * 4. **Update state**: Append deltas to indexes (consolidated multiplicity tracking automatic)\n *\n * **Consolidated multiplicity tracking** enables O(1) presence checks instead of scanning index buckets.\n *\n * ## State\n *\n * **Indexes** store the actual data:\n * - `indexA: Index<K, V1>` - all left-side rows accumulated over time\n * - `indexB: Index<K, V2>` - all right-side rows accumulated over time\n *\n * **Consolidated multiplicity tracking** (built into Index):\n * - Each Index maintains sum of multiplicities per key internally\n * - Provides O(1) presence checks: `index.hasPresence(key)` and `index.getConsolidatedMultiplicity(key)`\n * - Avoids scanning entire index buckets just to check if key has any rows\n *\n * ## Join Types\n *\n * - **Inner**: Standard delta terms only\n * - **Outer**: Inner results + null-extended unmatched rows with transition handling\n * - **Anti**: Unmatched rows only (no inner results)\n *\n * ## Key Optimizations\n *\n * - **No temp copying**: Uses `(A⊎ΔA)⋈ΔB = A⋈ΔB ⊎ ΔA⋈ΔB` distributive property\n * - **Early-out checks**: Skip phases when no deltas present\n * - **Zero-entry pruning**: Keep maps compact, O(distinct keys) memory\n * - **Final presence logic**: Avoid emit→retract churn within same tick\n *\n * ## Correctness\n *\n * - **Ordering**: Pre-append snapshots for emissions, post-emit state updates\n * - **Presence**: Key matched iff mass ≠ 0, transitions trigger null handling\n * - **Bag semantics**: Proper multiplicity handling including negatives\n */\n\nimport { BinaryOperator, DifferenceStreamWriter } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport { Index } from '../indexes.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'\n\n/**\n * Type of join to perform\n */\nexport type JoinType = `inner` | `left` | `right` | `full` | `anti`\n\n/**\n * Operator that joins two input streams using direct join algorithms\n */\nexport class JoinOperator<K, V1, V2> extends BinaryOperator<\n  [K, V1] | [K, V2] | [K, [V1, V2]] | [K, [V1 | null, V2 | null]]\n> {\n  #indexA = new Index<K, V1>()\n  #indexB = new Index<K, V2>()\n  #mode: JoinType\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, V1]>,\n    inputB: DifferenceStreamReader<[K, V2]>,\n    output: DifferenceStreamWriter<any>,\n    mode: JoinType = `inner`,\n  ) {\n    super(id, inputA, inputB, output)\n    this.#mode = mode\n  }\n\n  run(): void {\n    // Build deltas from input messages\n    const deltaA = Index.fromMultiSets<K, V1>(\n      this.inputAMessages() as Array<MultiSet<[K, V1]>>,\n    )\n    const deltaB = Index.fromMultiSets<K, V2>(\n      this.inputBMessages() as Array<MultiSet<[K, V2]>>,\n    )\n\n    // Early-out if nothing changed\n    if (deltaA.size === 0 && deltaB.size === 0) return\n\n    const results = new MultiSet<any>()\n\n    // Emit inner results (all modes except anti)\n    if (this.#mode !== `anti`) {\n      this.emitInnerResults(deltaA, deltaB, results)\n    }\n\n    // Emit left outer/anti results\n    if (\n      this.#mode === `left` ||\n      this.#mode === `full` ||\n      this.#mode === `anti`\n    ) {\n      this.emitLeftOuterResults(deltaA, deltaB, results)\n    }\n\n    // Emit right outer results\n    if (this.#mode === `right` || this.#mode === `full`) {\n      this.emitRightOuterResults(deltaA, deltaB, results)\n    }\n\n    // Update state and send results\n    // IMPORTANT: All emissions use pre-append snapshots of indexA/indexB.\n    // Now append ALL deltas to indices - this happens unconditionally for every key,\n    // regardless of whether presence flipped. Consolidated multiplicity tracking is automatic.\n    this.#indexA.append(deltaA)\n    this.#indexB.append(deltaB)\n\n    // Send results\n    if (results.getInner().length > 0) {\n      this.output.sendData(results)\n    }\n  }\n\n  private emitInnerResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit the three standard delta terms: ΔA⋈B_old, A_old⋈ΔB, ΔA⋈ΔB\n    if (deltaA.size > 0) results.extend(deltaA.join(this.#indexB))\n    if (deltaB.size > 0) results.extend(this.#indexA.join(deltaB))\n    if (deltaA.size > 0 && deltaB.size > 0) results.extend(deltaA.join(deltaB))\n  }\n\n  private emitLeftOuterResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit unmatched left rows from deltaA\n    if (deltaA.size > 0) {\n      for (const [key, valueIterator] of deltaA.entriesIterators()) {\n        const currentMultiplicityB =\n          this.#indexB.getConsolidatedMultiplicity(key)\n        const deltaMultiplicityB = deltaB.getConsolidatedMultiplicity(key)\n        const finalMultiplicityB = currentMultiplicityB + deltaMultiplicityB\n\n        if (finalMultiplicityB === 0) {\n          for (const [value, multiplicity] of valueIterator) {\n            if (multiplicity !== 0) {\n              results.add([key, [value, null]], multiplicity)\n            }\n          }\n        }\n      }\n    }\n\n    // Handle presence transitions from right side changes\n    if (deltaB.size > 0) {\n      for (const key of deltaB.getPresenceKeys()) {\n        const before = this.#indexB.getConsolidatedMultiplicity(key)\n        const deltaMult = deltaB.getConsolidatedMultiplicity(key)\n        if (deltaMult === 0) continue\n        const after = before + deltaMult\n\n        // Skip transition handling if presence doesn't flip (both zero or both non-zero)\n        // Note: Index updates happen later regardless - we're only skipping null-extension emissions here\n        if ((before === 0) === (after === 0)) continue\n\n        // Determine the type of transition:\n        // - 0 → non-zero: Right becomes non-empty, left rows transition from unmatched to matched\n        //   → RETRACT previously emitted null-extended rows (emit with negative multiplicity)\n        // - non-zero → 0: Right becomes empty, left rows transition from matched to unmatched\n        //   → EMIT new null-extended rows (emit with positive multiplicity)\n        const transitioningToMatched = before === 0\n\n        for (const [value, multiplicity] of this.#indexA.getIterator(key)) {\n          if (multiplicity !== 0) {\n            results.add(\n              [key, [value, null]],\n              transitioningToMatched ? -multiplicity : +multiplicity,\n            )\n          }\n        }\n      }\n    }\n  }\n\n  private emitRightOuterResults(\n    deltaA: Index<K, V1>,\n    deltaB: Index<K, V2>,\n    results: MultiSet<any>,\n  ): void {\n    // Emit unmatched right rows from deltaB\n    if (deltaB.size > 0) {\n      for (const [key, valueIterator] of deltaB.entriesIterators()) {\n        const currentMultiplicityA =\n          this.#indexA.getConsolidatedMultiplicity(key)\n        const deltaMultiplicityA = deltaA.getConsolidatedMultiplicity(key)\n        const finalMultiplicityA = currentMultiplicityA + deltaMultiplicityA\n\n        if (finalMultiplicityA === 0) {\n          for (const [value, multiplicity] of valueIterator) {\n            if (multiplicity !== 0) {\n              results.add([key, [null, value]], multiplicity)\n            }\n          }\n        }\n      }\n    }\n\n    // Handle presence transitions from left side changes\n    if (deltaA.size > 0) {\n      for (const key of deltaA.getPresenceKeys()) {\n        const before = this.#indexA.getConsolidatedMultiplicity(key)\n        const deltaMult = deltaA.getConsolidatedMultiplicity(key)\n        if (deltaMult === 0) continue\n        const after = before + deltaMult\n\n        // Skip transition handling if presence doesn't flip (both zero or both non-zero)\n        // Note: Index updates happen later regardless - we're only skipping null-extension emissions here\n        if ((before === 0) === (after === 0)) continue\n\n        // Determine the type of transition:\n        // - 0 → non-zero: Left becomes non-empty, right rows transition from unmatched to matched\n        //   → RETRACT previously emitted null-extended rows (emit with negative multiplicity)\n        // - non-zero → 0: Left becomes empty, right rows transition from matched to unmatched\n        //   → EMIT new null-extended rows (emit with positive multiplicity)\n        const transitioningToMatched = before === 0\n\n        for (const [value, multiplicity] of this.#indexB.getIterator(key)) {\n          if (multiplicity !== 0) {\n            results.add(\n              [key, [null, value]],\n              transitioningToMatched ? -multiplicity : +multiplicity,\n            )\n          }\n        }\n      }\n    }\n  }\n}\n\n/**\n * Joins two input streams\n * @param other - The other stream to join with\n * @param type - The type of join to perform\n */\nexport function join<\n  K,\n  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),\n  V2,\n  T,\n>(\n  other: IStreamBuilder<KeyValue<K, V2>>,\n  type: JoinType = `inner`,\n): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {\n  return (\n    stream: IStreamBuilder<T>,\n  ): IStreamBuilder<KeyValue<K, [V1 | null, V2 | null]>> => {\n    if (stream.graph !== other.graph) {\n      throw new Error(`Cannot join streams from different graphs`)\n    }\n    const output = new StreamBuilder<KeyValue<K, [V1 | null, V2 | null]>>(\n      stream.graph,\n      new DifferenceStreamWriter<KeyValue<K, [V1 | null, V2 | null]>>(),\n    )\n    const operator = new JoinOperator<K, V1, V2>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader() as DifferenceStreamReader<KeyValue<K, V1>>,\n      other.connectReader(),\n      output.writer,\n      type,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":["BinaryOperator","Index","MultiSet","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;;;AAiEO,MAAM,qBAAgCA,MAAAA,eAE3C;AAAA,EACA,UAAU,IAAIC,QAAAA,MAAA;AAAA,EACd,UAAU,IAAIA,QAAAA,MAAA;AAAA,EACd;AAAA,EAEA,YACE,IACA,QACA,QACA,QACA,OAAiB,SACjB;AACA,UAAM,IAAI,QAAQ,QAAQ,MAAM;AAChC,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,MAAY;AAEV,UAAM,SAASA,QAAAA,MAAM;AAAA,MACnB,KAAK,eAAA;AAAA,IAAe;AAEtB,UAAM,SAASA,QAAAA,MAAM;AAAA,MACnB,KAAK,eAAA;AAAA,IAAe;AAItB,QAAI,OAAO,SAAS,KAAK,OAAO,SAAS,EAAG;AAE5C,UAAM,UAAU,IAAIC,kBAAA;AAGpB,QAAI,KAAK,UAAU,QAAQ;AACzB,WAAK,iBAAiB,QAAQ,QAAQ,OAAO;AAAA,IAC/C;AAGA,QACE,KAAK,UAAU,UACf,KAAK,UAAU,UACf,KAAK,UAAU,QACf;AACA,WAAK,qBAAqB,QAAQ,QAAQ,OAAO;AAAA,IACnD;AAGA,QAAI,KAAK,UAAU,WAAW,KAAK,UAAU,QAAQ;AACnD,WAAK,sBAAsB,QAAQ,QAAQ,OAAO;AAAA,IACpD;AAMA,SAAK,QAAQ,OAAO,MAAM;AAC1B,SAAK,QAAQ,OAAO,MAAM;AAG1B,QAAI,QAAQ,WAAW,SAAS,GAAG;AACjC,WAAK,OAAO,SAAS,OAAO;AAAA,IAC9B;AAAA,EACF;AAAA,EAEQ,iBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,EAAG,SAAQ,OAAO,OAAO,KAAK,KAAK,OAAO,CAAC;AAC7D,QAAI,OAAO,OAAO,EAAG,SAAQ,OAAO,KAAK,QAAQ,KAAK,MAAM,CAAC;AAC7D,QAAI,OAAO,OAAO,KAAK,OAAO,OAAO,EAAG,SAAQ,OAAO,OAAO,KAAK,MAAM,CAAC;AAAA,EAC5E;AAAA,EAEQ,qBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,CAAC,KAAK,aAAa,KAAK,OAAO,oBAAoB;AAC5D,cAAM,uBACJ,KAAK,QAAQ,4BAA4B,GAAG;AAC9C,cAAM,qBAAqB,OAAO,4BAA4B,GAAG;AACjE,cAAM,qBAAqB,uBAAuB;AAElD,YAAI,uBAAuB,GAAG;AAC5B,qBAAW,CAAC,OAAO,YAAY,KAAK,eAAe;AACjD,gBAAI,iBAAiB,GAAG;AACtB,sBAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,GAAG,YAAY;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,OAAO,OAAO,mBAAmB;AAC1C,cAAM,SAAS,KAAK,QAAQ,4BAA4B,GAAG;AAC3D,cAAM,YAAY,OAAO,4BAA4B,GAAG;AACxD,YAAI,cAAc,EAAG;AACrB,cAAM,QAAQ,SAAS;AAIvB,YAAK,WAAW,OAAQ,UAAU,GAAI;AAOtC,cAAM,yBAAyB,WAAW;AAE1C,mBAAW,CAAC,OAAO,YAAY,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG;AACjE,cAAI,iBAAiB,GAAG;AACtB,oBAAQ;AAAA,cACN,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,cACnB,yBAAyB,CAAC,eAAe,CAAC;AAAA,YAAA;AAAA,UAE9C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,sBACN,QACA,QACA,SACM;AAEN,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,CAAC,KAAK,aAAa,KAAK,OAAO,oBAAoB;AAC5D,cAAM,uBACJ,KAAK,QAAQ,4BAA4B,GAAG;AAC9C,cAAM,qBAAqB,OAAO,4BAA4B,GAAG;AACjE,cAAM,qBAAqB,uBAAuB;AAElD,YAAI,uBAAuB,GAAG;AAC5B,qBAAW,CAAC,OAAO,YAAY,KAAK,eAAe;AACjD,gBAAI,iBAAiB,GAAG;AACtB,sBAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,GAAG,YAAY;AAAA,YAChD;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,OAAO,OAAO,GAAG;AACnB,iBAAW,OAAO,OAAO,mBAAmB;AAC1C,cAAM,SAAS,KAAK,QAAQ,4BAA4B,GAAG;AAC3D,cAAM,YAAY,OAAO,4BAA4B,GAAG;AACxD,YAAI,cAAc,EAAG;AACrB,cAAM,QAAQ,SAAS;AAIvB,YAAK,WAAW,OAAQ,UAAU,GAAI;AAOtC,cAAM,yBAAyB,WAAW;AAE1C,mBAAW,CAAC,OAAO,YAAY,KAAK,KAAK,QAAQ,YAAY,GAAG,GAAG;AACjE,cAAI,iBAAiB,GAAG;AACtB,oBAAQ;AAAA,cACN,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;AAAA,cACnB,yBAAyB,CAAC,eAAe,CAAC;AAAA,YAAA;AAAA,UAE9C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAOO,SAAS,KAMd,OACA,OAAiB,SACsC;AACvD,SAAO,CACL,WACwD;AACxD,QAAI,OAAO,UAAU,MAAM,OAAO;AAChC,YAAM,IAAI,MAAM,2CAA2C;AAAA,IAC7D;AACA,UAAM,SAAS,IAAIC,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAA4D;AAElE,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,MAAM,cAAA;AAAA,MACN,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;;;"}

package/dist/cjs/operators/join.d.cts

@@ -48,43 +48,3 @@ export declare function join<K, V1 extends (T extends KeyValue<infer _KT, infer
     V1 | null,
     V2 | null
 ]>>;
-/**
- * Joins two input streams (inner join)
- * @param other - The other stream to join with
- */
-export declare function innerJoin<K, V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never), V2, T>(other: IStreamBuilder<KeyValue<K, V2>>): PipedOperator<T, KeyValue<K, [
-    V1,
-    V2
-]>>;
-/**
- * Joins two input streams (anti join)
- * @param other - The other stream to join with
- */
-export declare function antiJoin<K, V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never), V2, T>(other: IStreamBuilder<KeyValue<K, V2>>): PipedOperator<T, KeyValue<K, [
-    V1,
-    null
-]>>;
-/**
- * Joins two input streams (left join)
- * @param other - The other stream to join with
- */
-export declare function leftJoin<K, V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never), V2, T>(other: IStreamBuilder<KeyValue<K, V2>>): PipedOperator<T, KeyValue<K, [
-    V1,
-    V2 | null
-]>>;
-/**
- * Joins two input streams (right join)
- * @param other - The other stream to join with
- */
-export declare function rightJoin<K, V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never), V2, T>(other: IStreamBuilder<KeyValue<K, V2>>): PipedOperator<T, KeyValue<K, [
-    V1 | null,
-    V2
-]>>;
-/**
- * Joins two input streams (full join)
- * @param other - The other stream to join with
- */
-export declare function fullJoin<K, V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never), V2, T>(other: IStreamBuilder<KeyValue<K, V2>>): PipedOperator<T, KeyValue<K, [
-    V1 | null,
-    V2 | null
-]>>;