@tanstack/db-ivm 0.1.9 → 0.1.10

package/dist/esm/operators/reduce.js

@@ -1,37 +1,29 @@
-var __typeError = (msg) => {
-  throw TypeError(msg);
-};
-var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
-var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
-var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
-var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
-var _index, _indexOut, _f;
 import { UnaryOperator, DifferenceStreamWriter } from "../graph.js";
 import { StreamBuilder } from "../d2.js";
 import { MultiSet } from "../multiset.js";
 import { Index } from "../indexes.js";
 class ReduceOperator extends UnaryOperator {
+  #index = new Index();
+  #indexOut = new Index();
+  #f;
   constructor(id, inputA, output, f) {
     super(id, inputA, output);
-    __privateAdd(this, _index, new Index());
-    __privateAdd(this, _indexOut, new Index());
-    __privateAdd(this, _f);
-    __privateSet(this, _f, f);
+    this.#f = f;
   }
   run() {
     const keysTodo = /* @__PURE__ */ new Set();
     for (const message of this.inputMessages()) {
       for (const [item, multiplicity] of message.getInner()) {
         const [key, value] = item;
-        __privateGet(this, _index).addValue(key, [value, multiplicity]);
+        this.#index.addValue(key, [value, multiplicity]);
         keysTodo.add(key);
       }
     }
     const result = [];
     for (const key of keysTodo) {
-      const curr = __privateGet(this, _index).get(key);
-      const currOut = __privateGet(this, _indexOut).get(key);
-      const out = __privateGet(this, _f).call(this, curr);
+      const curr = this.#index.get(key);
+      const currOut = this.#indexOut.get(key);
+      const out = this.#f(curr);
       const newOutputMap = /* @__PURE__ */ new Map();
       const oldOutputMap = /* @__PURE__ */ new Map();
       for (const [value, multiplicity] of out) {
@@ -45,14 +37,14 @@ class ReduceOperator extends UnaryOperator {
       for (const [value, multiplicity] of oldOutputMap) {
         if (!newOutputMap.has(value)) {
           result.push([[key, value], -multiplicity]);
-          __privateGet(this, _indexOut).addValue(key, [value, -multiplicity]);
+          this.#indexOut.addValue(key, [value, -multiplicity]);
         }
       }
       for (const [value, multiplicity] of newOutputMap) {
         if (!oldOutputMap.has(value)) {
           if (multiplicity !== 0) {
             result.push([[key, value], multiplicity]);
-            __privateGet(this, _indexOut).addValue(key, [value, multiplicity]);
+            this.#indexOut.addValue(key, [value, multiplicity]);
           }
         }
       }
@@ -62,7 +54,7 @@ class ReduceOperator extends UnaryOperator {
           const delta = newMultiplicity - oldMultiplicity;
           if (delta !== 0) {
             result.push([[key, value], delta]);
-            __privateGet(this, _indexOut).addValue(key, [value, delta]);
+            this.#indexOut.addValue(key, [value, delta]);
           }
         }
       }
@@ -72,9 +64,6 @@ class ReduceOperator extends UnaryOperator {
     }
   }
 }
-_index = new WeakMap();
-_indexOut = new WeakMap();
-_f = new WeakMap();
 function reduce(f) {
   return (stream) => {
     const output = new StreamBuilder(

package/dist/esm/operators/reduce.js.map

@@ -1 +1 @@
-{"version":3,"file":"reduce.js","sources":["../../../src/operators/reduce.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport { MultiSet } from \"../multiset.js\"\nimport { Index } from \"../indexes.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { IStreamBuilder, KeyValue } from \"../types.js\"\n\n/**\n * Base operator for reduction operations (version-free)\n */\nexport class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {\n  #index = new Index<K, V1>()\n  #indexOut = new Index<K, V2>()\n  #f: (values: Array<[V1, number]>) => Array<[V2, number]>\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, V1]>,\n    output: DifferenceStreamWriter<[K, V2]>,\n    f: (values: Array<[V1, number]>) => Array<[V2, number]>\n  ) {\n    super(id, inputA, output)\n    this.#f = f\n  }\n\n  run(): void {\n    // Collect all input messages and update the index\n    const keysTodo = new Set<K>()\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.#index.addValue(key, [value, multiplicity])\n        keysTodo.add(key)\n      }\n    }\n\n    // For each key, compute the reduction and delta\n    const result: Array<[[K, V2], number]> = []\n    for (const key of keysTodo) {\n      const curr = this.#index.get(key)\n      const currOut = this.#indexOut.get(key)\n      const out = this.#f(curr)\n\n      // Create maps for current and previous outputs using values directly as keys\n      const newOutputMap = new Map<V2, number>()\n      const oldOutputMap = new Map<V2, number>()\n\n      // Process new output\n      for (const [value, multiplicity] of out) {\n        const existing = newOutputMap.get(value) ?? 0\n        newOutputMap.set(value, existing + multiplicity)\n      }\n\n      // Process previous output\n      for (const [value, multiplicity] of currOut) {\n        const existing = oldOutputMap.get(value) ?? 0\n        oldOutputMap.set(value, existing + multiplicity)\n      }\n\n      // First, emit removals for old values that are no longer present\n      for (const [value, multiplicity] of oldOutputMap) {\n        if (!newOutputMap.has(value)) {\n          // Remove the old value entirely\n          result.push([[key, value], -multiplicity])\n          this.#indexOut.addValue(key, [value, -multiplicity])\n        }\n      }\n\n      // Then, emit additions for new values that are not present in old\n      for (const [value, multiplicity] of newOutputMap) {\n        if (!oldOutputMap.has(value)) {\n          // Add the new value only if it has non-zero multiplicity\n          if (multiplicity !== 0) {\n            result.push([[key, value], multiplicity])\n            this.#indexOut.addValue(key, [value, multiplicity])\n          }\n        }\n      }\n\n      // Finally, emit multiplicity changes for values that were present and are still present\n      for (const [value, newMultiplicity] of newOutputMap) {\n        const oldMultiplicity = oldOutputMap.get(value)\n        if (oldMultiplicity !== undefined) {\n          const delta = newMultiplicity - oldMultiplicity\n          // Only emit actual changes, i.e. non-zero deltas\n          if (delta !== 0) {\n            result.push([[key, value], delta])\n            this.#indexOut.addValue(key, [value, delta])\n          }\n        }\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n}\n\n/**\n * Reduces the elements in the stream by key (version-free)\n */\nexport function reduce<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  R,\n  T,\n>(f: (values: Array<[V1Type, number]>) => Array<[R, number]>) {\n  return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<KType, R>> => {\n    const output = new StreamBuilder<KeyValue<KType, R>>(\n      stream.graph,\n      new DifferenceStreamWriter<KeyValue<KType, R>>()\n    )\n    const operator = new ReduceOperator<KType, V1Type, R>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader() as DifferenceStreamReader<KeyValue<KType, V1Type>>,\n      output.writer,\n      f\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAUO,MAAM,uBAAkC,cAAgC;AAAA,EAK7E,YACE,IACA,QACA,QACA,GACA;AACA,UAAM,IAAI,QAAQ,MAAM;AAV1B,+BAAS,IAAI,MAAA;AACb,kCAAY,IAAI,MAAA;AAChB;AASE,uBAAK,IAAK;AAAA,EACZ;AAAA,EAEA,MAAY;AAEV,UAAM,+BAAe,IAAA;AACrB,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,2BAAK,QAAO,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAC/C,iBAAS,IAAI,GAAG;AAAA,MAClB;AAAA,IACF;AAGA,UAAM,SAAmC,CAAA;AACzC,eAAW,OAAO,UAAU;AAC1B,YAAM,OAAO,mBAAK,QAAO,IAAI,GAAG;AAChC,YAAM,UAAU,mBAAK,WAAU,IAAI,GAAG;AACtC,YAAM,MAAM,mBAAK,IAAL,WAAQ;AAGpB,YAAM,mCAAmB,IAAA;AACzB,YAAM,mCAAmB,IAAA;AAGzB,iBAAW,CAAC,OAAO,YAAY,KAAK,KAAK;AACvC,cAAM,WAAW,aAAa,IAAI,KAAK,KAAK;AAC5C,qBAAa,IAAI,OAAO,WAAW,YAAY;AAAA,MACjD;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,SAAS;AAC3C,cAAM,WAAW,aAAa,IAAI,KAAK,KAAK;AAC5C,qBAAa,IAAI,OAAO,WAAW,YAAY;AAAA,MACjD;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,cAAc;AAChD,YAAI,CAAC,aAAa,IAAI,KAAK,GAAG;AAE5B,iBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,YAAY,CAAC;AACzC,6BAAK,WAAU,SAAS,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;AAAA,QACrD;AAAA,MACF;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,cAAc;AAChD,YAAI,CAAC,aAAa,IAAI,KAAK,GAAG;AAE5B,cAAI,iBAAiB,GAAG;AACtB,mBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,CAAC;AACxC,+BAAK,WAAU,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,UACpD;AAAA,QACF;AAAA,MACF;AAGA,iBAAW,CAAC,OAAO,eAAe,KAAK,cAAc;AACnD,cAAM,kBAAkB,aAAa,IAAI,KAAK;AAC9C,YAAI,oBAAoB,QAAW;AACjC,gBAAM,QAAQ,kBAAkB;AAEhC,cAAI,UAAU,GAAG;AACf,mBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,CAAC;AACjC,+BAAK,WAAU,SAAS,KAAK,CAAC,OAAO,KAAK,CAAC;AAAA,UAC7C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAtFE;AACA;AACA;AAyFK,SAAS,OAKd,GAA4D;AAC5D,SAAO,CAAC,WAAkE;AACxE,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA2C;AAEjD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"reduce.js","sources":["../../../src/operators/reduce.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport { MultiSet } from \"../multiset.js\"\nimport { Index } from \"../indexes.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { IStreamBuilder, KeyValue } from \"../types.js\"\n\n/**\n * Base operator for reduction operations (version-free)\n */\nexport class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {\n  #index = new Index<K, V1>()\n  #indexOut = new Index<K, V2>()\n  #f: (values: Array<[V1, number]>) => Array<[V2, number]>\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, V1]>,\n    output: DifferenceStreamWriter<[K, V2]>,\n    f: (values: Array<[V1, number]>) => Array<[V2, number]>\n  ) {\n    super(id, inputA, output)\n    this.#f = f\n  }\n\n  run(): void {\n    // Collect all input messages and update the index\n    const keysTodo = new Set<K>()\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.#index.addValue(key, [value, multiplicity])\n        keysTodo.add(key)\n      }\n    }\n\n    // For each key, compute the reduction and delta\n    const result: Array<[[K, V2], number]> = []\n    for (const key of keysTodo) {\n      const curr = this.#index.get(key)\n      const currOut = this.#indexOut.get(key)\n      const out = this.#f(curr)\n\n      // Create maps for current and previous outputs using values directly as keys\n      const newOutputMap = new Map<V2, number>()\n      const oldOutputMap = new Map<V2, number>()\n\n      // Process new output\n      for (const [value, multiplicity] of out) {\n        const existing = newOutputMap.get(value) ?? 0\n        newOutputMap.set(value, existing + multiplicity)\n      }\n\n      // Process previous output\n      for (const [value, multiplicity] of currOut) {\n        const existing = oldOutputMap.get(value) ?? 0\n        oldOutputMap.set(value, existing + multiplicity)\n      }\n\n      // First, emit removals for old values that are no longer present\n      for (const [value, multiplicity] of oldOutputMap) {\n        if (!newOutputMap.has(value)) {\n          // Remove the old value entirely\n          result.push([[key, value], -multiplicity])\n          this.#indexOut.addValue(key, [value, -multiplicity])\n        }\n      }\n\n      // Then, emit additions for new values that are not present in old\n      for (const [value, multiplicity] of newOutputMap) {\n        if (!oldOutputMap.has(value)) {\n          // Add the new value only if it has non-zero multiplicity\n          if (multiplicity !== 0) {\n            result.push([[key, value], multiplicity])\n            this.#indexOut.addValue(key, [value, multiplicity])\n          }\n        }\n      }\n\n      // Finally, emit multiplicity changes for values that were present and are still present\n      for (const [value, newMultiplicity] of newOutputMap) {\n        const oldMultiplicity = oldOutputMap.get(value)\n        if (oldMultiplicity !== undefined) {\n          const delta = newMultiplicity - oldMultiplicity\n          // Only emit actual changes, i.e. non-zero deltas\n          if (delta !== 0) {\n            result.push([[key, value], delta])\n            this.#indexOut.addValue(key, [value, delta])\n          }\n        }\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n}\n\n/**\n * Reduces the elements in the stream by key (version-free)\n */\nexport function reduce<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  R,\n  T,\n>(f: (values: Array<[V1Type, number]>) => Array<[R, number]>) {\n  return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<KType, R>> => {\n    const output = new StreamBuilder<KeyValue<KType, R>>(\n      stream.graph,\n      new DifferenceStreamWriter<KeyValue<KType, R>>()\n    )\n    const operator = new ReduceOperator<KType, V1Type, R>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader() as DifferenceStreamReader<KeyValue<KType, V1Type>>,\n      output.writer,\n      f\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;AAUO,MAAM,uBAAkC,cAAgC;AAAA,EAC7E,SAAS,IAAI,MAAA;AAAA,EACb,YAAY,IAAI,MAAA;AAAA,EAChB;AAAA,EAEA,YACE,IACA,QACA,QACA,GACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAY;AAEV,UAAM,+BAAe,IAAA;AACrB,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,aAAK,OAAO,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAC/C,iBAAS,IAAI,GAAG;AAAA,MAClB;AAAA,IACF;AAGA,UAAM,SAAmC,CAAA;AACzC,eAAW,OAAO,UAAU;AAC1B,YAAM,OAAO,KAAK,OAAO,IAAI,GAAG;AAChC,YAAM,UAAU,KAAK,UAAU,IAAI,GAAG;AACtC,YAAM,MAAM,KAAK,GAAG,IAAI;AAGxB,YAAM,mCAAmB,IAAA;AACzB,YAAM,mCAAmB,IAAA;AAGzB,iBAAW,CAAC,OAAO,YAAY,KAAK,KAAK;AACvC,cAAM,WAAW,aAAa,IAAI,KAAK,KAAK;AAC5C,qBAAa,IAAI,OAAO,WAAW,YAAY;AAAA,MACjD;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,SAAS;AAC3C,cAAM,WAAW,aAAa,IAAI,KAAK,KAAK;AAC5C,qBAAa,IAAI,OAAO,WAAW,YAAY;AAAA,MACjD;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,cAAc;AAChD,YAAI,CAAC,aAAa,IAAI,KAAK,GAAG;AAE5B,iBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,YAAY,CAAC;AACzC,eAAK,UAAU,SAAS,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;AAAA,QACrD;AAAA,MACF;AAGA,iBAAW,CAAC,OAAO,YAAY,KAAK,cAAc;AAChD,YAAI,CAAC,aAAa,IAAI,KAAK,GAAG;AAE5B,cAAI,iBAAiB,GAAG;AACtB,mBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,CAAC;AACxC,iBAAK,UAAU,SAAS,KAAK,CAAC,OAAO,YAAY,CAAC;AAAA,UACpD;AAAA,QACF;AAAA,MACF;AAGA,iBAAW,CAAC,OAAO,eAAe,KAAK,cAAc;AACnD,cAAM,kBAAkB,aAAa,IAAI,KAAK;AAC9C,YAAI,oBAAoB,QAAW;AACjC,gBAAM,QAAQ,kBAAkB;AAEhC,cAAI,UAAU,GAAG;AACf,mBAAO,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,CAAC;AACjC,iBAAK,UAAU,SAAS,KAAK,CAAC,OAAO,KAAK,CAAC;AAAA,UAC7C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAKO,SAAS,OAKd,GAA4D;AAC5D,SAAO,CAAC,WAAkE;AACxE,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA2C;AAEjD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}

package/dist/esm/operators/tap.js

@@ -1,25 +1,16 @@
-var __typeError = (msg) => {
-  throw TypeError(msg);
-};
-var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
-var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
-var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
-var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
-var _f;
 import { LinearUnaryOperator, DifferenceStreamWriter } from "../graph.js";
 import { StreamBuilder } from "../d2.js";
 class TapOperator extends LinearUnaryOperator {
+  #f;
   constructor(id, inputA, output, f) {
     super(id, inputA, output);
-    __privateAdd(this, _f);
-    __privateSet(this, _f, f);
+    this.#f = f;
   }
   inner(collection) {
-    __privateGet(this, _f).call(this, collection);
+    this.#f(collection);
     return collection;
   }
 }
-_f = new WeakMap();
 function tap(f) {
   return (stream) => {
     const output = new StreamBuilder(

package/dist/esm/operators/tap.js.map

@@ -1 +1 @@
-{"version":3,"file":"tap.js","sources":["../../../src/operators/tap.ts"],"sourcesContent":["import { DifferenceStreamWriter, LinearUnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport type { IStreamBuilder, PipedOperator } from \"../types.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { MultiSet } from \"../multiset.js\"\n\n/**\n * Operator that applies a function to each multi-set in the input stream\n */\nexport class TapOperator<T> extends LinearUnaryOperator<T, T> {\n  #f: (data: MultiSet<T>) => void\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<T>,\n    output: DifferenceStreamWriter<T>,\n    f: (data: MultiSet<T>) => void\n  ) {\n    super(id, inputA, output)\n    this.#f = f\n  }\n\n  inner(collection: MultiSet<T>): MultiSet<T> {\n    this.#f(collection)\n    return collection\n  }\n}\n\n/**\n * Invokes a function for each multi-set in the input stream.\n * This operator doesn't modify the stream and is used to perform side effects.\n * @param f - The function to invoke on each multi-set\n * @returns The input stream\n */\nexport function tap<T>(f: (data: MultiSet<T>) => void): PipedOperator<T, T> {\n  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n    const output = new StreamBuilder<T>(\n      stream.graph,\n      new DifferenceStreamWriter<T>()\n    )\n    const operator = new TapOperator<T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      f\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;;;;;;AASO,MAAM,oBAAuB,oBAA0B;AAAA,EAG5D,YACE,IACA,QACA,QACA,GACA;AACA,UAAM,IAAI,QAAQ,MAAM;AAR1B;AASE,uBAAK,IAAK;AAAA,EACZ;AAAA,EAEA,MAAM,YAAsC;AAC1C,uBAAK,IAAL,WAAQ;AACR,WAAO;AAAA,EACT;AACF;AAhBE;AAwBK,SAAS,IAAO,GAAqD;AAC1E,SAAO,CAAC,WAAiD;AACvD,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA0B;AAEhC,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"tap.js","sources":["../../../src/operators/tap.ts"],"sourcesContent":["import { DifferenceStreamWriter, LinearUnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport type { IStreamBuilder, PipedOperator } from \"../types.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { MultiSet } from \"../multiset.js\"\n\n/**\n * Operator that applies a function to each multi-set in the input stream\n */\nexport class TapOperator<T> extends LinearUnaryOperator<T, T> {\n  #f: (data: MultiSet<T>) => void\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<T>,\n    output: DifferenceStreamWriter<T>,\n    f: (data: MultiSet<T>) => void\n  ) {\n    super(id, inputA, output)\n    this.#f = f\n  }\n\n  inner(collection: MultiSet<T>): MultiSet<T> {\n    this.#f(collection)\n    return collection\n  }\n}\n\n/**\n * Invokes a function for each multi-set in the input stream.\n * This operator doesn't modify the stream and is used to perform side effects.\n * @param f - The function to invoke on each multi-set\n * @returns The input stream\n */\nexport function tap<T>(f: (data: MultiSet<T>) => void): PipedOperator<T, T> {\n  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n    const output = new StreamBuilder<T>(\n      stream.graph,\n      new DifferenceStreamWriter<T>()\n    )\n    const operator = new TapOperator<T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      f\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;AASO,MAAM,oBAAuB,oBAA0B;AAAA,EAC5D;AAAA,EAEA,YACE,IACA,QACA,QACA,GACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,YAAsC;AAC1C,SAAK,GAAG,UAAU;AAClB,WAAO;AAAA,EACT;AACF;AAQO,SAAS,IAAO,GAAqD;AAC1E,SAAO,CAAC,WAAiD;AACvD,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA0B;AAEhC,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}

package/dist/esm/operators/topK.d.ts

@@ -1,5 +1,4 @@
-import { PipedOperator } from '../types.js';
-import { KeyValue } from '../types.js';
+import { KeyValue, PipedOperator } from '../types.js';
 interface TopKOptions {
     limit?: number;
     offset?: number;

package/dist/esm/operators/topK.js

@@ -1,8 +1,8 @@
 import { MultiSet } from "../multiset.js";
 import { reduce } from "./reduce.js";
 function topK(comparator, options) {
-  const limit = (options == null ? void 0 : options.limit) ?? Infinity;
-  const offset = (options == null ? void 0 : options.offset) ?? 0;
+  const limit = options?.limit ?? Infinity;
+  const offset = options?.offset ?? 0;
   return (stream) => {
     const reduced = stream.pipe(
       reduce((values) => {
@@ -15,8 +15,8 @@ function topK(comparator, options) {
   };
 }
 function topKWithIndex(comparator, options) {
-  const limit = (options == null ? void 0 : options.limit) ?? Infinity;
-  const offset = (options == null ? void 0 : options.offset) ?? 0;
+  const limit = options?.limit ?? Infinity;
+  const offset = options?.offset ?? 0;
   return (stream) => {
     const reduced = stream.pipe(
       reduce((values) => {

package/dist/esm/operators/topK.js.map

@@ -1 +1 @@
-{"version":3,"file":"topK.js","sources":["../../../src/operators/topK.ts"],"sourcesContent":["import { MultiSet } from \"../multiset.js\"\nimport { reduce } from \"./reduce.js\"\nimport type { IStreamBuilder, PipedOperator } from \"../types\"\nimport type { KeyValue } from \"../types.js\"\n\ninterface TopKOptions {\n  limit?: number\n  offset?: number\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * This works on a keyed stream, where the key is the first element of the tuple\n * The ordering is withing a key group, i.e. elements are sorted within a key group\n * and the limit + offset is applied to that sorted group.\n * To order the entire stream, key by the same value for all elements such as null.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that limits the number of results\n */\nexport function topK<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  T,\n>(\n  comparator: (a: V1Type, b: V1Type) => number,\n  options?: TopKOptions\n): PipedOperator<T, T> {\n  const limit = options?.limit ?? Infinity\n  const offset = options?.offset ?? 0\n\n  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n    const reduced = stream.pipe(\n      reduce((values) => {\n        // `values` is a list of tuples, first element is the value, second is the multiplicity\n        const consolidated = new MultiSet(values).consolidate()\n        const sortedValues = consolidated\n          .getInner()\n          .sort((a, b) => comparator(a[0] as V1Type, b[0] as V1Type))\n        return sortedValues.slice(offset, offset + limit)\n      })\n    )\n    return reduced as IStreamBuilder<T>\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * This works on a keyed stream, where the key is the first element of the tuple\n * The ordering is withing a key group, i.e. elements are sorted within a key group\n * and the limit + offset is applied to that sorted group.\n * To order the entire stream, key by the same value for all elements such as null.\n * Adds the index of the element to the result as [key, [value, index]]\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithIndex<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  T,\n>(\n  comparator: (a: V1Type, b: V1Type) => number,\n  options?: TopKOptions\n): PipedOperator<T, KeyValue<KType, [V1Type, number]>> {\n  const limit = options?.limit ?? Infinity\n  const offset = options?.offset ?? 0\n\n  return (\n    stream: IStreamBuilder<T>\n  ): IStreamBuilder<KeyValue<KType, [V1Type, number]>> => {\n    const reduced = stream.pipe(\n      reduce<KType, V1Type, [V1Type, number], T>((values) => {\n        // `values` is a list of tuples, first element is the value, second is the multiplicity\n        const consolidated = new MultiSet(values).consolidate()\n        let i = offset\n        const sortedValues = consolidated\n          .getInner()\n          .sort((a, b) => comparator(a[0], b[0]))\n          .slice(offset, offset + limit)\n          .map(([value, multiplicity]): [[V1Type, number], number] => [\n            [value, i++],\n            multiplicity,\n          ])\n        return sortedValues\n      })\n    )\n    return reduced\n  }\n}\n"],"names":[],"mappings":";;AAqBO,SAAS,KAKd,YACA,SACqB;AACrB,QAAM,SAAQ,mCAAS,UAAS;AAChC,QAAM,UAAS,mCAAS,WAAU;AAElC,SAAO,CAAC,WAAiD;AACvD,UAAM,UAAU,OAAO;AAAA,MACrB,OAAO,CAAC,WAAW;AAEjB,cAAM,eAAe,IAAI,SAAS,MAAM,EAAE,YAAA;AAC1C,cAAM,eAAe,aAClB,SAAA,EACA,KAAK,CAAC,GAAG,MAAM,WAAW,EAAE,CAAC,GAAa,EAAE,CAAC,CAAW,CAAC;AAC5D,eAAO,aAAa,MAAM,QAAQ,SAAS,KAAK;AAAA,MAClD,CAAC;AAAA,IAAA;AAEH,WAAO;AAAA,EACT;AACF;AAcO,SAAS,cAKd,YACA,SACqD;AACrD,QAAM,SAAQ,mCAAS,UAAS;AAChC,QAAM,UAAS,mCAAS,WAAU;AAElC,SAAO,CACL,WACsD;AACtD,UAAM,UAAU,OAAO;AAAA,MACrB,OAA2C,CAAC,WAAW;AAErD,cAAM,eAAe,IAAI,SAAS,MAAM,EAAE,YAAA;AAC1C,YAAI,IAAI;AACR,cAAM,eAAe,aAClB,SAAA,EACA,KAAK,CAAC,GAAG,MAAM,WAAW,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EACrC,MAAM,QAAQ,SAAS,KAAK,EAC5B,IAAI,CAAC,CAAC,OAAO,YAAY,MAAkC;AAAA,UAC1D,CAAC,OAAO,GAAG;AAAA,UACX;AAAA,QAAA,CACD;AACH,eAAO;AAAA,MACT,CAAC;AAAA,IAAA;AAEH,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"topK.js","sources":["../../../src/operators/topK.ts"],"sourcesContent":["import { MultiSet } from \"../multiset.js\"\nimport { reduce } from \"./reduce.js\"\nimport type { IStreamBuilder, KeyValue, PipedOperator } from \"../types\"\n\ninterface TopKOptions {\n  limit?: number\n  offset?: number\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * This works on a keyed stream, where the key is the first element of the tuple\n * The ordering is withing a key group, i.e. elements are sorted within a key group\n * and the limit + offset is applied to that sorted group.\n * To order the entire stream, key by the same value for all elements such as null.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that limits the number of results\n */\nexport function topK<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  T,\n>(\n  comparator: (a: V1Type, b: V1Type) => number,\n  options?: TopKOptions\n): PipedOperator<T, T> {\n  const limit = options?.limit ?? Infinity\n  const offset = options?.offset ?? 0\n\n  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n    const reduced = stream.pipe(\n      reduce((values) => {\n        // `values` is a list of tuples, first element is the value, second is the multiplicity\n        const consolidated = new MultiSet(values).consolidate()\n        const sortedValues = consolidated\n          .getInner()\n          .sort((a, b) => comparator(a[0] as V1Type, b[0] as V1Type))\n        return sortedValues.slice(offset, offset + limit)\n      })\n    )\n    return reduced as IStreamBuilder<T>\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * This works on a keyed stream, where the key is the first element of the tuple\n * The ordering is withing a key group, i.e. elements are sorted within a key group\n * and the limit + offset is applied to that sorted group.\n * To order the entire stream, key by the same value for all elements such as null.\n * Adds the index of the element to the result as [key, [value, index]]\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithIndex<\n  KType extends T extends KeyValue<infer K, infer _V> ? K : never,\n  V1Type extends T extends KeyValue<KType, infer V> ? V : never,\n  T,\n>(\n  comparator: (a: V1Type, b: V1Type) => number,\n  options?: TopKOptions\n): PipedOperator<T, KeyValue<KType, [V1Type, number]>> {\n  const limit = options?.limit ?? Infinity\n  const offset = options?.offset ?? 0\n\n  return (\n    stream: IStreamBuilder<T>\n  ): IStreamBuilder<KeyValue<KType, [V1Type, number]>> => {\n    const reduced = stream.pipe(\n      reduce<KType, V1Type, [V1Type, number], T>((values) => {\n        // `values` is a list of tuples, first element is the value, second is the multiplicity\n        const consolidated = new MultiSet(values).consolidate()\n        let i = offset\n        const sortedValues = consolidated\n          .getInner()\n          .sort((a, b) => comparator(a[0], b[0]))\n          .slice(offset, offset + limit)\n          .map(([value, multiplicity]): [[V1Type, number], number] => [\n            [value, i++],\n            multiplicity,\n          ])\n        return sortedValues\n      })\n    )\n    return reduced\n  }\n}\n"],"names":[],"mappings":";;AAoBO,SAAS,KAKd,YACA,SACqB;AACrB,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,SAAS,SAAS,UAAU;AAElC,SAAO,CAAC,WAAiD;AACvD,UAAM,UAAU,OAAO;AAAA,MACrB,OAAO,CAAC,WAAW;AAEjB,cAAM,eAAe,IAAI,SAAS,MAAM,EAAE,YAAA;AAC1C,cAAM,eAAe,aAClB,SAAA,EACA,KAAK,CAAC,GAAG,MAAM,WAAW,EAAE,CAAC,GAAa,EAAE,CAAC,CAAW,CAAC;AAC5D,eAAO,aAAa,MAAM,QAAQ,SAAS,KAAK;AAAA,MAClD,CAAC;AAAA,IAAA;AAEH,WAAO;AAAA,EACT;AACF;AAcO,SAAS,cAKd,YACA,SACqD;AACrD,QAAM,QAAQ,SAAS,SAAS;AAChC,QAAM,SAAS,SAAS,UAAU;AAElC,SAAO,CACL,WACsD;AACtD,UAAM,UAAU,OAAO;AAAA,MACrB,OAA2C,CAAC,WAAW;AAErD,cAAM,eAAe,IAAI,SAAS,MAAM,EAAE,YAAA;AAC1C,YAAI,IAAI;AACR,cAAM,eAAe,aAClB,SAAA,EACA,KAAK,CAAC,GAAG,MAAM,WAAW,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EACrC,MAAM,QAAQ,SAAS,KAAK,EAC5B,IAAI,CAAC,CAAC,OAAO,YAAY,MAAkC;AAAA,UAC1D,CAAC,OAAO,GAAG;AAAA,UACX;AAAA,QAAA,CACD;AACH,eAAO;AAAA,MACT,CAAC;AAAA,IAAA;AAEH,WAAO;AAAA,EACT;AACF;"}

package/dist/esm/operators/topKWithFractionalIndex.js

@@ -1,48 +1,38 @@
-var __typeError = (msg) => {
-  throw TypeError(msg);
-};
-var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
-var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
-var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
-var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
-var __privateMethod = (obj, member, method) => (__accessCheck(obj, member, "access private method"), method);
-var _sortedValues, _comparator, _topKStart, _topKEnd, _TopKArray_instances, findIndex_fn, _index, _topK, _limit;
 import { generateKeyBetween } from "fractional-indexing";
 import { UnaryOperator, DifferenceStreamWriter } from "../graph.js";
 import { StreamBuilder } from "../d2.js";
 import { MultiSet } from "../multiset.js";
 import { binarySearch, globalObjectIdGenerator } from "../utils.js";
 class TopKArray {
+  #sortedValues = [];
+  #comparator;
+  #topKStart;
+  #topKEnd;
   constructor(offset, limit, comparator) {
-    __privateAdd(this, _TopKArray_instances);
-    __privateAdd(this, _sortedValues, []);
-    __privateAdd(this, _comparator);
-    __privateAdd(this, _topKStart);
-    __privateAdd(this, _topKEnd);
-    __privateSet(this, _topKStart, offset);
-    __privateSet(this, _topKEnd, offset + limit);
-    __privateSet(this, _comparator, comparator);
+    this.#topKStart = offset;
+    this.#topKEnd = offset + limit;
+    this.#comparator = comparator;
   }
   get size() {
-    const offset = __privateGet(this, _topKStart);
-    const limit = __privateGet(this, _topKEnd) - __privateGet(this, _topKStart);
-    const available = __privateGet(this, _sortedValues).length - offset;
+    const offset = this.#topKStart;
+    const limit = this.#topKEnd - this.#topKStart;
+    const available = this.#sortedValues.length - offset;
     return Math.max(0, Math.min(limit, available));
   }
   insert(value) {
     const result = { moveIn: null, moveOut: null };
-    const index = __privateMethod(this, _TopKArray_instances, findIndex_fn).call(this, value);
-    const indexBefore = index === 0 ? null : getIndex(__privateGet(this, _sortedValues)[index - 1]);
-    const indexAfter = index === __privateGet(this, _sortedValues).length ? null : getIndex(__privateGet(this, _sortedValues)[index]);
+    const index = this.#findIndex(value);
+    const indexBefore = index === 0 ? null : getIndex(this.#sortedValues[index - 1]);
+    const indexAfter = index === this.#sortedValues.length ? null : getIndex(this.#sortedValues[index]);
     const fractionalIndex = generateKeyBetween(indexBefore, indexAfter);
     const val = indexedValue(value, fractionalIndex);
-    __privateGet(this, _sortedValues).splice(index, 0, val);
-    if (index < __privateGet(this, _topKEnd)) {
-      const moveInIndex = Math.max(index, __privateGet(this, _topKStart));
-      if (moveInIndex < __privateGet(this, _sortedValues).length) {
-        result.moveIn = __privateGet(this, _sortedValues)[moveInIndex];
-        if (__privateGet(this, _topKEnd) < __privateGet(this, _sortedValues).length) {
-          result.moveOut = __privateGet(this, _sortedValues)[__privateGet(this, _topKEnd)];
+    this.#sortedValues.splice(index, 0, val);
+    if (index < this.#topKEnd) {
+      const moveInIndex = Math.max(index, this.#topKStart);
+      if (moveInIndex < this.#sortedValues.length) {
+        result.moveIn = this.#sortedValues[moveInIndex];
+        if (this.#topKEnd < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[this.#topKEnd];
         }
       }
     }
@@ -56,54 +46,48 @@ class TopKArray {
    */
   delete(value) {
     const result = { moveIn: null, moveOut: null };
-    const index = __privateMethod(this, _TopKArray_instances, findIndex_fn).call(this, value);
-    const [removedElem] = __privateGet(this, _sortedValues).splice(index, 1);
-    if (index < __privateGet(this, _topKEnd)) {
+    const index = this.#findIndex(value);
+    const [removedElem] = this.#sortedValues.splice(index, 1);
+    if (index < this.#topKEnd) {
       result.moveOut = removedElem;
-      if (index < __privateGet(this, _topKStart)) {
-        const moveOutIndex = __privateGet(this, _topKStart) - 1;
-        if (moveOutIndex < __privateGet(this, _sortedValues).length) {
-          result.moveOut = __privateGet(this, _sortedValues)[moveOutIndex];
+      if (index < this.#topKStart) {
+        const moveOutIndex = this.#topKStart - 1;
+        if (moveOutIndex < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[moveOutIndex];
         } else {
           result.moveOut = null;
         }
       }
-      const moveInIndex = __privateGet(this, _topKEnd) - 1;
-      if (moveInIndex < __privateGet(this, _sortedValues).length) {
-        result.moveIn = __privateGet(this, _sortedValues)[moveInIndex];
+      const moveInIndex = this.#topKEnd - 1;
+      if (moveInIndex < this.#sortedValues.length) {
+        result.moveIn = this.#sortedValues[moveInIndex];
       }
     }
     return result;
   }
+  // TODO: see if there is a way to refactor the code for insert and delete in the topK above
+  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right
+  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern
+  #findIndex(value) {
+    return binarySearch(
+      this.#sortedValues,
+      indexedValue(value, ``),
+      (a, b) => this.#comparator(getValue(a), getValue(b))
+    );
+  }
 }
-_sortedValues = new WeakMap();
-_comparator = new WeakMap();
-_topKStart = new WeakMap();
-_topKEnd = new WeakMap();
-_TopKArray_instances = new WeakSet();
-// TODO: see if there is a way to refactor the code for insert and delete in the topK above
-//       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right
-//       so i have the feeling there is a common pattern here and we can implement both cases using that pattern
-findIndex_fn = function(value) {
-  return binarySearch(
-    __privateGet(this, _sortedValues),
-    indexedValue(value, ``),
-    (a, b) => __privateGet(this, _comparator).call(this, getValue(a), getValue(b))
-  );
-};
 class TopKWithFractionalIndexOperator extends UnaryOperator {
+  #index = /* @__PURE__ */ new Map();
+  // maps keys to their multiplicity
+  /**
+   * topK data structure that supports insertions and deletions
+   * and returns changes to the topK.
+   */
+  #topK;
+  #limit;
   constructor(id, inputA, output, comparator, options) {
-    var _a;
     super(id, inputA, output);
-    __privateAdd(this, _index, /* @__PURE__ */ new Map());
-    // maps keys to their multiplicity
-    /**
-     * topK data structure that supports insertions and deletions
-     * and returns changes to the topK.
-     */
-    __privateAdd(this, _topK);
-    __privateAdd(this, _limit);
-    __privateSet(this, _limit, options.limit ?? Infinity);
+    this.#limit = options.limit ?? Infinity;
     const offset = options.offset ?? 0;
     const compareTaggedValues = (a, b) => {
       const valueComparison = comparator(getVal(a), getVal(b));
@@ -114,8 +98,8 @@ class TopKWithFractionalIndexOperator extends UnaryOperator {
       const tieBreakerB = getTag(b);
       return tieBreakerA - tieBreakerB;
     };
-    __privateSet(this, _topK, this.createTopK(offset, __privateGet(this, _limit), compareTaggedValues));
-    (_a = options.setSizeCallback) == null ? void 0 : _a.call(options, () => __privateGet(this, _topK).size);
+    this.#topK = this.createTopK(offset, this.#limit, compareTaggedValues);
+    options.setSizeCallback?.(() => this.#topK.size);
   }
   createTopK(offset, limit, comparator) {
     return new TopKArray(offset, limit, comparator);
@@ -140,10 +124,10 @@ class TopKWithFractionalIndexOperator extends UnaryOperator {
     };
     if (oldMultiplicity <= 0 && newMultiplicity > 0) {
       const taggedValue = tagValue(key, value);
-      res = __privateGet(this, _topK).insert(taggedValue);
+      res = this.#topK.insert(taggedValue);
     } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
       const taggedValue = tagValue(key, value);
-      res = __privateGet(this, _topK).delete(taggedValue);
+      res = this.#topK.delete(taggedValue);
     } else ;
     if (res.moveIn) {
       const index = getIndex(res.moveIn);
@@ -162,22 +146,19 @@ class TopKWithFractionalIndexOperator extends UnaryOperator {
     return;
   }
   getMultiplicity(key) {
-    return __privateGet(this, _index).get(key) ?? 0;
+    return this.#index.get(key) ?? 0;
   }
   addKey(key, multiplicity) {
     const oldMultiplicity = this.getMultiplicity(key);
     const newMultiplicity = oldMultiplicity + multiplicity;
     if (newMultiplicity === 0) {
-      __privateGet(this, _index).delete(key);
+      this.#index.delete(key);
     } else {
-      __privateGet(this, _index).set(key, newMultiplicity);
+      this.#index.set(key, newMultiplicity);
     }
     return { oldMultiplicity, newMultiplicity };
   }
 }
-_index = new WeakMap();
-_topK = new WeakMap();
-_limit = new WeakMap();
 function topKWithFractionalIndex(comparator, options) {
   const opts = options || {};
   return (stream) => {

package/dist/esm/operators/topKWithFractionalIndex.js.map

@@ -1 +1 @@
-{"version":3,"file":"topKWithFractionalIndex.js","sources":["../../../src/operators/topKWithFractionalIndex.ts"],"sourcesContent":["import { generateKeyBetween } from \"fractional-indexing\"\nimport { DifferenceStreamWriter, UnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport { MultiSet } from \"../multiset.js\"\nimport { binarySearch, globalObjectIdGenerator } from \"../utils.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { IStreamBuilder, PipedOperator } from \"../types.js\"\n\nexport interface TopKWithFractionalIndexOptions {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n}\n\nexport type TopKChanges<V> = {\n  /** Indicates which element moves into the topK (if any) */\n  moveIn: IndexedValue<V> | null\n  /** Indicates which element moves out of the topK (if any) */\n  moveOut: IndexedValue<V> | null\n}\n\n/**\n * A topK data structure that supports insertions and deletions\n * and returns changes to the topK.\n */\nexport interface TopK<V> {\n  size: number\n  insert: (value: V) => TopKChanges<V>\n  delete: (value: V) => TopKChanges<V>\n}\n\n/**\n * Implementation of a topK data structure.\n * Uses a sorted array internally to store the values and keeps a topK window over that array.\n * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted\n * at the start of the array which causes all the elements to shift to the right/left.\n */\nclass TopKArray<V> implements TopK<V> {\n  #sortedValues: Array<IndexedValue<V>> = []\n  #comparator: (a: V, b: V) => number\n  #topKStart: number\n  #topKEnd: number\n\n  constructor(\n    offset: number,\n    limit: number,\n    comparator: (a: V, b: V) => number\n  ) {\n    this.#topKStart = offset\n    this.#topKEnd = offset + limit\n    this.#comparator = comparator\n  }\n\n  get size(): number {\n    const offset = this.#topKStart\n    const limit = this.#topKEnd - this.#topKStart\n    const available = this.#sortedValues.length - offset\n    return Math.max(0, Math.min(limit, available))\n  }\n\n  insert(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup insert position\n    const index = this.#findIndex(value)\n    // Generate fractional index based on the fractional indices of the elements before and after it\n    const indexBefore =\n      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)\n    const indexAfter =\n      index === this.#sortedValues.length\n        ? null\n        : getIndex(this.#sortedValues[index]!)\n    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)\n\n    // Insert the value at the correct position\n    const val = indexedValue(value, fractionalIndex)\n    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !\n    this.#sortedValues.splice(index, 0, val)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The inserted element is either before the top K or within the top K\n      // If it is before the top K then it moves the element that was right before the topK into the topK\n      // If it is within the top K then the inserted element moves into the top K\n      // In both cases the last element of the old top K now moves out of the top K\n      const moveInIndex = Math.max(index, this.#topKStart)\n      if (moveInIndex < this.#sortedValues.length) {\n        // We actually have a topK\n        // because in some cases there may not be enough elements in the array to reach the start of the topK\n        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK\n        result.moveIn = this.#sortedValues[moveInIndex]!\n\n        // We need to remove the element that falls out of the top K\n        // The element that falls out of the top K has shifted one to the right\n        // because of the element we inserted, so we find it at index topKEnd\n        if (this.#topKEnd < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[this.#topKEnd]!\n        }\n      }\n    }\n\n    return result\n  }\n\n  /**\n   * Deletes a value that may or may not be in the topK.\n   * IMPORTANT: this assumes that the value is present in the collection\n   *            if it's not the case it will remove the element\n   *            that is on the position where the provided `value` would be.\n   */\n  delete(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup delete position\n    const index = this.#findIndex(value)\n    // Remove the value at that position\n    const [removedElem] = this.#sortedValues.splice(index, 1)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The removed element is either before the top K or within the top K\n      // If it is before the top K then the first element of the topK moves out of the topK\n      // If it is within the top K then the removed element moves out of the topK\n      result.moveOut = removedElem!\n      if (index < this.#topKStart) {\n        // The removed element is before the topK\n        // so actually, the first element of the topK moves out of the topK\n        // and not the element that we removed\n        // The first element of the topK is now at index topKStart - 1\n        // since we removed an element before the topK\n        const moveOutIndex = this.#topKStart - 1\n        if (moveOutIndex < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[moveOutIndex]!\n        } else {\n          // No value is moving out of the topK\n          // because there are no elements in the topK\n          result.moveOut = null\n        }\n      }\n\n      // Since we removed an element that was before or in the topK\n      // the first element after the topK moved one position to the left\n      // and thus falls into the topK now\n      const moveInIndex = this.#topKEnd - 1\n      if (moveInIndex < this.#sortedValues.length) {\n        result.moveIn = this.#sortedValues[moveInIndex]!\n      }\n    }\n\n    return result\n  }\n\n  // TODO: see if there is a way to refactor the code for insert and delete in the topK above\n  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right\n  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern\n\n  #findIndex(value: V): number {\n    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>\n      this.#comparator(getValue(a), getValue(b))\n    )\n  }\n}\n\n/**\n * Operator for fractional indexed topK operations\n * This operator maintains fractional indices for sorted elements\n * and only updates indices when elements move position\n */\nexport class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<\n  [K, T],\n  [K, IndexedValue<T>]\n> {\n  #index: Map<K, number> = new Map() // maps keys to their multiplicity\n\n  /**\n   * topK data structure that supports insertions and deletions\n   * and returns changes to the topK.\n   */\n  #topK: TopK<TaggedValue<K, T>>\n\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: TopKWithFractionalIndexOptions\n  ) {\n    super(id, inputA, output)\n    this.#limit = options.limit ?? Infinity\n    const offset = options.offset ?? 0\n    const compareTaggedValues = (\n      a: TaggedValue<K, T>,\n      b: TaggedValue<K, T>\n    ) => {\n      // First compare on the value\n      const valueComparison = comparator(getVal(a), getVal(b))\n      if (valueComparison !== 0) {\n        return valueComparison\n      }\n      // If the values are equal, compare on the tag (object identity)\n      const tieBreakerA = getTag(a)\n      const tieBreakerB = getTag(b)\n      return tieBreakerA - tieBreakerB\n    }\n    this.#topK = this.createTopK(offset, this.#limit, compareTaggedValues)\n    options.setSizeCallback?.(() => this.#topK.size)\n  }\n\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: TaggedValue<K, T>, b: TaggedValue<K, T>) => number\n  ): TopK<TaggedValue<K, T>> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.processElement(key, value, multiplicity, result)\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>\n  ): void {\n    const { oldMultiplicity, newMultiplicity } = this.addKey(key, multiplicity)\n\n    let res: TopKChanges<TaggedValue<K, T>> = {\n      moveIn: null,\n      moveOut: null,\n    }\n    if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n      // The value was invisible but should now be visible\n      // Need to insert it into the array of sorted values\n      const taggedValue = tagValue(key, value)\n      res = this.#topK.insert(taggedValue)\n    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n      // The value was visible but should now be invisible\n      // Need to remove it from the array of sorted values\n      const taggedValue = tagValue(key, value)\n      res = this.#topK.delete(taggedValue)\n    } else {\n      // The value was invisible and it remains invisible\n      // or it was visible and remains visible\n      // so it doesn't affect the topK\n    }\n\n    if (res.moveIn) {\n      const index = getIndex(res.moveIn)\n      const taggedValue = getValue(res.moveIn)\n      const k = getKey(taggedValue)\n      const val = getVal(taggedValue)\n      result.push([[k, [val, index]], 1])\n    }\n\n    if (res.moveOut) {\n      const index = getIndex(res.moveOut)\n      const taggedValue = getValue(res.moveOut)\n      const k = getKey(taggedValue)\n      const val = getVal(taggedValue)\n      result.push([[k, [val, index]], -1])\n    }\n\n    return\n  }\n\n  private getMultiplicity(key: K): number {\n    return this.#index.get(key) ?? 0\n  }\n\n  private addKey(\n    key: K,\n    multiplicity: number\n  ): { oldMultiplicity: number; newMultiplicity: number } {\n    const oldMultiplicity = this.getMultiplicity(key)\n    const newMultiplicity = oldMultiplicity + multiplicity\n    if (newMultiplicity === 0) {\n      this.#index.delete(key)\n    } else {\n      this.#index.set(key, newMultiplicity)\n    }\n    return { oldMultiplicity, newMultiplicity }\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithFractionalIndex<KType, T>(\n  comparator: (a: T, b: T) => number,\n  options?: TopKWithFractionalIndexOptions\n): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {\n  const opts = options || {}\n\n  return (\n    stream: IStreamBuilder<[KType, T]>\n  ): IStreamBuilder<[KType, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[KType, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[KType, IndexedValue<T>]>()\n    )\n    const operator = new TopKWithFractionalIndexOperator<KType, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      opts\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n\n// Abstraction for fractionally indexed values\nexport type FractionalIndex = string\nexport type IndexedValue<V> = [V, FractionalIndex]\n\nexport function indexedValue<V>(\n  value: V,\n  index: FractionalIndex\n): IndexedValue<V> {\n  return [value, index]\n}\n\nexport function getValue<V>(indexedVal: IndexedValue<V>): V {\n  return indexedVal[0]\n}\n\nexport function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {\n  return indexedVal[1]\n}\n\nexport type Tag = number\nexport type TaggedValue<K, V> = [K, V, Tag]\n\nfunction tagValue<K, V>(key: K, value: V): TaggedValue<K, V> {\n  return [key, value, globalObjectIdGenerator.getId(key)]\n}\n\nfunction getKey<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): K {\n  return tieBreakerTaggedValue[0]\n}\n\nfunction getVal<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): V {\n  return tieBreakerTaggedValue[1]\n}\n\nfunction getTag<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): Tag {\n  return tieBreakerTaggedValue[2]\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;;AAqCA,MAAM,UAAgC;AAAA,EAMpC,YACE,QACA,OACA,YACA;AAVJ;AACE,sCAAwC,CAAA;AACxC;AACA;AACA;AAOE,uBAAK,YAAa;AAClB,uBAAK,UAAW,SAAS;AACzB,uBAAK,aAAc;AAAA,EACrB;AAAA,EAEA,IAAI,OAAe;AACjB,UAAM,SAAS,mBAAK;AACpB,UAAM,QAAQ,mBAAK,YAAW,mBAAK;AACnC,UAAM,YAAY,mBAAK,eAAc,SAAS;AAC9C,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,SAAS,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,sBAAK,oCAAL,WAAgB;AAE9B,UAAM,cACJ,UAAU,IAAI,OAAO,SAAS,mBAAK,eAAc,QAAQ,CAAC,CAAE;AAC9D,UAAM,aACJ,UAAU,mBAAK,eAAc,SACzB,OACA,SAAS,mBAAK,eAAc,KAAK,CAAE;AACzC,UAAM,kBAAkB,mBAAmB,aAAa,UAAU;AAGlE,UAAM,MAAM,aAAa,OAAO,eAAe;AAE/C,uBAAK,eAAc,OAAO,OAAO,GAAG,GAAG;AAGvC,QAAI,QAAQ,mBAAK,WAAU;AAKzB,YAAM,cAAc,KAAK,IAAI,OAAO,mBAAK,WAAU;AACnD,UAAI,cAAc,mBAAK,eAAc,QAAQ;AAI3C,eAAO,SAAS,mBAAK,eAAc,WAAW;AAK9C,YAAI,mBAAK,YAAW,mBAAK,eAAc,QAAQ;AAC7C,iBAAO,UAAU,mBAAK,eAAc,mBAAK,SAAQ;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,sBAAK,oCAAL,WAAgB;AAE9B,UAAM,CAAC,WAAW,IAAI,mBAAK,eAAc,OAAO,OAAO,CAAC;AAGxD,QAAI,QAAQ,mBAAK,WAAU;AAIzB,aAAO,UAAU;AACjB,UAAI,QAAQ,mBAAK,aAAY;AAM3B,cAAM,eAAe,mBAAK,cAAa;AACvC,YAAI,eAAe,mBAAK,eAAc,QAAQ;AAC5C,iBAAO,UAAU,mBAAK,eAAc,YAAY;AAAA,QAClD,OAAO;AAGL,iBAAO,UAAU;AAAA,QACnB;AAAA,MACF;AAKA,YAAM,cAAc,mBAAK,YAAW;AACpC,UAAI,cAAc,mBAAK,eAAc,QAAQ;AAC3C,eAAO,SAAS,mBAAK,eAAc,WAAW;AAAA,MAChD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAWF;AA3HE;AACA;AACA;AACA;AAJF;AAAA;AAAA;AAAA;AAuHE,wBAAW,OAAkB;AAC3B,SAAO;AAAA,IAAa,mBAAK;AAAA,IAAe,aAAa,OAAO,EAAE;AAAA,IAAG,CAAC,GAAG,MACnE,mBAAK,aAAL,WAAiB,SAAS,CAAC,GAAG,SAAS,CAAC;AAAA,EAAC;AAE7C;AAQK,MAAM,wCAA8C,cAGzD;AAAA,EAWA,YACE,IACA,QACA,QACA,YACA,SACA;;AACA,UAAM,IAAI,QAAQ,MAAM;AAjB1B,mDAA6B,IAAA;AAM7B;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA;AAUE,uBAAK,QAAS,QAAQ,SAAS;AAC/B,UAAM,SAAS,QAAQ,UAAU;AACjC,UAAM,sBAAsB,CAC1B,GACA,MACG;AAEH,YAAM,kBAAkB,WAAW,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACvD,UAAI,oBAAoB,GAAG;AACzB,eAAO;AAAA,MACT;AAEA,YAAM,cAAc,OAAO,CAAC;AAC5B,YAAM,cAAc,OAAO,CAAC;AAC5B,aAAO,cAAc;AAAA,IACvB;AACA,uBAAK,OAAQ,KAAK,WAAW,QAAQ,mBAAK,SAAQ,mBAAmB;AACrE,kBAAQ,oBAAR,iCAA0B,MAAM,mBAAK,OAAM;AAAA,EAC7C;AAAA,EAEU,WACR,QACA,OACA,YACyB;AACzB,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,aAAK,eAAe,KAAK,OAAO,cAAc,MAAM;AAAA,MACtD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,eACE,KACA,OACA,cACA,QACM;AACN,UAAM,EAAE,iBAAiB,gBAAA,IAAoB,KAAK,OAAO,KAAK,YAAY;AAE1E,QAAI,MAAsC;AAAA,MACxC,QAAQ;AAAA,MACR,SAAS;AAAA,IAAA;AAEX,QAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAG/C,YAAM,cAAc,SAAS,KAAK,KAAK;AACvC,YAAM,mBAAK,OAAM,OAAO,WAAW;AAAA,IACrC,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAGtD,YAAM,cAAc,SAAS,KAAK,KAAK;AACvC,YAAM,mBAAK,OAAM,OAAO,WAAW;AAAA,IACrC,MAAO;AAMP,QAAI,IAAI,QAAQ;AACd,YAAM,QAAQ,SAAS,IAAI,MAAM;AACjC,YAAM,cAAc,SAAS,IAAI,MAAM;AACvC,YAAM,IAAI,OAAO,WAAW;AAC5B,YAAM,MAAM,OAAO,WAAW;AAC9B,aAAO,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,IACpC;AAEA,QAAI,IAAI,SAAS;AACf,YAAM,QAAQ,SAAS,IAAI,OAAO;AAClC,YAAM,cAAc,SAAS,IAAI,OAAO;AACxC,YAAM,IAAI,OAAO,WAAW;AAC5B,YAAM,MAAM,OAAO,WAAW;AAC9B,aAAO,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,EAAE,CAAC;AAAA,IACrC;AAEA;AAAA,EACF;AAAA,EAEQ,gBAAgB,KAAgB;AACtC,WAAO,mBAAK,QAAO,IAAI,GAAG,KAAK;AAAA,EACjC;AAAA,EAEQ,OACN,KACA,cACsD;AACtD,UAAM,kBAAkB,KAAK,gBAAgB,GAAG;AAChD,UAAM,kBAAkB,kBAAkB;AAC1C,QAAI,oBAAoB,GAAG;AACzB,yBAAK,QAAO,OAAO,GAAG;AAAA,IACxB,OAAO;AACL,yBAAK,QAAO,IAAI,KAAK,eAAe;AAAA,IACtC;AACA,WAAO,EAAE,iBAAiB,gBAAA;AAAA,EAC5B;AACF;AA5HE;AAMA;AAEA;AAgIK,SAAS,wBACd,YACA,SACqD;AACrD,QAAM,OAAO,WAAW,CAAA;AAExB,SAAO,CACL,WAC6C;AAC7C,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAAiD;AAEvD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;AAMO,SAAS,aACd,OACA,OACiB;AACjB,SAAO,CAAC,OAAO,KAAK;AACtB;AAEO,SAAS,SAAY,YAAgC;AAC1D,SAAO,WAAW,CAAC;AACrB;AAEO,SAAS,SAAY,YAA8C;AACxE,SAAO,WAAW,CAAC;AACrB;AAKA,SAAS,SAAe,KAAQ,OAA6B;AAC3D,SAAO,CAAC,KAAK,OAAO,wBAAwB,MAAM,GAAG,CAAC;AACxD;AAEA,SAAS,OAAa,uBAA6C;AACjE,SAAO,sBAAsB,CAAC;AAChC;AAEA,SAAS,OAAa,uBAA6C;AACjE,SAAO,sBAAsB,CAAC;AAChC;AAEA,SAAS,OAAa,uBAA+C;AACnE,SAAO,sBAAsB,CAAC;AAChC;"}
+{"version":3,"file":"topKWithFractionalIndex.js","sources":["../../../src/operators/topKWithFractionalIndex.ts"],"sourcesContent":["import { generateKeyBetween } from \"fractional-indexing\"\nimport { DifferenceStreamWriter, UnaryOperator } from \"../graph.js\"\nimport { StreamBuilder } from \"../d2.js\"\nimport { MultiSet } from \"../multiset.js\"\nimport { binarySearch, globalObjectIdGenerator } from \"../utils.js\"\nimport type { DifferenceStreamReader } from \"../graph.js\"\nimport type { IStreamBuilder, PipedOperator } from \"../types.js\"\n\nexport interface TopKWithFractionalIndexOptions {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n}\n\nexport type TopKChanges<V> = {\n  /** Indicates which element moves into the topK (if any) */\n  moveIn: IndexedValue<V> | null\n  /** Indicates which element moves out of the topK (if any) */\n  moveOut: IndexedValue<V> | null\n}\n\n/**\n * A topK data structure that supports insertions and deletions\n * and returns changes to the topK.\n */\nexport interface TopK<V> {\n  size: number\n  insert: (value: V) => TopKChanges<V>\n  delete: (value: V) => TopKChanges<V>\n}\n\n/**\n * Implementation of a topK data structure.\n * Uses a sorted array internally to store the values and keeps a topK window over that array.\n * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted\n * at the start of the array which causes all the elements to shift to the right/left.\n */\nclass TopKArray<V> implements TopK<V> {\n  #sortedValues: Array<IndexedValue<V>> = []\n  #comparator: (a: V, b: V) => number\n  #topKStart: number\n  #topKEnd: number\n\n  constructor(\n    offset: number,\n    limit: number,\n    comparator: (a: V, b: V) => number\n  ) {\n    this.#topKStart = offset\n    this.#topKEnd = offset + limit\n    this.#comparator = comparator\n  }\n\n  get size(): number {\n    const offset = this.#topKStart\n    const limit = this.#topKEnd - this.#topKStart\n    const available = this.#sortedValues.length - offset\n    return Math.max(0, Math.min(limit, available))\n  }\n\n  insert(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup insert position\n    const index = this.#findIndex(value)\n    // Generate fractional index based on the fractional indices of the elements before and after it\n    const indexBefore =\n      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)\n    const indexAfter =\n      index === this.#sortedValues.length\n        ? null\n        : getIndex(this.#sortedValues[index]!)\n    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)\n\n    // Insert the value at the correct position\n    const val = indexedValue(value, fractionalIndex)\n    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !\n    this.#sortedValues.splice(index, 0, val)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The inserted element is either before the top K or within the top K\n      // If it is before the top K then it moves the element that was right before the topK into the topK\n      // If it is within the top K then the inserted element moves into the top K\n      // In both cases the last element of the old top K now moves out of the top K\n      const moveInIndex = Math.max(index, this.#topKStart)\n      if (moveInIndex < this.#sortedValues.length) {\n        // We actually have a topK\n        // because in some cases there may not be enough elements in the array to reach the start of the topK\n        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK\n        result.moveIn = this.#sortedValues[moveInIndex]!\n\n        // We need to remove the element that falls out of the top K\n        // The element that falls out of the top K has shifted one to the right\n        // because of the element we inserted, so we find it at index topKEnd\n        if (this.#topKEnd < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[this.#topKEnd]!\n        }\n      }\n    }\n\n    return result\n  }\n\n  /**\n   * Deletes a value that may or may not be in the topK.\n   * IMPORTANT: this assumes that the value is present in the collection\n   *            if it's not the case it will remove the element\n   *            that is on the position where the provided `value` would be.\n   */\n  delete(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup delete position\n    const index = this.#findIndex(value)\n    // Remove the value at that position\n    const [removedElem] = this.#sortedValues.splice(index, 1)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The removed element is either before the top K or within the top K\n      // If it is before the top K then the first element of the topK moves out of the topK\n      // If it is within the top K then the removed element moves out of the topK\n      result.moveOut = removedElem!\n      if (index < this.#topKStart) {\n        // The removed element is before the topK\n        // so actually, the first element of the topK moves out of the topK\n        // and not the element that we removed\n        // The first element of the topK is now at index topKStart - 1\n        // since we removed an element before the topK\n        const moveOutIndex = this.#topKStart - 1\n        if (moveOutIndex < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[moveOutIndex]!\n        } else {\n          // No value is moving out of the topK\n          // because there are no elements in the topK\n          result.moveOut = null\n        }\n      }\n\n      // Since we removed an element that was before or in the topK\n      // the first element after the topK moved one position to the left\n      // and thus falls into the topK now\n      const moveInIndex = this.#topKEnd - 1\n      if (moveInIndex < this.#sortedValues.length) {\n        result.moveIn = this.#sortedValues[moveInIndex]!\n      }\n    }\n\n    return result\n  }\n\n  // TODO: see if there is a way to refactor the code for insert and delete in the topK above\n  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right\n  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern\n\n  #findIndex(value: V): number {\n    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>\n      this.#comparator(getValue(a), getValue(b))\n    )\n  }\n}\n\n/**\n * Operator for fractional indexed topK operations\n * This operator maintains fractional indices for sorted elements\n * and only updates indices when elements move position\n */\nexport class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<\n  [K, T],\n  [K, IndexedValue<T>]\n> {\n  #index: Map<K, number> = new Map() // maps keys to their multiplicity\n\n  /**\n   * topK data structure that supports insertions and deletions\n   * and returns changes to the topK.\n   */\n  #topK: TopK<TaggedValue<K, T>>\n\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: TopKWithFractionalIndexOptions\n  ) {\n    super(id, inputA, output)\n    this.#limit = options.limit ?? Infinity\n    const offset = options.offset ?? 0\n    const compareTaggedValues = (\n      a: TaggedValue<K, T>,\n      b: TaggedValue<K, T>\n    ) => {\n      // First compare on the value\n      const valueComparison = comparator(getVal(a), getVal(b))\n      if (valueComparison !== 0) {\n        return valueComparison\n      }\n      // If the values are equal, compare on the tag (object identity)\n      const tieBreakerA = getTag(a)\n      const tieBreakerB = getTag(b)\n      return tieBreakerA - tieBreakerB\n    }\n    this.#topK = this.createTopK(offset, this.#limit, compareTaggedValues)\n    options.setSizeCallback?.(() => this.#topK.size)\n  }\n\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: TaggedValue<K, T>, b: TaggedValue<K, T>) => number\n  ): TopK<TaggedValue<K, T>> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.processElement(key, value, multiplicity, result)\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>\n  ): void {\n    const { oldMultiplicity, newMultiplicity } = this.addKey(key, multiplicity)\n\n    let res: TopKChanges<TaggedValue<K, T>> = {\n      moveIn: null,\n      moveOut: null,\n    }\n    if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n      // The value was invisible but should now be visible\n      // Need to insert it into the array of sorted values\n      const taggedValue = tagValue(key, value)\n      res = this.#topK.insert(taggedValue)\n    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n      // The value was visible but should now be invisible\n      // Need to remove it from the array of sorted values\n      const taggedValue = tagValue(key, value)\n      res = this.#topK.delete(taggedValue)\n    } else {\n      // The value was invisible and it remains invisible\n      // or it was visible and remains visible\n      // so it doesn't affect the topK\n    }\n\n    if (res.moveIn) {\n      const index = getIndex(res.moveIn)\n      const taggedValue = getValue(res.moveIn)\n      const k = getKey(taggedValue)\n      const val = getVal(taggedValue)\n      result.push([[k, [val, index]], 1])\n    }\n\n    if (res.moveOut) {\n      const index = getIndex(res.moveOut)\n      const taggedValue = getValue(res.moveOut)\n      const k = getKey(taggedValue)\n      const val = getVal(taggedValue)\n      result.push([[k, [val, index]], -1])\n    }\n\n    return\n  }\n\n  private getMultiplicity(key: K): number {\n    return this.#index.get(key) ?? 0\n  }\n\n  private addKey(\n    key: K,\n    multiplicity: number\n  ): { oldMultiplicity: number; newMultiplicity: number } {\n    const oldMultiplicity = this.getMultiplicity(key)\n    const newMultiplicity = oldMultiplicity + multiplicity\n    if (newMultiplicity === 0) {\n      this.#index.delete(key)\n    } else {\n      this.#index.set(key, newMultiplicity)\n    }\n    return { oldMultiplicity, newMultiplicity }\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithFractionalIndex<KType, T>(\n  comparator: (a: T, b: T) => number,\n  options?: TopKWithFractionalIndexOptions\n): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {\n  const opts = options || {}\n\n  return (\n    stream: IStreamBuilder<[KType, T]>\n  ): IStreamBuilder<[KType, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[KType, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[KType, IndexedValue<T>]>()\n    )\n    const operator = new TopKWithFractionalIndexOperator<KType, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      opts\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n\n// Abstraction for fractionally indexed values\nexport type FractionalIndex = string\nexport type IndexedValue<V> = [V, FractionalIndex]\n\nexport function indexedValue<V>(\n  value: V,\n  index: FractionalIndex\n): IndexedValue<V> {\n  return [value, index]\n}\n\nexport function getValue<V>(indexedVal: IndexedValue<V>): V {\n  return indexedVal[0]\n}\n\nexport function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {\n  return indexedVal[1]\n}\n\nexport type Tag = number\nexport type TaggedValue<K, V> = [K, V, Tag]\n\nfunction tagValue<K, V>(key: K, value: V): TaggedValue<K, V> {\n  return [key, value, globalObjectIdGenerator.getId(key)]\n}\n\nfunction getKey<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): K {\n  return tieBreakerTaggedValue[0]\n}\n\nfunction getVal<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): V {\n  return tieBreakerTaggedValue[1]\n}\n\nfunction getTag<K, V>(tieBreakerTaggedValue: TaggedValue<K, V>): Tag {\n  return tieBreakerTaggedValue[2]\n}\n"],"names":[],"mappings":";;;;;AAqCA,MAAM,UAAgC;AAAA,EACpC,gBAAwC,CAAA;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,QACA,OACA,YACA;AACA,SAAK,aAAa;AAClB,SAAK,WAAW,SAAS;AACzB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,OAAe;AACjB,UAAM,SAAS,KAAK;AACpB,UAAM,QAAQ,KAAK,WAAW,KAAK;AACnC,UAAM,YAAY,KAAK,cAAc,SAAS;AAC9C,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,SAAS,CAAC;AAAA,EAC/C;AAAA,EAEA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,cACJ,UAAU,IAAI,OAAO,SAAS,KAAK,cAAc,QAAQ,CAAC,CAAE;AAC9D,UAAM,aACJ,UAAU,KAAK,cAAc,SACzB,OACA,SAAS,KAAK,cAAc,KAAK,CAAE;AACzC,UAAM,kBAAkB,mBAAmB,aAAa,UAAU;AAGlE,UAAM,MAAM,aAAa,OAAO,eAAe;AAE/C,SAAK,cAAc,OAAO,OAAO,GAAG,GAAG;AAGvC,QAAI,QAAQ,KAAK,UAAU;AAKzB,YAAM,cAAc,KAAK,IAAI,OAAO,KAAK,UAAU;AACnD,UAAI,cAAc,KAAK,cAAc,QAAQ;AAI3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAK9C,YAAI,KAAK,WAAW,KAAK,cAAc,QAAQ;AAC7C,iBAAO,UAAU,KAAK,cAAc,KAAK,QAAQ;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,CAAC,WAAW,IAAI,KAAK,cAAc,OAAO,OAAO,CAAC;AAGxD,QAAI,QAAQ,KAAK,UAAU;AAIzB,aAAO,UAAU;AACjB,UAAI,QAAQ,KAAK,YAAY;AAM3B,cAAM,eAAe,KAAK,aAAa;AACvC,YAAI,eAAe,KAAK,cAAc,QAAQ;AAC5C,iBAAO,UAAU,KAAK,cAAc,YAAY;AAAA,QAClD,OAAO;AAGL,iBAAO,UAAU;AAAA,QACnB;AAAA,MACF;AAKA,YAAM,cAAc,KAAK,WAAW;AACpC,UAAI,cAAc,KAAK,cAAc,QAAQ;AAC3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAAA,MAChD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,OAAkB;AAC3B,WAAO;AAAA,MAAa,KAAK;AAAA,MAAe,aAAa,OAAO,EAAE;AAAA,MAAG,CAAC,GAAG,MACnE,KAAK,YAAY,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC;AAAA,IAAA;AAAA,EAE7C;AACF;AAOO,MAAM,wCAA8C,cAGzD;AAAA,EACA,6BAA6B,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAM7B;AAAA,EAEA;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,SAAS,QAAQ,SAAS;AAC/B,UAAM,SAAS,QAAQ,UAAU;AACjC,UAAM,sBAAsB,CAC1B,GACA,MACG;AAEH,YAAM,kBAAkB,WAAW,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACvD,UAAI,oBAAoB,GAAG;AACzB,eAAO;AAAA,MACT;AAEA,YAAM,cAAc,OAAO,CAAC;AAC5B,YAAM,cAAc,OAAO,CAAC;AAC5B,aAAO,cAAc;AAAA,IACvB;AACA,SAAK,QAAQ,KAAK,WAAW,QAAQ,KAAK,QAAQ,mBAAmB;AACrE,YAAQ,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,EACjD;AAAA,EAEU,WACR,QACA,OACA,YACyB;AACzB,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,aAAK,eAAe,KAAK,OAAO,cAAc,MAAM;AAAA,MACtD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,eACE,KACA,OACA,cACA,QACM;AACN,UAAM,EAAE,iBAAiB,gBAAA,IAAoB,KAAK,OAAO,KAAK,YAAY;AAE1E,QAAI,MAAsC;AAAA,MACxC,QAAQ;AAAA,MACR,SAAS;AAAA,IAAA;AAEX,QAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAG/C,YAAM,cAAc,SAAS,KAAK,KAAK;AACvC,YAAM,KAAK,MAAM,OAAO,WAAW;AAAA,IACrC,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAGtD,YAAM,cAAc,SAAS,KAAK,KAAK;AACvC,YAAM,KAAK,MAAM,OAAO,WAAW;AAAA,IACrC,MAAO;AAMP,QAAI,IAAI,QAAQ;AACd,YAAM,QAAQ,SAAS,IAAI,MAAM;AACjC,YAAM,cAAc,SAAS,IAAI,MAAM;AACvC,YAAM,IAAI,OAAO,WAAW;AAC5B,YAAM,MAAM,OAAO,WAAW;AAC9B,aAAO,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,IACpC;AAEA,QAAI,IAAI,SAAS;AACf,YAAM,QAAQ,SAAS,IAAI,OAAO;AAClC,YAAM,cAAc,SAAS,IAAI,OAAO;AACxC,YAAM,IAAI,OAAO,WAAW;AAC5B,YAAM,MAAM,OAAO,WAAW;AAC9B,aAAO,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,EAAE,CAAC;AAAA,IACrC;AAEA;AAAA,EACF;AAAA,EAEQ,gBAAgB,KAAgB;AACtC,WAAO,KAAK,OAAO,IAAI,GAAG,KAAK;AAAA,EACjC;AAAA,EAEQ,OACN,KACA,cACsD;AACtD,UAAM,kBAAkB,KAAK,gBAAgB,GAAG;AAChD,UAAM,kBAAkB,kBAAkB;AAC1C,QAAI,oBAAoB,GAAG;AACzB,WAAK,OAAO,OAAO,GAAG;AAAA,IACxB,OAAO;AACL,WAAK,OAAO,IAAI,KAAK,eAAe;AAAA,IACtC;AACA,WAAO,EAAE,iBAAiB,gBAAA;AAAA,EAC5B;AACF;AAYO,SAAS,wBACd,YACA,SACqD;AACrD,QAAM,OAAO,WAAW,CAAA;AAExB,SAAO,CACL,WAC6C;AAC7C,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAAiD;AAEvD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;AAMO,SAAS,aACd,OACA,OACiB;AACjB,SAAO,CAAC,OAAO,KAAK;AACtB;AAEO,SAAS,SAAY,YAAgC;AAC1D,SAAO,WAAW,CAAC;AACrB;AAEO,SAAS,SAAY,YAA8C;AACxE,SAAO,WAAW,CAAC;AACrB;AAKA,SAAS,SAAe,KAAQ,OAA6B;AAC3D,SAAO,CAAC,KAAK,OAAO,wBAAwB,MAAM,GAAG,CAAC;AACxD;AAEA,SAAS,OAAa,uBAA6C;AACjE,SAAO,sBAAsB,CAAC;AAChC;AAEA,SAAS,OAAa,uBAA6C;AACjE,SAAO,sBAAsB,CAAC;AAChC;AAEA,SAAS,OAAa,uBAA+C;AACnE,SAAO,sBAAsB,CAAC;AAChC;"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tanstack/db-ivm",
   "description": "Incremental View Maintenance for TanStack DB based on Differential Dataflow",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "dependencies": {
     "fractional-indexing": "^3.2.0",
     "sorted-btree": "^1.8.1"

package/src/indexes.ts

@@ -150,11 +150,30 @@ export class Index<TKey, TValue, TPrefix = any> {
    *   hash to identify identical values, storing them in a third level value map.
    */
   #inner: IndexMap<TKey, TValue, TPrefix>
+  #consolidatedMultiplicity: Map<TKey, number> = new Map() // sum of multiplicities per key
 
   constructor() {
     this.#inner = new Map()
   }
 
+  /**
+   * Create an Index from multiple MultiSet messages.
+   * @param messages - Array of MultiSet messages to build the index from.
+   * @returns A new Index containing all the data from the messages.
+   */
+  static fromMultiSets<K, V>(messages: Array<MultiSet<[K, V]>>): Index<K, V> {
+    const index = new Index<K, V>()
+
+    for (const message of messages) {
+      for (const [item, multiplicity] of message.getInner()) {
+        const [key, value] = item
+        index.addValue(key, [value, multiplicity])
+      }
+    }
+
+    return index
+  }
+
   /**
    * This method returns a string representation of the index.
    * @param indent - Whether to indent the string representation.
@@ -184,6 +203,32 @@ export class Index<TKey, TValue, TPrefix = any> {
     return this.#inner.has(key)
   }
 
+  /**
+   * Check if a key has presence (non-zero consolidated multiplicity).
+   * @param key - The key to check.
+   * @returns True if the key has non-zero consolidated multiplicity, false otherwise.
+   */
+  hasPresence(key: TKey): boolean {
+    return (this.#consolidatedMultiplicity.get(key) || 0) !== 0
+  }
+
+  /**
+   * Get the consolidated multiplicity (sum of multiplicities) for a key.
+   * @param key - The key to get the consolidated multiplicity for.
+   * @returns The consolidated multiplicity for the key.
+   */
+  getConsolidatedMultiplicity(key: TKey): number {
+    return this.#consolidatedMultiplicity.get(key) || 0
+  }
+
+  /**
+   * Get all keys that have presence (non-zero consolidated multiplicity).
+   * @returns An iterator of keys with non-zero consolidated multiplicity.
+   */
+  getPresenceKeys(): Iterable<TKey> {
+    return this.#consolidatedMultiplicity.keys()
+  }
+
   /**
    * This method returns all values for a given key.
    * @param key - The key to get the values for.
@@ -257,6 +302,15 @@ export class Index<TKey, TValue, TPrefix = any> {
     // If the multiplicity is 0, do nothing
     if (multiplicity === 0) return
 
+    // Update consolidated multiplicity tracking
+    const newConsolidatedMultiplicity =
+      (this.#consolidatedMultiplicity.get(key) || 0) + multiplicity
+    if (newConsolidatedMultiplicity === 0) {
+      this.#consolidatedMultiplicity.delete(key)
+    } else {
+      this.#consolidatedMultiplicity.set(key, newConsolidatedMultiplicity)
+    }
+
     const mapOrSingleValue = this.#inner.get(key)
 
     if (mapOrSingleValue === undefined) {

package/src/multiset.ts

@@ -209,6 +209,12 @@ export class MultiSet<T> {
     chunkedArrayPush(this.#inner, otherArray)
   }
 
+  add(item: T, multiplicity: number): void {
+    if (multiplicity !== 0) {
+      this.#inner.push([item, multiplicity])
+    }
+  }
+
   getInner(): MultiSetArray<T> {
     return this.#inner
   }

package/src/operators/filterBy.ts

@@ -1,8 +1,7 @@
 import { map } from "./map.js"
 import { innerJoin } from "./join.js"
 import { consolidate } from "./consolidate.js"
-import type { KeyValue } from "../types.js"
-import type { IStreamBuilder, PipedOperator } from "../types"
+import type { IStreamBuilder, KeyValue, PipedOperator } from "../types.js"
 
 /**
  * Filters the elements of a keyed stream, by keys of another stream.