@tanstack/db-ivm 0.1.20 → 0.1.22

package/dist/esm/hashing/hash.js

@@ -6,10 +6,12 @@ const UNDEFINED = randomHash();
 const KEY = randomHash();
 const FUNCTIONS = randomHash();
 const DATE_MARKER = randomHash();
-const OBJECT_MARKER = randomHash();
-const ARRAY_MARKER = randomHash();
-const MAP_MARKER = randomHash();
-const SET_MARKER = randomHash();
+const STRUCTURAL_MARKERS = {
+  object: randomHash(),
+  array: randomHash(),
+  map: randomHash(),
+  set: randomHash()
+};
 const UINT8ARRAY_MARKER = randomHash();
 const TEMPORAL_MARKER = randomHash();
 const MAX_STRUCTURAL_HASH_WORK = 1e6;
@@ -30,6 +32,7 @@ function isTemporal(input) {
 }
 const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128;
 const hashCache = /* @__PURE__ */ new WeakMap();
+const referenceValues = /* @__PURE__ */ new WeakSet();
 function registerOpaqueHash(value) {
   cachedReferenceHash(value);
 }
@@ -54,20 +57,12 @@ function hashObject(input, context) {
     } else if (isTemporal(input)) {
       valueHash = hashTemporal(input);
     } else {
-      let plainObjectInput = input;
-      let marker = OBJECT_MARKER;
-      if (input instanceof Array) {
-        marker = ARRAY_MARKER;
-      }
-      if (input instanceof Map) {
-        marker = MAP_MARKER;
-        plainObjectInput = [...input.entries()];
-      }
-      if (input instanceof Set) {
-        marker = SET_MARKER;
-        plainObjectInput = [...input.entries()];
-      }
-      valueHash = hashPlainObject(plainObjectInput, marker, context);
+      const [kind, plainObjectInput] = structuralShape(input);
+      valueHash = hashPlainObject(
+        plainObjectInput,
+        STRUCTURAL_MARKERS[kind],
+        context
+      );
     }
   } finally {
     context.activeObjects.delete(input);
@@ -180,6 +175,43 @@ function isReferenceHashedObject(input) {
 function isBinaryValue(input) {
   return typeof Buffer !== `undefined` && input instanceof Buffer || input instanceof Uint8Array;
 }
+function structuralShape(input) {
+  if (input instanceof Map) return [`map`, [...input.entries()]];
+  if (input instanceof Set) return [`set`, [...input.values()]];
+  return [input instanceof Array ? `array` : `object`, input];
+}
+function equalHashValues(left, right) {
+  const compared = /* @__PURE__ */ new Map();
+  function equal(a, b) {
+    if (a === b || Number.isNaN(a) && Number.isNaN(b)) return true;
+    if (a === null || b === null || typeof a !== `object` || typeof b !== `object`)
+      return false;
+    if (referenceValues.has(a) || referenceValues.has(b) || isReferenceHashedObject(a) || isReferenceHashedObject(b))
+      return false;
+    if (a instanceof Date || b instanceof Date)
+      return a instanceof Date && b instanceof Date && equal(a.getTime(), b.getTime());
+    if (isBinaryValue(a) || isBinaryValue(b))
+      return isBinaryValue(a) && isBinaryValue(b) && a.byteLength === b.byteLength && a.every((value, index) => value === b[index]);
+    if (isTemporal(a) || isTemporal(b))
+      return isTemporal(a) && isTemporal(b) && a[Symbol.toStringTag] === b[Symbol.toStringTag] && a.toString() === b.toString();
+    const peers = compared.get(a);
+    if (peers?.has(b)) return true;
+    if (peers) peers.add(b);
+    else compared.set(a, /* @__PURE__ */ new Set([b]));
+    const [aKind, aShape] = structuralShape(a);
+    const [bKind, bShape] = structuralShape(b);
+    if (aKind !== bKind) return false;
+    const keys = (value) => Reflect.ownKeys(value).filter(
+      (key) => Object.prototype.propertyIsEnumerable.call(value, key)
+    );
+    const aKeys = keys(aShape);
+    if (aKeys.length !== keys(bShape).length) return false;
+    return aKeys.every(
+      (key) => Object.prototype.propertyIsEnumerable.call(bShape, key) && equal(aShape[key], bShape[key])
+    );
+  }
+  return equal(left, right);
+}
 let nextRefId = 1;
 function cachedReferenceHash(fn) {
   let valueHash = hashCache.get(fn);
@@ -187,6 +219,7 @@ function cachedReferenceHash(fn) {
     valueHash = nextRefId ^ FUNCTIONS;
     nextRefId++;
     hashCache.set(fn, valueHash);
+    referenceValues.add(fn);
   }
   return valueHash;
 }
@@ -194,6 +227,7 @@ function keySort(a, b) {
   return a.localeCompare(b);
 }
 export {
+  equalHashValues,
   hash,
   registerOpaqueHash
 };

package/dist/esm/hashing/hash.js.map

@@ -1 +1 @@
-{"version":3,"file":"hash.js","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, getSymbolIdentity, randomHash } from './murmur.js'\nimport type { Hasher } from './murmur.js'\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst OBJECT_MARKER = randomHash()\nconst ARRAY_MARKER = randomHash()\nconst MAP_MARKER = randomHash()\nconst SET_MARKER = randomHash()\nconst UINT8ARRAY_MARKER = randomHash()\nconst TEMPORAL_MARKER = randomHash()\n// Bound structural recursion and value visits. Shared acyclic subtrees are\n// cached; cycles are rejected rather than given context-dependent hashes.\nconst MAX_STRUCTURAL_HASH_WORK = 1_000_000\nconst MAX_STRUCTURAL_HASH_DEPTH = 768\n\nconst temporalTypes = new Set([\n  `Temporal.Duration`,\n  `Temporal.Instant`,\n  `Temporal.PlainDate`,\n  `Temporal.PlainDateTime`,\n  `Temporal.PlainMonthDay`,\n  `Temporal.PlainTime`,\n  `Temporal.PlainYearMonth`,\n  `Temporal.ZonedDateTime`,\n])\n\ninterface TemporalLike {\n  [Symbol.toStringTag]: string\n  toString: () => string\n}\n\nfunction isTemporal(input: object): input is TemporalLike {\n  const tag = (input as Record<symbol, unknown>)[Symbol.toStringTag]\n  return typeof tag === `string` && temporalTypes.has(tag)\n}\n\n// Maximum byte length for Uint8Arrays to hash by content instead of reference\n// Arrays smaller than this will be hashed by content, allowing proper equality comparisons\n// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays\nconst UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128\n\nconst hashCache = new WeakMap<object, number>()\n\n/** @internal Register a mutable handle before it enters a structural value. */\nexport function registerOpaqueHash(value: object): void {\n  cachedReferenceHash(value)\n}\n\ntype HashContext = {\n  activeObjects: Set<object>\n  work: number\n  pendingHashes: Map<object, number>\n}\n\nexport function hash(input: any): number {\n  const hasher = new MurmurHashStream()\n  updateHasher(hasher, input)\n  return hasher.digest()\n}\n\nfunction hashObject(input: object, context: HashContext): number {\n  if (context.activeObjects.size >= MAX_STRUCTURAL_HASH_DEPTH) {\n    throw new RangeError(\n      `Value is too complex to hash safely: structural depth`,\n    )\n  }\n\n  context.activeObjects.add(input)\n\n  let valueHash: number | undefined\n  try {\n    if (input instanceof Date) {\n      valueHash = hashDate(input)\n    } else if (isBinaryValue(input)) {\n      valueHash = hashUint8Array(input)\n    } else if (isTemporal(input)) {\n      valueHash = hashTemporal(input)\n    } else {\n      let plainObjectInput = input\n      let marker = OBJECT_MARKER\n\n      if (input instanceof Array) {\n        marker = ARRAY_MARKER\n      }\n\n      if (input instanceof Map) {\n        marker = MAP_MARKER\n        plainObjectInput = [...input.entries()]\n      }\n\n      if (input instanceof Set) {\n        marker = SET_MARKER\n        plainObjectInput = [...input.entries()]\n      }\n\n      valueHash = hashPlainObject(plainObjectInput, marker, context)\n    }\n  } finally {\n    context.activeObjects.delete(input)\n  }\n\n  context.pendingHashes.set(input, valueHash)\n  return valueHash\n}\n\nfunction hashDate(input: Date): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(DATE_MARKER)\n  hasher.update(input.getTime())\n  return hasher.digest()\n}\n\nfunction hashUint8Array(input: Uint8Array): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(UINT8ARRAY_MARKER)\n  // Hash the byte length first to differentiate arrays of different sizes\n  hasher.update(input.byteLength)\n  // Hash each byte in the array\n  for (let i = 0; i < input.byteLength; i++) {\n    hasher.writeByte(input[i]!)\n  }\n  return hasher.digest()\n}\n\nfunction hashTemporal(input: TemporalLike): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(TEMPORAL_MARKER)\n  hasher.update(input[Symbol.toStringTag])\n  hasher.update(input.toString())\n  return hasher.digest()\n}\n\nfunction hashPlainObject(\n  input: object,\n  marker: number,\n  context: HashContext,\n): number {\n  const hasher = new MurmurHashStream()\n\n  // Mark the type of the input\n  hasher.update(marker)\n  const keys = Object.keys(input)\n  keys.sort(keySort)\n  for (const key of keys) {\n    hasher.update(KEY)\n    hasher.update(key)\n    updateHasher(hasher, input[key as keyof typeof input], context)\n  }\n  const symbolKeys = Object.getOwnPropertySymbols(input)\n    .filter((key) => Object.prototype.propertyIsEnumerable.call(input, key))\n    .sort((left, right) => getSymbolIdentity(left) - getSymbolIdentity(right))\n  for (const key of symbolKeys) {\n    hasher.update(KEY)\n    hasher.update(key)\n    updateHasher(hasher, input[key as keyof typeof input], context)\n  }\n\n  return hasher.digest()\n}\n\nfunction updateHasher(\n  hasher: Hasher,\n  input: unknown,\n  context?: HashContext,\n): void {\n  if (context && ++context.work > MAX_STRUCTURAL_HASH_WORK) {\n    throw new RangeError(`Value is too complex to hash safely: structural work`)\n  }\n  if (input === null) {\n    hasher.update(NULL)\n    return\n  }\n  switch (typeof input) {\n    case `undefined`:\n      hasher.update(UNDEFINED)\n      return\n    case `boolean`:\n      hasher.update(input ? TRUE : FALSE)\n      return\n    case `number`:\n      // Normalize NaNs and -0\n      hasher.update(isNaN(input) ? NaN : input === 0 ? 0 : input)\n      return\n    case `bigint`:\n    case `string`:\n    case `symbol`:\n      hasher.update(input)\n      return\n    case `object`:\n      hasher.update(getCachedHash(input, context))\n      return\n    case `function`:\n      // Functions are assigned a globally unique ID\n      // and that ID is cached in the weak map\n      hasher.update(cachedReferenceHash(input))\n      return\n    default:\n      console.warn(\n        `Ignored input during hashing because it is of type ${typeof input} which is not supported`,\n      )\n  }\n}\n\nfunction getCachedHash(input: object, context?: HashContext): number {\n  if (!context) {\n    const cached = hashCache.get(input)\n    if (cached !== undefined) return cached\n    if (isReferenceHashedObject(input)) return cachedReferenceHash(input)\n\n    // Only an uncached structural root needs graph traversal state. Commit its\n    // cache entries after success so a failed traversal cannot poison retries.\n    context = {\n      activeObjects: new Set(),\n      work: 0,\n      pendingHashes: new Map(),\n    }\n    const result = hashObject(input, context)\n    for (const [object, valueHash] of context.pendingHashes) {\n      hashCache.set(object, valueHash)\n    }\n    return result\n  }\n\n  if (context.activeObjects.has(input)) {\n    throw new TypeError(`Cannot hash cyclic structural values`)\n  }\n\n  // Opaque leaves cannot contain structural back-references. Resolve them\n  // before entering structural recursion, even when they have user properties.\n  if (isReferenceHashedObject(input)) return cachedReferenceHash(input)\n\n  const valueHash = hashCache.get(input) ?? context.pendingHashes.get(input)\n  if (valueHash !== undefined) return valueHash\n\n  return hashObject(input, context)\n}\n\nfunction isReferenceHashedObject(input: object): boolean {\n  return (\n    input instanceof File ||\n    (isBinaryValue(input) &&\n      input.byteLength > UINT8ARRAY_CONTENT_HASH_THRESHOLD)\n  )\n}\n\nfunction isBinaryValue(input: object): input is Uint8Array {\n  return (\n    (typeof Buffer !== `undefined` && input instanceof Buffer) ||\n    input instanceof Uint8Array\n  )\n}\n\nlet nextRefId = 1\nfunction cachedReferenceHash(fn: object): number {\n  let valueHash = hashCache.get(fn)\n  if (valueHash === undefined) {\n    valueHash = nextRefId ^ FUNCTIONS\n    nextRefId++\n    hashCache.set(fn, valueHash)\n  }\n  return valueHash\n}\n\n/**\n * Strings sorted lexicographically.\n */\nfunction keySort(a: string, b: string): number {\n  return a.localeCompare(b)\n}\n"],"names":["valueHash"],"mappings":";AAQA,MAAM,OAAO,WAAA;AACb,MAAM,QAAQ,WAAA;AACd,MAAM,OAAO,WAAA;AACb,MAAM,YAAY,WAAA;AAClB,MAAM,MAAM,WAAA;AACZ,MAAM,YAAY,WAAA;AAClB,MAAM,cAAc,WAAA;AACpB,MAAM,gBAAgB,WAAA;AACtB,MAAM,eAAe,WAAA;AACrB,MAAM,aAAa,WAAA;AACnB,MAAM,aAAa,WAAA;AACnB,MAAM,oBAAoB,WAAA;AAC1B,MAAM,kBAAkB,WAAA;AAGxB,MAAM,2BAA2B;AACjC,MAAM,4BAA4B;AAElC,MAAM,oCAAoB,IAAI;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAOD,SAAS,WAAW,OAAsC;AACxD,QAAM,MAAO,MAAkC,OAAO,WAAW;AACjE,SAAO,OAAO,QAAQ,YAAY,cAAc,IAAI,GAAG;AACzD;AAKA,MAAM,oCAAoC;AAE1C,MAAM,gCAAgB,QAAA;AAGf,SAAS,mBAAmB,OAAqB;AACtD,sBAAoB,KAAK;AAC3B;AAQO,SAAS,KAAK,OAAoB;AACvC,QAAM,SAAS,IAAI,iBAAA;AACnB,eAAa,QAAQ,KAAK;AAC1B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,WAAW,OAAe,SAA8B;AAC/D,MAAI,QAAQ,cAAc,QAAQ,2BAA2B;AAC3D,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AAEA,UAAQ,cAAc,IAAI,KAAK;AAE/B,MAAI;AACJ,MAAI;AACF,QAAI,iBAAiB,MAAM;AACzB,kBAAY,SAAS,KAAK;AAAA,IAC5B,WAAW,cAAc,KAAK,GAAG;AAC/B,kBAAY,eAAe,KAAK;AAAA,IAClC,WAAW,WAAW,KAAK,GAAG;AAC5B,kBAAY,aAAa,KAAK;AAAA,IAChC,OAAO;AACL,UAAI,mBAAmB;AACvB,UAAI,SAAS;AAEb,UAAI,iBAAiB,OAAO;AAC1B,iBAAS;AAAA,MACX;AAEA,UAAI,iBAAiB,KAAK;AACxB,iBAAS;AACT,2BAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,MACxC;AAEA,UAAI,iBAAiB,KAAK;AACxB,iBAAS;AACT,2BAAmB,CAAC,GAAG,MAAM,SAAS;AAAA,MACxC;AAEA,kBAAY,gBAAgB,kBAAkB,QAAQ,OAAO;AAAA,IAC/D;AAAA,EACF,UAAA;AACE,YAAQ,cAAc,OAAO,KAAK;AAAA,EACpC;AAEA,UAAQ,cAAc,IAAI,OAAO,SAAS;AAC1C,SAAO;AACT;AAEA,SAAS,SAAS,OAAqB;AACrC,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,WAAW;AACzB,SAAO,OAAO,MAAM,SAAS;AAC7B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,eAAe,OAA2B;AACjD,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,iBAAiB;AAE/B,SAAO,OAAO,MAAM,UAAU;AAE9B,WAAS,IAAI,GAAG,IAAI,MAAM,YAAY,KAAK;AACzC,WAAO,UAAU,MAAM,CAAC,CAAE;AAAA,EAC5B;AACA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aAAa,OAA6B;AACjD,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,eAAe;AAC7B,SAAO,OAAO,MAAM,OAAO,WAAW,CAAC;AACvC,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,gBACP,OACA,QACA,SACQ;AACR,QAAM,SAAS,IAAI,iBAAA;AAGnB,SAAO,OAAO,MAAM;AACpB,QAAM,OAAO,OAAO,KAAK,KAAK;AAC9B,OAAK,KAAK,OAAO;AACjB,aAAW,OAAO,MAAM;AACtB,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,GAAG,OAAO;AAAA,EAChE;AACA,QAAM,aAAa,OAAO,sBAAsB,KAAK,EAClD,OAAO,CAAC,QAAQ,OAAO,UAAU,qBAAqB,KAAK,OAAO,GAAG,CAAC,EACtE,KAAK,CAAC,MAAM,UAAU,kBAAkB,IAAI,IAAI,kBAAkB,KAAK,CAAC;AAC3E,aAAW,OAAO,YAAY;AAC5B,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,GAAG,OAAO;AAAA,EAChE;AAEA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aACP,QACA,OACA,SACM;AACN,MAAI,WAAW,EAAE,QAAQ,OAAO,0BAA0B;AACxD,UAAM,IAAI,WAAW,sDAAsD;AAAA,EAC7E;AACA,MAAI,UAAU,MAAM;AAClB,WAAO,OAAO,IAAI;AAClB;AAAA,EACF;AACA,UAAQ,OAAO,OAAA;AAAA,IACb,KAAK;AACH,aAAO,OAAO,SAAS;AACvB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,QAAQ,OAAO,KAAK;AAClC;AAAA,IACF,KAAK;AAEH,aAAO,OAAO,MAAM,KAAK,IAAI,MAAM,UAAU,IAAI,IAAI,KAAK;AAC1D;AAAA,IACF,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO,OAAO,KAAK;AACnB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,cAAc,OAAO,OAAO,CAAC;AAC3C;AAAA,IACF,KAAK;AAGH,aAAO,OAAO,oBAAoB,KAAK,CAAC;AACxC;AAAA,IACF;AACE,cAAQ;AAAA,QACN,sDAAsD,OAAO,KAAK;AAAA,MAAA;AAAA,EACpE;AAEN;AAEA,SAAS,cAAc,OAAe,SAA+B;AACnE,MAAI,CAAC,SAAS;AACZ,UAAM,SAAS,UAAU,IAAI,KAAK;AAClC,QAAI,WAAW,OAAW,QAAO;AACjC,QAAI,wBAAwB,KAAK,EAAG,QAAO,oBAAoB,KAAK;AAIpE,cAAU;AAAA,MACR,mCAAmB,IAAA;AAAA,MACnB,MAAM;AAAA,MACN,mCAAmB,IAAA;AAAA,IAAI;AAEzB,UAAM,SAAS,WAAW,OAAO,OAAO;AACxC,eAAW,CAAC,QAAQA,UAAS,KAAK,QAAQ,eAAe;AACvD,gBAAU,IAAI,QAAQA,UAAS;AAAA,IACjC;AACA,WAAO;AAAA,EACT;AAEA,MAAI,QAAQ,cAAc,IAAI,KAAK,GAAG;AACpC,UAAM,IAAI,UAAU,sCAAsC;AAAA,EAC5D;AAIA,MAAI,wBAAwB,KAAK,EAAG,QAAO,oBAAoB,KAAK;AAEpE,QAAM,YAAY,UAAU,IAAI,KAAK,KAAK,QAAQ,cAAc,IAAI,KAAK;AACzE,MAAI,cAAc,OAAW,QAAO;AAEpC,SAAO,WAAW,OAAO,OAAO;AAClC;AAEA,SAAS,wBAAwB,OAAwB;AACvD,SACE,iBAAiB,QAChB,cAAc,KAAK,KAClB,MAAM,aAAa;AAEzB;AAEA,SAAS,cAAc,OAAoC;AACzD,SACG,OAAO,WAAW,eAAe,iBAAiB,UACnD,iBAAiB;AAErB;AAEA,IAAI,YAAY;AAChB,SAAS,oBAAoB,IAAoB;AAC/C,MAAI,YAAY,UAAU,IAAI,EAAE;AAChC,MAAI,cAAc,QAAW;AAC3B,gBAAY,YAAY;AACxB;AACA,cAAU,IAAI,IAAI,SAAS;AAAA,EAC7B;AACA,SAAO;AACT;AAKA,SAAS,QAAQ,GAAW,GAAmB;AAC7C,SAAO,EAAE,cAAc,CAAC;AAC1B;"}
+{"version":3,"file":"hash.js","sources":["../../../src/hashing/hash.ts"],"sourcesContent":["import { MurmurHashStream, getSymbolIdentity, randomHash } from './murmur.js'\nimport type { Hasher } from './murmur.js'\n\n/*\n * Implementation of structural hashing based on the Composites polyfill implementation:\n * https://github.com/tc39/proposal-composites\n */\n\nconst TRUE = randomHash()\nconst FALSE = randomHash()\nconst NULL = randomHash()\nconst UNDEFINED = randomHash()\nconst KEY = randomHash()\nconst FUNCTIONS = randomHash()\nconst DATE_MARKER = randomHash()\nconst STRUCTURAL_MARKERS = {\n  object: randomHash(),\n  array: randomHash(),\n  map: randomHash(),\n  set: randomHash(),\n}\nconst UINT8ARRAY_MARKER = randomHash()\nconst TEMPORAL_MARKER = randomHash()\n// Bound structural recursion and value visits. Shared acyclic subtrees are\n// cached; cycles are rejected rather than given context-dependent hashes.\nconst MAX_STRUCTURAL_HASH_WORK = 1_000_000\nconst MAX_STRUCTURAL_HASH_DEPTH = 768\n\nconst temporalTypes = new Set([\n  `Temporal.Duration`,\n  `Temporal.Instant`,\n  `Temporal.PlainDate`,\n  `Temporal.PlainDateTime`,\n  `Temporal.PlainMonthDay`,\n  `Temporal.PlainTime`,\n  `Temporal.PlainYearMonth`,\n  `Temporal.ZonedDateTime`,\n])\n\ninterface TemporalLike {\n  [Symbol.toStringTag]: string\n  toString: () => string\n}\n\nfunction isTemporal(input: object): input is TemporalLike {\n  const tag = (input as Record<symbol, unknown>)[Symbol.toStringTag]\n  return typeof tag === `string` && temporalTypes.has(tag)\n}\n\n// Maximum byte length for Uint8Arrays to hash by content instead of reference\n// Arrays smaller than this will be hashed by content, allowing proper equality comparisons\n// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays\nconst UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128\n\nconst hashCache = new WeakMap<object, number>()\nconst referenceValues = new WeakSet<object>()\n\n/** @internal Register a mutable handle before it enters a structural value. */\nexport function registerOpaqueHash(value: object): void {\n  cachedReferenceHash(value)\n}\n\ntype HashContext = {\n  activeObjects: Set<object>\n  work: number\n  pendingHashes: Map<object, number>\n}\n\nexport function hash(input: any): number {\n  const hasher = new MurmurHashStream()\n  updateHasher(hasher, input)\n  return hasher.digest()\n}\n\nfunction hashObject(input: object, context: HashContext): number {\n  if (context.activeObjects.size >= MAX_STRUCTURAL_HASH_DEPTH) {\n    throw new RangeError(\n      `Value is too complex to hash safely: structural depth`,\n    )\n  }\n\n  context.activeObjects.add(input)\n\n  let valueHash: number | undefined\n  try {\n    if (input instanceof Date) {\n      valueHash = hashDate(input)\n    } else if (isBinaryValue(input)) {\n      valueHash = hashUint8Array(input)\n    } else if (isTemporal(input)) {\n      valueHash = hashTemporal(input)\n    } else {\n      const [kind, plainObjectInput] = structuralShape(input)\n      valueHash = hashPlainObject(\n        plainObjectInput,\n        STRUCTURAL_MARKERS[kind],\n        context,\n      )\n    }\n  } finally {\n    context.activeObjects.delete(input)\n  }\n\n  context.pendingHashes.set(input, valueHash)\n  return valueHash\n}\n\nfunction hashDate(input: Date): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(DATE_MARKER)\n  hasher.update(input.getTime())\n  return hasher.digest()\n}\n\nfunction hashUint8Array(input: Uint8Array): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(UINT8ARRAY_MARKER)\n  // Hash the byte length first to differentiate arrays of different sizes\n  hasher.update(input.byteLength)\n  // Hash each byte in the array\n  for (let i = 0; i < input.byteLength; i++) {\n    hasher.writeByte(input[i]!)\n  }\n  return hasher.digest()\n}\n\nfunction hashTemporal(input: TemporalLike): number {\n  const hasher = new MurmurHashStream()\n  hasher.update(TEMPORAL_MARKER)\n  hasher.update(input[Symbol.toStringTag])\n  hasher.update(input.toString())\n  return hasher.digest()\n}\n\nfunction hashPlainObject(\n  input: object,\n  marker: number,\n  context: HashContext,\n): number {\n  const hasher = new MurmurHashStream()\n\n  // Mark the type of the input\n  hasher.update(marker)\n  const keys = Object.keys(input)\n  keys.sort(keySort)\n  for (const key of keys) {\n    hasher.update(KEY)\n    hasher.update(key)\n    updateHasher(hasher, input[key as keyof typeof input], context)\n  }\n  const symbolKeys = Object.getOwnPropertySymbols(input)\n    .filter((key) => Object.prototype.propertyIsEnumerable.call(input, key))\n    .sort((left, right) => getSymbolIdentity(left) - getSymbolIdentity(right))\n  for (const key of symbolKeys) {\n    hasher.update(KEY)\n    hasher.update(key)\n    updateHasher(hasher, input[key as keyof typeof input], context)\n  }\n\n  return hasher.digest()\n}\n\nfunction updateHasher(\n  hasher: Hasher,\n  input: unknown,\n  context?: HashContext,\n): void {\n  if (context && ++context.work > MAX_STRUCTURAL_HASH_WORK) {\n    throw new RangeError(`Value is too complex to hash safely: structural work`)\n  }\n  if (input === null) {\n    hasher.update(NULL)\n    return\n  }\n  switch (typeof input) {\n    case `undefined`:\n      hasher.update(UNDEFINED)\n      return\n    case `boolean`:\n      hasher.update(input ? TRUE : FALSE)\n      return\n    case `number`:\n      // Normalize NaNs and -0\n      hasher.update(isNaN(input) ? NaN : input === 0 ? 0 : input)\n      return\n    case `bigint`:\n    case `string`:\n    case `symbol`:\n      hasher.update(input)\n      return\n    case `object`:\n      hasher.update(getCachedHash(input, context))\n      return\n    case `function`:\n      // Functions are assigned a globally unique ID\n      // and that ID is cached in the weak map\n      hasher.update(cachedReferenceHash(input))\n      return\n    default:\n      console.warn(\n        `Ignored input during hashing because it is of type ${typeof input} which is not supported`,\n      )\n  }\n}\n\nfunction getCachedHash(input: object, context?: HashContext): number {\n  if (!context) {\n    const cached = hashCache.get(input)\n    if (cached !== undefined) return cached\n    if (isReferenceHashedObject(input)) return cachedReferenceHash(input)\n\n    // Only an uncached structural root needs graph traversal state. Commit its\n    // cache entries after success so a failed traversal cannot poison retries.\n    context = {\n      activeObjects: new Set(),\n      work: 0,\n      pendingHashes: new Map(),\n    }\n    const result = hashObject(input, context)\n    for (const [object, valueHash] of context.pendingHashes) {\n      hashCache.set(object, valueHash)\n    }\n    return result\n  }\n\n  if (context.activeObjects.has(input)) {\n    throw new TypeError(`Cannot hash cyclic structural values`)\n  }\n\n  // Opaque leaves cannot contain structural back-references. Resolve them\n  // before entering structural recursion, even when they have user properties.\n  if (isReferenceHashedObject(input)) return cachedReferenceHash(input)\n\n  const valueHash = hashCache.get(input) ?? context.pendingHashes.get(input)\n  if (valueHash !== undefined) return valueHash\n\n  return hashObject(input, context)\n}\n\nfunction isReferenceHashedObject(input: object): boolean {\n  return (\n    input instanceof File ||\n    (isBinaryValue(input) &&\n      input.byteLength > UINT8ARRAY_CONTENT_HASH_THRESHOLD)\n  )\n}\n\nfunction isBinaryValue(input: object): input is Uint8Array {\n  return (\n    (typeof Buffer !== `undefined` && input instanceof Buffer) ||\n    input instanceof Uint8Array\n  )\n}\n\nfunction structuralShape(\n  input: object,\n): [keyof typeof STRUCTURAL_MARKERS, object] {\n  if (input instanceof Map) return [`map`, [...input.entries()]]\n  if (input instanceof Set) return [`set`, [...input.values()]]\n  return [input instanceof Array ? `array` : `object`, input]\n}\n\n/** @internal Compare immutable structural values without computing a digest.\n * Pair memoization also permits cyclic values that structural hashing rejects.\n * Reference-valued leaves remain opaque, including registered mutable handles.\n */\nexport function equalHashValues(left: unknown, right: unknown): boolean {\n  const compared = new Map<object, Set<object>>()\n  function equal(a: unknown, b: unknown): boolean {\n    if (a === b || (Number.isNaN(a) && Number.isNaN(b))) return true\n    if (\n      a === null ||\n      b === null ||\n      typeof a !== `object` ||\n      typeof b !== `object`\n    )\n      return false\n    if (\n      referenceValues.has(a) ||\n      referenceValues.has(b) ||\n      isReferenceHashedObject(a) ||\n      isReferenceHashedObject(b)\n    )\n      return false\n    if (a instanceof Date || b instanceof Date)\n      return (\n        a instanceof Date &&\n        b instanceof Date &&\n        equal(a.getTime(), b.getTime())\n      )\n    if (isBinaryValue(a) || isBinaryValue(b))\n      return (\n        isBinaryValue(a) &&\n        isBinaryValue(b) &&\n        a.byteLength === b.byteLength &&\n        a.every((value, index) => value === b[index])\n      )\n    if (isTemporal(a) || isTemporal(b))\n      return (\n        isTemporal(a) &&\n        isTemporal(b) &&\n        a[Symbol.toStringTag] === b[Symbol.toStringTag] &&\n        a.toString() === b.toString()\n      )\n\n    // Revisited pairs close cycles and avoid expanding shared subtrees.\n    const peers = compared.get(a)\n    if (peers?.has(b)) return true\n    if (peers) peers.add(b)\n    else compared.set(a, new Set([b]))\n    const [aKind, aShape] = structuralShape(a)\n    const [bKind, bShape] = structuralShape(b)\n    if (aKind !== bKind) return false\n    const keys = (value: object) =>\n      Reflect.ownKeys(value).filter((key) =>\n        Object.prototype.propertyIsEnumerable.call(value, key),\n      )\n    const aKeys = keys(aShape)\n    if (aKeys.length !== keys(bShape).length) return false\n    return aKeys.every(\n      (key) =>\n        Object.prototype.propertyIsEnumerable.call(bShape, key) &&\n        equal(aShape[key as keyof object], bShape[key as keyof object]),\n    )\n  }\n  return equal(left, right)\n}\n\nlet nextRefId = 1\nfunction cachedReferenceHash(fn: object): number {\n  let valueHash = hashCache.get(fn)\n  if (valueHash === undefined) {\n    valueHash = nextRefId ^ FUNCTIONS\n    nextRefId++\n    hashCache.set(fn, valueHash)\n    referenceValues.add(fn)\n  }\n  return valueHash\n}\n\n/**\n * Strings sorted lexicographically.\n */\nfunction keySort(a: string, b: string): number {\n  return a.localeCompare(b)\n}\n"],"names":["valueHash"],"mappings":";AAQA,MAAM,OAAO,WAAA;AACb,MAAM,QAAQ,WAAA;AACd,MAAM,OAAO,WAAA;AACb,MAAM,YAAY,WAAA;AAClB,MAAM,MAAM,WAAA;AACZ,MAAM,YAAY,WAAA;AAClB,MAAM,cAAc,WAAA;AACpB,MAAM,qBAAqB;AAAA,EACzB,QAAQ,WAAA;AAAA,EACR,OAAO,WAAA;AAAA,EACP,KAAK,WAAA;AAAA,EACL,KAAK,WAAA;AACP;AACA,MAAM,oBAAoB,WAAA;AAC1B,MAAM,kBAAkB,WAAA;AAGxB,MAAM,2BAA2B;AACjC,MAAM,4BAA4B;AAElC,MAAM,oCAAoB,IAAI;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAOD,SAAS,WAAW,OAAsC;AACxD,QAAM,MAAO,MAAkC,OAAO,WAAW;AACjE,SAAO,OAAO,QAAQ,YAAY,cAAc,IAAI,GAAG;AACzD;AAKA,MAAM,oCAAoC;AAE1C,MAAM,gCAAgB,QAAA;AACtB,MAAM,sCAAsB,QAAA;AAGrB,SAAS,mBAAmB,OAAqB;AACtD,sBAAoB,KAAK;AAC3B;AAQO,SAAS,KAAK,OAAoB;AACvC,QAAM,SAAS,IAAI,iBAAA;AACnB,eAAa,QAAQ,KAAK;AAC1B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,WAAW,OAAe,SAA8B;AAC/D,MAAI,QAAQ,cAAc,QAAQ,2BAA2B;AAC3D,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AAEA,UAAQ,cAAc,IAAI,KAAK;AAE/B,MAAI;AACJ,MAAI;AACF,QAAI,iBAAiB,MAAM;AACzB,kBAAY,SAAS,KAAK;AAAA,IAC5B,WAAW,cAAc,KAAK,GAAG;AAC/B,kBAAY,eAAe,KAAK;AAAA,IAClC,WAAW,WAAW,KAAK,GAAG;AAC5B,kBAAY,aAAa,KAAK;AAAA,IAChC,OAAO;AACL,YAAM,CAAC,MAAM,gBAAgB,IAAI,gBAAgB,KAAK;AACtD,kBAAY;AAAA,QACV;AAAA,QACA,mBAAmB,IAAI;AAAA,QACvB;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF,UAAA;AACE,YAAQ,cAAc,OAAO,KAAK;AAAA,EACpC;AAEA,UAAQ,cAAc,IAAI,OAAO,SAAS;AAC1C,SAAO;AACT;AAEA,SAAS,SAAS,OAAqB;AACrC,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,WAAW;AACzB,SAAO,OAAO,MAAM,SAAS;AAC7B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,eAAe,OAA2B;AACjD,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,iBAAiB;AAE/B,SAAO,OAAO,MAAM,UAAU;AAE9B,WAAS,IAAI,GAAG,IAAI,MAAM,YAAY,KAAK;AACzC,WAAO,UAAU,MAAM,CAAC,CAAE;AAAA,EAC5B;AACA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aAAa,OAA6B;AACjD,QAAM,SAAS,IAAI,iBAAA;AACnB,SAAO,OAAO,eAAe;AAC7B,SAAO,OAAO,MAAM,OAAO,WAAW,CAAC;AACvC,SAAO,OAAO,MAAM,UAAU;AAC9B,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,gBACP,OACA,QACA,SACQ;AACR,QAAM,SAAS,IAAI,iBAAA;AAGnB,SAAO,OAAO,MAAM;AACpB,QAAM,OAAO,OAAO,KAAK,KAAK;AAC9B,OAAK,KAAK,OAAO;AACjB,aAAW,OAAO,MAAM;AACtB,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,GAAG,OAAO;AAAA,EAChE;AACA,QAAM,aAAa,OAAO,sBAAsB,KAAK,EAClD,OAAO,CAAC,QAAQ,OAAO,UAAU,qBAAqB,KAAK,OAAO,GAAG,CAAC,EACtE,KAAK,CAAC,MAAM,UAAU,kBAAkB,IAAI,IAAI,kBAAkB,KAAK,CAAC;AAC3E,aAAW,OAAO,YAAY;AAC5B,WAAO,OAAO,GAAG;AACjB,WAAO,OAAO,GAAG;AACjB,iBAAa,QAAQ,MAAM,GAAyB,GAAG,OAAO;AAAA,EAChE;AAEA,SAAO,OAAO,OAAA;AAChB;AAEA,SAAS,aACP,QACA,OACA,SACM;AACN,MAAI,WAAW,EAAE,QAAQ,OAAO,0BAA0B;AACxD,UAAM,IAAI,WAAW,sDAAsD;AAAA,EAC7E;AACA,MAAI,UAAU,MAAM;AAClB,WAAO,OAAO,IAAI;AAClB;AAAA,EACF;AACA,UAAQ,OAAO,OAAA;AAAA,IACb,KAAK;AACH,aAAO,OAAO,SAAS;AACvB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,QAAQ,OAAO,KAAK;AAClC;AAAA,IACF,KAAK;AAEH,aAAO,OAAO,MAAM,KAAK,IAAI,MAAM,UAAU,IAAI,IAAI,KAAK;AAC1D;AAAA,IACF,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,aAAO,OAAO,KAAK;AACnB;AAAA,IACF,KAAK;AACH,aAAO,OAAO,cAAc,OAAO,OAAO,CAAC;AAC3C;AAAA,IACF,KAAK;AAGH,aAAO,OAAO,oBAAoB,KAAK,CAAC;AACxC;AAAA,IACF;AACE,cAAQ;AAAA,QACN,sDAAsD,OAAO,KAAK;AAAA,MAAA;AAAA,EACpE;AAEN;AAEA,SAAS,cAAc,OAAe,SAA+B;AACnE,MAAI,CAAC,SAAS;AACZ,UAAM,SAAS,UAAU,IAAI,KAAK;AAClC,QAAI,WAAW,OAAW,QAAO;AACjC,QAAI,wBAAwB,KAAK,EAAG,QAAO,oBAAoB,KAAK;AAIpE,cAAU;AAAA,MACR,mCAAmB,IAAA;AAAA,MACnB,MAAM;AAAA,MACN,mCAAmB,IAAA;AAAA,IAAI;AAEzB,UAAM,SAAS,WAAW,OAAO,OAAO;AACxC,eAAW,CAAC,QAAQA,UAAS,KAAK,QAAQ,eAAe;AACvD,gBAAU,IAAI,QAAQA,UAAS;AAAA,IACjC;AACA,WAAO;AAAA,EACT;AAEA,MAAI,QAAQ,cAAc,IAAI,KAAK,GAAG;AACpC,UAAM,IAAI,UAAU,sCAAsC;AAAA,EAC5D;AAIA,MAAI,wBAAwB,KAAK,EAAG,QAAO,oBAAoB,KAAK;AAEpE,QAAM,YAAY,UAAU,IAAI,KAAK,KAAK,QAAQ,cAAc,IAAI,KAAK;AACzE,MAAI,cAAc,OAAW,QAAO;AAEpC,SAAO,WAAW,OAAO,OAAO;AAClC;AAEA,SAAS,wBAAwB,OAAwB;AACvD,SACE,iBAAiB,QAChB,cAAc,KAAK,KAClB,MAAM,aAAa;AAEzB;AAEA,SAAS,cAAc,OAAoC;AACzD,SACG,OAAO,WAAW,eAAe,iBAAiB,UACnD,iBAAiB;AAErB;AAEA,SAAS,gBACP,OAC2C;AAC3C,MAAI,iBAAiB,IAAK,QAAO,CAAC,OAAO,CAAC,GAAG,MAAM,QAAA,CAAS,CAAC;AAC7D,MAAI,iBAAiB,IAAK,QAAO,CAAC,OAAO,CAAC,GAAG,MAAM,OAAA,CAAQ,CAAC;AAC5D,SAAO,CAAC,iBAAiB,QAAQ,UAAU,UAAU,KAAK;AAC5D;AAMO,SAAS,gBAAgB,MAAe,OAAyB;AACtE,QAAM,+BAAe,IAAA;AACrB,WAAS,MAAM,GAAY,GAAqB;AAC9C,QAAI,MAAM,KAAM,OAAO,MAAM,CAAC,KAAK,OAAO,MAAM,CAAC,EAAI,QAAO;AAC5D,QACE,MAAM,QACN,MAAM,QACN,OAAO,MAAM,YACb,OAAO,MAAM;AAEb,aAAO;AACT,QACE,gBAAgB,IAAI,CAAC,KACrB,gBAAgB,IAAI,CAAC,KACrB,wBAAwB,CAAC,KACzB,wBAAwB,CAAC;AAEzB,aAAO;AACT,QAAI,aAAa,QAAQ,aAAa;AACpC,aACE,aAAa,QACb,aAAa,QACb,MAAM,EAAE,QAAA,GAAW,EAAE,SAAS;AAElC,QAAI,cAAc,CAAC,KAAK,cAAc,CAAC;AACrC,aACE,cAAc,CAAC,KACf,cAAc,CAAC,KACf,EAAE,eAAe,EAAE,cACnB,EAAE,MAAM,CAAC,OAAO,UAAU,UAAU,EAAE,KAAK,CAAC;AAEhD,QAAI,WAAW,CAAC,KAAK,WAAW,CAAC;AAC/B,aACE,WAAW,CAAC,KACZ,WAAW,CAAC,KACZ,EAAE,OAAO,WAAW,MAAM,EAAE,OAAO,WAAW,KAC9C,EAAE,SAAA,MAAe,EAAE,SAAA;AAIvB,UAAM,QAAQ,SAAS,IAAI,CAAC;AAC5B,QAAI,OAAO,IAAI,CAAC,EAAG,QAAO;AAC1B,QAAI,MAAO,OAAM,IAAI,CAAC;AAAA,QACjB,UAAS,IAAI,GAAG,oBAAI,IAAI,CAAC,CAAC,CAAC,CAAC;AACjC,UAAM,CAAC,OAAO,MAAM,IAAI,gBAAgB,CAAC;AACzC,UAAM,CAAC,OAAO,MAAM,IAAI,gBAAgB,CAAC;AACzC,QAAI,UAAU,MAAO,QAAO;AAC5B,UAAM,OAAO,CAAC,UACZ,QAAQ,QAAQ,KAAK,EAAE;AAAA,MAAO,CAAC,QAC7B,OAAO,UAAU,qBAAqB,KAAK,OAAO,GAAG;AAAA,IAAA;AAEzD,UAAM,QAAQ,KAAK,MAAM;AACzB,QAAI,MAAM,WAAW,KAAK,MAAM,EAAE,OAAQ,QAAO;AACjD,WAAO,MAAM;AAAA,MACX,CAAC,QACC,OAAO,UAAU,qBAAqB,KAAK,QAAQ,GAAG,KACtD,MAAM,OAAO,GAAmB,GAAG,OAAO,GAAmB,CAAC;AAAA,IAAA;AAAA,EAEpE;AACA,SAAO,MAAM,MAAM,KAAK;AAC1B;AAEA,IAAI,YAAY;AAChB,SAAS,oBAAoB,IAAoB;AAC/C,MAAI,YAAY,UAAU,IAAI,EAAE;AAChC,MAAI,cAAc,QAAW;AAC3B,gBAAY,YAAY;AACxB;AACA,cAAU,IAAI,IAAI,SAAS;AAC3B,oBAAgB,IAAI,EAAE;AAAA,EACxB;AACA,SAAO;AACT;AAKA,SAAS,QAAQ,GAAW,GAAmB;AAC7C,SAAO,EAAE,cAAc,CAAC;AAC1B;"}

package/dist/esm/operators/distinct.d.ts

@@ -12,5 +12,5 @@ export declare class DistinctOperator<T extends KeyValue<any, any>> extends Unar
 /**
  * Removes duplicate values
  */
-export declare function distinct<T extends KeyValue<any, any>>(by?: (value: T) => any): (stream: IStreamBuilder<T>) => IStreamBuilder<T>;
+export declare function distinct<T extends KeyValue<any, any>>(by?: (value: T) => any): (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<number, GetValue<T>>>;
 export {};

package/dist/esm/operators/distinct.js.map

@@ -1 +1 @@
-{"version":3,"file":"distinct.js","sources":["../../../src/operators/distinct.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { hash } from '../hashing/index.js'\nimport { MultiSet } from '../multiset.js'\nimport type { Hash } from '../hashing/index.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, KeyValue } from '../types.js'\n\ntype Multiplicity = number\n\ntype GetValue<T> = T extends KeyValue<any, infer V> ? V : never\n\n/**\n * Operator that removes duplicates\n */\nexport class DistinctOperator<\n  T extends KeyValue<any, any>,\n> extends UnaryOperator<T, KeyValue<number, GetValue<T>>> {\n  #by: (value: T) => any\n  #values: Map<Hash, Multiplicity> // keeps track of the number of times each value has been seen\n\n  constructor(\n    id: number,\n    input: DifferenceStreamReader<T>,\n    output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,\n    by: (value: T) => any = (value: T) => value,\n  ) {\n    super(id, input, output)\n    this.#by = by\n    this.#values = new Map()\n  }\n\n  run(): void {\n    const updatedValues = new Map<Hash, [Multiplicity, T]>()\n\n    // Compute the new multiplicity for each value\n    for (const message of this.inputMessages()) {\n      for (const [value, diff] of message.getInner()) {\n        const hashedValue = hash(this.#by(value))\n\n        const oldMultiplicity =\n          updatedValues.get(hashedValue)?.[0] ??\n          this.#values.get(hashedValue) ??\n          0\n        const newMultiplicity = oldMultiplicity + diff\n        updatedValues.set(hashedValue, [newMultiplicity, value])\n      }\n    }\n\n    const result: Array<[KeyValue<number, GetValue<T>>, number]> = []\n\n    // Check which values became visible or disappeared\n    for (const [\n      hashedValue,\n      [newMultiplicity, value],\n    ] of updatedValues.entries()) {\n      const oldMultiplicity = this.#values.get(hashedValue) ?? 0\n\n      if (newMultiplicity === 0) {\n        this.#values.delete(hashedValue)\n      } else {\n        this.#values.set(hashedValue, newMultiplicity)\n      }\n\n      if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n        // The value wasn't present in the stream\n        // but with this change it is now present in the stream\n        result.push([[hash(this.#by(value)), value[1]], 1])\n      } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n        // The value was present in the stream\n        // but with this change it is no longer present in the stream\n        result.push([[hash(this.#by(value)), value[1]], -1])\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n}\n\n/**\n * Removes duplicate values\n */\nexport function distinct<T extends KeyValue<any, any>>(\n  by: (value: T) => any = (value: T) => value,\n) {\n  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n    const output = new StreamBuilder<T>(\n      stream.graph,\n      new DifferenceStreamWriter<T>(),\n    )\n    const operator = new DistinctOperator<T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      by,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;AAeO,MAAM,yBAEH,cAAgD;AAAA,EACxD;AAAA,EACA;AAAA;AAAA,EAEA,YACE,IACA,OACA,QACA,KAAwB,CAAC,UAAa,OACtC;AACA,UAAM,IAAI,OAAO,MAAM;AACvB,SAAK,MAAM;AACX,SAAK,8BAAc,IAAA;AAAA,EACrB;AAAA,EAEA,MAAY;AACV,UAAM,oCAAoB,IAAA;AAG1B,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,OAAO,IAAI,KAAK,QAAQ,YAAY;AAC9C,cAAM,cAAc,KAAK,KAAK,IAAI,KAAK,CAAC;AAExC,cAAM,kBACJ,cAAc,IAAI,WAAW,IAAI,CAAC,KAClC,KAAK,QAAQ,IAAI,WAAW,KAC5B;AACF,cAAM,kBAAkB,kBAAkB;AAC1C,sBAAc,IAAI,aAAa,CAAC,iBAAiB,KAAK,CAAC;AAAA,MACzD;AAAA,IACF;AAEA,UAAM,SAAyD,CAAA;AAG/D,eAAW;AAAA,MACT;AAAA,MACA,CAAC,iBAAiB,KAAK;AAAA,IAAA,KACpB,cAAc,WAAW;AAC5B,YAAM,kBAAkB,KAAK,QAAQ,IAAI,WAAW,KAAK;AAEzD,UAAI,oBAAoB,GAAG;AACzB,aAAK,QAAQ,OAAO,WAAW;AAAA,MACjC,OAAO;AACL,aAAK,QAAQ,IAAI,aAAa,eAAe;AAAA,MAC/C;AAEA,UAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAG/C,eAAO,KAAK,CAAC,CAAC,KAAK,KAAK,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAAA,MACpD,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAGtD,eAAO,KAAK,CAAC,CAAC,KAAK,KAAK,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;AAAA,MACrD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAKO,SAAS,SACd,KAAwB,CAAC,UAAa,OACtC;AACA,SAAO,CAAC,WAAiD;AACvD,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA0B;AAEhC,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"distinct.js","sources":["../../../src/operators/distinct.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { hash } from '../hashing/index.js'\nimport { MultiSet } from '../multiset.js'\nimport type { Hash } from '../hashing/index.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, KeyValue } from '../types.js'\n\ntype Multiplicity = number\n\ntype GetValue<T> = T extends KeyValue<any, infer V> ? V : never\n\n/**\n * Operator that removes duplicates\n */\nexport class DistinctOperator<\n  T extends KeyValue<any, any>,\n> extends UnaryOperator<T, KeyValue<number, GetValue<T>>> {\n  #by: (value: T) => any\n  #values: Map<Hash, Multiplicity> // keeps track of the number of times each value has been seen\n\n  constructor(\n    id: number,\n    input: DifferenceStreamReader<T>,\n    output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,\n    by: (value: T) => any = (value: T) => value,\n  ) {\n    super(id, input, output)\n    this.#by = by\n    this.#values = new Map()\n  }\n\n  run(): void {\n    const updatedValues = new Map<Hash, [Multiplicity, T]>()\n\n    // Compute the new multiplicity for each value\n    for (const message of this.inputMessages()) {\n      for (const [value, diff] of message.getInner()) {\n        const hashedValue = hash(this.#by(value))\n\n        const oldMultiplicity =\n          updatedValues.get(hashedValue)?.[0] ??\n          this.#values.get(hashedValue) ??\n          0\n        const newMultiplicity = oldMultiplicity + diff\n        updatedValues.set(hashedValue, [newMultiplicity, value])\n      }\n    }\n\n    const result: Array<[KeyValue<number, GetValue<T>>, number]> = []\n\n    // Check which values became visible or disappeared\n    for (const [\n      hashedValue,\n      [newMultiplicity, value],\n    ] of updatedValues.entries()) {\n      const oldMultiplicity = this.#values.get(hashedValue) ?? 0\n\n      if (newMultiplicity === 0) {\n        this.#values.delete(hashedValue)\n      } else {\n        this.#values.set(hashedValue, newMultiplicity)\n      }\n\n      if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n        // The value wasn't present in the stream\n        // but with this change it is now present in the stream\n        result.push([[hash(this.#by(value)), value[1]], 1])\n      } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n        // The value was present in the stream\n        // but with this change it is no longer present in the stream\n        result.push([[hash(this.#by(value)), value[1]], -1])\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n}\n\n/**\n * Removes duplicate values\n */\nexport function distinct<T extends KeyValue<any, any>>(\n  by: (value: T) => any = (value: T) => value,\n) {\n  return (\n    stream: IStreamBuilder<T>,\n  ): IStreamBuilder<KeyValue<number, GetValue<T>>> => {\n    const output = new StreamBuilder<KeyValue<number, GetValue<T>>>(\n      stream.graph,\n      new DifferenceStreamWriter<KeyValue<number, GetValue<T>>>(),\n    )\n    const operator = new DistinctOperator<T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      by,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;AAeO,MAAM,yBAEH,cAAgD;AAAA,EACxD;AAAA,EACA;AAAA;AAAA,EAEA,YACE,IACA,OACA,QACA,KAAwB,CAAC,UAAa,OACtC;AACA,UAAM,IAAI,OAAO,MAAM;AACvB,SAAK,MAAM;AACX,SAAK,8BAAc,IAAA;AAAA,EACrB;AAAA,EAEA,MAAY;AACV,UAAM,oCAAoB,IAAA;AAG1B,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,OAAO,IAAI,KAAK,QAAQ,YAAY;AAC9C,cAAM,cAAc,KAAK,KAAK,IAAI,KAAK,CAAC;AAExC,cAAM,kBACJ,cAAc,IAAI,WAAW,IAAI,CAAC,KAClC,KAAK,QAAQ,IAAI,WAAW,KAC5B;AACF,cAAM,kBAAkB,kBAAkB;AAC1C,sBAAc,IAAI,aAAa,CAAC,iBAAiB,KAAK,CAAC;AAAA,MACzD;AAAA,IACF;AAEA,UAAM,SAAyD,CAAA;AAG/D,eAAW;AAAA,MACT;AAAA,MACA,CAAC,iBAAiB,KAAK;AAAA,IAAA,KACpB,cAAc,WAAW;AAC5B,YAAM,kBAAkB,KAAK,QAAQ,IAAI,WAAW,KAAK;AAEzD,UAAI,oBAAoB,GAAG;AACzB,aAAK,QAAQ,OAAO,WAAW;AAAA,MACjC,OAAO;AACL,aAAK,QAAQ,IAAI,aAAa,eAAe;AAAA,MAC/C;AAEA,UAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAG/C,eAAO,KAAK,CAAC,CAAC,KAAK,KAAK,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;AAAA,MACpD,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAGtD,eAAO,KAAK,CAAC,CAAC,KAAK,KAAK,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;AAAA,MACrD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AACF;AAKO,SAAS,SACd,KAAwB,CAAC,UAAa,OACtC;AACA,SAAO,CACL,WACkD;AAClD,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAAsD;AAE5D,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}

package/dist/esm/operators/groupedTopKWithFractionalIndex.js

@@ -1,7 +1,7 @@
 import { UnaryOperator, DifferenceStreamWriter } from "../graph.js";
 import { StreamBuilder } from "../d2.js";
 import { MultiSet } from "../multiset.js";
-import { TopKState, handleMoveIn, handleMoveOut } from "./topKState.js";
+import { TopKState, handleMoveIn, handleMoveOut, topKBatch } from "./topKState.js";
 import { createKeyedComparator, TopKArray } from "./topKArray.js";
 class GroupedTopKWithFractionalIndexOperator extends UnaryOperator {
   #groupStates = /* @__PURE__ */ new Map();
@@ -73,11 +73,10 @@ class GroupedTopKWithFractionalIndexOperator extends UnaryOperator {
   }
   run() {
     const result = [];
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item;
-        this.#processElement(key, value, multiplicity, result);
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages()
+    )) {
+      this.#processElement(key, value, multiplicity, result);
     }
     if (result.length > 0) {
       this.output.sendData(new MultiSet(result));

package/dist/esm/operators/groupedTopKWithFractionalIndex.js.map

@@ -1 +1 @@
-{"version":3,"file":"groupedTopKWithFractionalIndex.js","sources":["../../../src/operators/groupedTopKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue, TopK } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface GroupedTopKWithFractionalIndexOptions<K, T> {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n  /**\n   * Function to extract a group key from the element's key and value.\n   * Elements with the same group key will be sorted and limited together.\n   */\n  groupKeyFn: (key: K, value: T) => unknown\n}\n\n/**\n * Operator for grouped fractional indexed topK operations.\n * This operator maintains separate topK windows for each group,\n * allowing per-group limits and ordering.\n *\n * The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].\n * Elements are grouped by the groupKeyFn, and each group maintains\n * its own sorted collection with independent limit/offset.\n */\nexport class GroupedTopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #groupStates: Map<unknown, TopKState<K, T>> = new Map()\n  #groupKeyFn: (key: K, value: T) => unknown\n  #comparator: (a: [K, T], b: [K, T]) => number\n  #offset: number\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: GroupedTopKWithFractionalIndexOptions<K, T>,\n  ) {\n    super(id, inputA, output)\n    this.#groupKeyFn = options.groupKeyFn\n    this.#limit = options.limit ?? Infinity\n    this.#offset = options.offset ?? 0\n    this.#comparator = createKeyedComparator(comparator)\n    options.setSizeCallback?.(() => this.#getTotalSize())\n    options.setWindowFn?.(this.#moveTopK.bind(this))\n  }\n\n  /**\n   * Creates a new TopK data structure for a group.\n   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).\n   */\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: [K, T], b: [K, T]) => number,\n  ): TopK<[K, T]> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  #getTotalSize(): number {\n    let size = 0\n    for (const state of this.#groupStates.values()) {\n      size += state.size\n    }\n    return size\n  }\n\n  #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {\n    let state = this.#groupStates.get(groupKey)\n    if (!state) {\n      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)\n      state = new TopKState(topK)\n      this.#groupStates.set(groupKey, state)\n    }\n    return state\n  }\n\n  #cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {\n    if (state.isEmpty) {\n      this.#groupStates.delete(groupKey)\n    }\n  }\n\n  /**\n   * Moves the topK window for all groups based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {\n    if (offset !== undefined) {\n      this.#offset = offset\n    }\n    if (limit !== undefined) {\n      this.#limit = limit\n    }\n\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    let hasChanges = false\n\n    for (const state of this.#groupStates.values()) {\n      const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit\n\n      diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n      diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n      if (diff.changes) {\n        hasChanges = true\n      }\n    }\n\n    if (hasChanges) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.#processElement(key, value, multiplicity, result)\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  #processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const groupKey = this.#groupKeyFn(key, value)\n    const state = this.#getOrCreateGroupState(groupKey)\n\n    const changes = state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n\n    // Cleanup empty groups to prevent memory leaks\n    this.#cleanupGroupIfEmpty(groupKey, state)\n  }\n}\n\n/**\n * Limits the number of results per group based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * This operator groups elements by the provided groupKeyFn and applies the limit/offset\n * independently to each group.\n *\n * @param comparator - A function that compares two elements for ordering\n * @param options - Configuration including groupKeyFn, limit, and offset\n * @returns A piped operator that orders elements per group and limits results per group\n */\nexport function groupedTopKWithFractionalIndex<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options: GroupedTopKWithFractionalIndexOptions<K, T>,\n): PipedOperator<[K, T], [K, IndexedValue<T>]> {\n  return (\n    stream: IStreamBuilder<[K, T]>,\n  ): IStreamBuilder<[K, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[K, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[K, IndexedValue<T>]>(),\n    )\n    const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      options,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;AAgCO,MAAM,+CAGH,cAA4C;AAAA,EACpD,mCAAkD,IAAA;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,cAAc,QAAQ;AAC3B,SAAK,SAAS,QAAQ,SAAS;AAC/B,SAAK,UAAU,QAAQ,UAAU;AACjC,SAAK,cAAc,sBAAsB,UAAU;AACnD,YAAQ,kBAAkB,MAAM,KAAK,cAAA,CAAe;AACpD,YAAQ,cAAc,KAAK,UAAU,KAAK,IAAI,CAAC;AAAA,EACjD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,WACR,QACA,OACA,YACc;AACd,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,gBAAwB;AACtB,QAAI,OAAO;AACX,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,cAAQ,MAAM;AAAA,IAChB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,UAAoC;AACzD,QAAI,QAAQ,KAAK,aAAa,IAAI,QAAQ;AAC1C,QAAI,CAAC,OAAO;AACV,YAAM,OAAO,KAAK,WAAW,KAAK,SAAS,KAAK,QAAQ,KAAK,WAAW;AACxE,cAAQ,IAAI,UAAU,IAAI;AAC1B,WAAK,aAAa,IAAI,UAAU,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,qBAAqB,UAAmB,OAA8B;AACpE,QAAI,MAAM,SAAS;AACjB,WAAK,aAAa,OAAO,QAAQ;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU,EAAE,QAAQ,SAAoD;AACtE,QAAI,WAAW,QAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,UAAU,QAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,SAAgD,CAAA;AACtD,QAAI,aAAa;AAEjB,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,YAAM,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,SAAS,OAAO,KAAK,QAAQ;AAEpE,WAAK,QAAQ,QAAQ,CAAC,WAAW,aAAa,QAAQ,MAAM,CAAC;AAC7D,WAAK,SAAS,QAAQ,CAAC,YAAY,cAAc,SAAS,MAAM,CAAC;AAEjE,UAAI,KAAK,SAAS;AAChB,qBAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,YAAY;AACd,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,aAAK,gBAAgB,KAAK,OAAO,cAAc,MAAM;AAAA,MACvD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,gBACE,KACA,OACA,cACA,QACM;AACN,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK;AAC5C,UAAM,QAAQ,KAAK,uBAAuB,QAAQ;AAElD,UAAM,UAAU,MAAM,eAAe,KAAK,OAAO,YAAY;AAC7D,iBAAa,QAAQ,QAAQ,MAAM;AACnC,kBAAc,QAAQ,SAAS,MAAM;AAGrC,SAAK,qBAAqB,UAAU,KAAK;AAAA,EAC3C;AACF;AAeO,SAAS,+BACd,YACA,SAC6C;AAC7C,SAAO,CACL,WACyC;AACzC,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA6C;AAEnD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"groupedTopKWithFractionalIndex.js","sources":["../../../src/operators/groupedTopKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport {\n  TopKState,\n  handleMoveIn,\n  handleMoveOut,\n  topKBatch,\n} from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue, TopK } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface GroupedTopKWithFractionalIndexOptions<K, T> {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n  /**\n   * Function to extract a group key from the element's key and value.\n   * Elements with the same group key will be sorted and limited together.\n   */\n  groupKeyFn: (key: K, value: T) => unknown\n}\n\n/**\n * Operator for grouped fractional indexed topK operations.\n * This operator maintains separate topK windows for each group,\n * allowing per-group limits and ordering.\n *\n * The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].\n * Elements are grouped by the groupKeyFn, and each group maintains\n * its own sorted collection with independent limit/offset.\n */\nexport class GroupedTopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #groupStates: Map<unknown, TopKState<K, T>> = new Map()\n  #groupKeyFn: (key: K, value: T) => unknown\n  #comparator: (a: [K, T], b: [K, T]) => number\n  #offset: number\n  #limit: number\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: GroupedTopKWithFractionalIndexOptions<K, T>,\n  ) {\n    super(id, inputA, output)\n    this.#groupKeyFn = options.groupKeyFn\n    this.#limit = options.limit ?? Infinity\n    this.#offset = options.offset ?? 0\n    this.#comparator = createKeyedComparator(comparator)\n    options.setSizeCallback?.(() => this.#getTotalSize())\n    options.setWindowFn?.(this.#moveTopK.bind(this))\n  }\n\n  /**\n   * Creates a new TopK data structure for a group.\n   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).\n   */\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: [K, T], b: [K, T]) => number,\n  ): TopK<[K, T]> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  #getTotalSize(): number {\n    let size = 0\n    for (const state of this.#groupStates.values()) {\n      size += state.size\n    }\n    return size\n  }\n\n  #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {\n    let state = this.#groupStates.get(groupKey)\n    if (!state) {\n      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)\n      state = new TopKState(topK)\n      this.#groupStates.set(groupKey, state)\n    }\n    return state\n  }\n\n  #cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {\n    if (state.isEmpty) {\n      this.#groupStates.delete(groupKey)\n    }\n  }\n\n  /**\n   * Moves the topK window for all groups based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {\n    if (offset !== undefined) {\n      this.#offset = offset\n    }\n    if (limit !== undefined) {\n      this.#limit = limit\n    }\n\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    let hasChanges = false\n\n    for (const state of this.#groupStates.values()) {\n      const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit\n\n      diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n      diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n      if (diff.changes) {\n        hasChanges = true\n      }\n    }\n\n    if (hasChanges) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const [[key, value], multiplicity] of topKBatch(\n      this.inputMessages(),\n    )) {\n      this.#processElement(key, value, multiplicity, result)\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  #processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const groupKey = this.#groupKeyFn(key, value)\n    const state = this.#getOrCreateGroupState(groupKey)\n\n    const changes = state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n\n    // Cleanup empty groups to prevent memory leaks\n    this.#cleanupGroupIfEmpty(groupKey, state)\n  }\n}\n\n/**\n * Limits the number of results per group based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * This operator groups elements by the provided groupKeyFn and applies the limit/offset\n * independently to each group.\n *\n * @param comparator - A function that compares two elements for ordering\n * @param options - Configuration including groupKeyFn, limit, and offset\n * @returns A piped operator that orders elements per group and limits results per group\n */\nexport function groupedTopKWithFractionalIndex<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options: GroupedTopKWithFractionalIndexOptions<K, T>,\n): PipedOperator<[K, T], [K, IndexedValue<T>]> {\n  return (\n    stream: IStreamBuilder<[K, T]>,\n  ): IStreamBuilder<[K, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[K, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[K, IndexedValue<T>]>(),\n    )\n    const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      options,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;AAqCO,MAAM,+CAGH,cAA4C;AAAA,EACpD,mCAAkD,IAAA;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,cAAc,QAAQ;AAC3B,SAAK,SAAS,QAAQ,SAAS;AAC/B,SAAK,UAAU,QAAQ,UAAU;AACjC,SAAK,cAAc,sBAAsB,UAAU;AACnD,YAAQ,kBAAkB,MAAM,KAAK,cAAA,CAAe;AACpD,YAAQ,cAAc,KAAK,UAAU,KAAK,IAAI,CAAC;AAAA,EACjD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,WACR,QACA,OACA,YACc;AACd,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,gBAAwB;AACtB,QAAI,OAAO;AACX,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,cAAQ,MAAM;AAAA,IAChB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,UAAoC;AACzD,QAAI,QAAQ,KAAK,aAAa,IAAI,QAAQ;AAC1C,QAAI,CAAC,OAAO;AACV,YAAM,OAAO,KAAK,WAAW,KAAK,SAAS,KAAK,QAAQ,KAAK,WAAW;AACxE,cAAQ,IAAI,UAAU,IAAI;AAC1B,WAAK,aAAa,IAAI,UAAU,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,qBAAqB,UAAmB,OAA8B;AACpE,QAAI,MAAM,SAAS;AACjB,WAAK,aAAa,OAAO,QAAQ;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAU,EAAE,QAAQ,SAAoD;AACtE,QAAI,WAAW,QAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,UAAU,QAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AAEA,UAAM,SAAgD,CAAA;AACtD,QAAI,aAAa;AAEjB,eAAW,SAAS,KAAK,aAAa,OAAA,GAAU;AAC9C,YAAM,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,SAAS,OAAO,KAAK,QAAQ;AAEpE,WAAK,QAAQ,QAAQ,CAAC,WAAW,aAAa,QAAQ,MAAM,CAAC;AAC7D,WAAK,SAAS,QAAQ,CAAC,YAAY,cAAc,SAAS,MAAM,CAAC;AAEjE,UAAI,KAAK,SAAS;AAChB,qBAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,YAAY;AACd,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,KAAK;AAAA,MACzC,KAAK,cAAA;AAAA,IAAc,GAClB;AACD,WAAK,gBAAgB,KAAK,OAAO,cAAc,MAAM;AAAA,IACvD;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,gBACE,KACA,OACA,cACA,QACM;AACN,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK;AAC5C,UAAM,QAAQ,KAAK,uBAAuB,QAAQ;AAElD,UAAM,UAAU,MAAM,eAAe,KAAK,OAAO,YAAY;AAC7D,iBAAa,QAAQ,QAAQ,MAAM;AACnC,kBAAc,QAAQ,SAAS,MAAM;AAGrC,SAAK,qBAAqB,UAAU,KAAK;AAAA,EAC3C;AACF;AAeO,SAAS,+BACd,YACA,SAC6C;AAC7C,SAAO,CACL,WACyC;AACzC,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAA6C;AAEnD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}

package/dist/esm/operators/topKArray.js

@@ -78,6 +78,7 @@ class TopKArray {
     const fractionalIndex = generateKeyBetween(indexBefore, indexAfter);
     const val = indexedValue(value, fractionalIndex);
     this.#sortedValues.splice(index, 0, val);
+    if (this.#topKStart === this.#topKEnd) return result;
     if (index < this.#topKEnd) {
       const moveInIndex = Math.max(index, this.#topKStart);
       if (moveInIndex < this.#sortedValues.length) {
@@ -99,6 +100,7 @@ class TopKArray {
     const result = { moveIn: null, moveOut: null };
     const index = this.#findIndex(value);
     const [removedElem] = this.#sortedValues.splice(index, 1);
+    if (this.#topKStart === this.#topKEnd) return result;
     if (index < this.#topKEnd) {
       result.moveOut = removedElem;
       if (index < this.#topKStart) {

package/dist/esm/operators/topKArray.js.map

@@ -1 +1 @@
-{"version":3,"file":"topKArray.js","sources":["../../../src/operators/topKArray.ts"],"sourcesContent":["import { generateKeyBetween } from 'fractional-indexing'\nimport { binarySearch, compareKeys, diffHalfOpen } from '../utils.js'\nimport type { HRange } from '../utils.js'\n\n// Abstraction for fractionally indexed values\nexport type FractionalIndex = string\nexport type IndexedValue<V> = [V, FractionalIndex]\n\nexport function indexedValue<V>(\n  value: V,\n  index: FractionalIndex,\n): IndexedValue<V> {\n  return [value, index]\n}\n\nexport function getValue<V>(indexedVal: IndexedValue<V>): V {\n  return indexedVal[0]\n}\n\nexport function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {\n  return indexedVal[1]\n}\n\n/**\n * Creates a comparator for [key, value] tuples that first compares values,\n * then uses the row key as a stable tie-breaker.\n */\nexport function createKeyedComparator<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n): (a: [K, T], b: [K, T]) => number {\n  return ([aKey, aVal], [bKey, bVal]) => {\n    // First compare on the value\n    const valueComparison = comparator(aVal, bVal)\n    if (valueComparison !== 0) {\n      return valueComparison\n    }\n    // If the values are equal, use the row key as tie-breaker\n    // This provides stable, deterministic ordering since keys are string | number\n    return compareKeys(aKey, bKey)\n  }\n}\n\nexport type TopKChanges<V> = {\n  /** Indicates which element moves into the topK (if any) */\n  moveIn: IndexedValue<V> | null\n  /** Indicates which element moves out of the topK (if any) */\n  moveOut: IndexedValue<V> | null\n}\n\nexport type TopKMoveChanges<V> = {\n  /** Flag that marks whether there were any changes to the topK */\n  changes: boolean\n  /** Indicates which elements move into the topK (if any) */\n  moveIns: Array<IndexedValue<V>>\n  /** Indicates which elements move out of the topK (if any) */\n  moveOuts: Array<IndexedValue<V>>\n}\n\n/**\n * A topK data structure that supports insertions and deletions\n * and returns changes to the topK.\n */\nexport interface TopK<V> {\n  size: number\n  insert: (value: V) => TopKChanges<V>\n  delete: (value: V) => TopKChanges<V>\n}\n\n/**\n * Implementation of a topK data structure.\n * Uses a sorted array internally to store the values and keeps a topK window over that array.\n * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted\n * at the start of the array which causes all the elements to shift to the right/left.\n */\nexport class TopKArray<V> implements TopK<V> {\n  #sortedValues: Array<IndexedValue<V>> = []\n  #comparator: (a: V, b: V) => number\n  #topKStart: number\n  #topKEnd: number\n\n  constructor(\n    offset: number,\n    limit: number,\n    comparator: (a: V, b: V) => number,\n  ) {\n    this.#topKStart = offset\n    this.#topKEnd = offset + limit\n    this.#comparator = comparator\n  }\n\n  get size(): number {\n    const offset = this.#topKStart\n    const limit = this.#topKEnd - this.#topKStart\n    const available = this.#sortedValues.length - offset\n    return Math.max(0, Math.min(limit, available))\n  }\n\n  /**\n   * Moves the topK window\n   */\n  move({\n    offset,\n    limit,\n  }: {\n    offset?: number\n    limit?: number\n  }): TopKMoveChanges<V> {\n    const oldOffset = this.#topKStart\n    const oldLimit = this.#topKEnd - this.#topKStart\n\n    // `this.#topKEnd` can be `Infinity` if it has no limit\n    // but `diffHalfOpen` expects a finite range\n    // so we restrict it to the size of the topK if topKEnd is infinite\n    const oldRange: HRange = [\n      this.#topKStart,\n      this.#topKEnd === Infinity ? this.#topKStart + this.size : this.#topKEnd,\n    ]\n\n    this.#topKStart = offset ?? oldOffset\n    this.#topKEnd = this.#topKStart + (limit ?? oldLimit) // can be `Infinity` if limit is `Infinity`\n\n    // Also handle `Infinity` in the newRange\n    const newRange: HRange = [\n      this.#topKStart,\n      this.#topKEnd === Infinity\n        ? Math.max(this.#topKStart + this.size, oldRange[1]) // since the new limit is Infinity we need to take everything (so we need to take the biggest (finite) topKEnd)\n        : this.#topKEnd,\n    ]\n    const { onlyInA, onlyInB } = diffHalfOpen(oldRange, newRange)\n\n    const moveIns: Array<IndexedValue<V>> = []\n    onlyInB.forEach((index) => {\n      const value = this.#sortedValues[index]\n      if (value) {\n        moveIns.push(value)\n      }\n    })\n\n    const moveOuts: Array<IndexedValue<V>> = []\n    onlyInA.forEach((index) => {\n      const value = this.#sortedValues[index]\n      if (value) {\n        moveOuts.push(value)\n      }\n    })\n\n    // It could be that there are changes (i.e. moveIns or moveOuts)\n    // but that the collection is lazy so we don't have the data yet that needs to move in/out\n    // so `moveIns` and `moveOuts` will be empty but `changes` will be true\n    // this will tell the caller that it needs to run the graph to load more data\n    return { moveIns, moveOuts, changes: onlyInA.length + onlyInB.length > 0 }\n  }\n\n  insert(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup insert position\n    const index = this.#findIndex(value)\n    // Generate fractional index based on the fractional indices of the elements before and after it\n    const indexBefore =\n      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)\n    const indexAfter =\n      index === this.#sortedValues.length\n        ? null\n        : getIndex(this.#sortedValues[index]!)\n    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)\n\n    // Insert the value at the correct position\n    const val = indexedValue(value, fractionalIndex)\n    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !\n    this.#sortedValues.splice(index, 0, val)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The inserted element is either before the top K or within the top K\n      // If it is before the top K then it moves the element that was right before the topK into the topK\n      // If it is within the top K then the inserted element moves into the top K\n      // In both cases the last element of the old top K now moves out of the top K\n      const moveInIndex = Math.max(index, this.#topKStart)\n      if (moveInIndex < this.#sortedValues.length) {\n        // We actually have a topK\n        // because in some cases there may not be enough elements in the array to reach the start of the topK\n        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK\n        result.moveIn = this.#sortedValues[moveInIndex]!\n\n        // We need to remove the element that falls out of the top K\n        // The element that falls out of the top K has shifted one to the right\n        // because of the element we inserted, so we find it at index topKEnd\n        if (this.#topKEnd < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[this.#topKEnd]!\n        }\n      }\n    }\n\n    return result\n  }\n\n  /**\n   * Deletes a value that may or may not be in the topK.\n   * IMPORTANT: this assumes that the value is present in the collection\n   *            if it's not the case it will remove the element\n   *            that is on the position where the provided `value` would be.\n   */\n  delete(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup delete position\n    const index = this.#findIndex(value)\n    // Remove the value at that position\n    const [removedElem] = this.#sortedValues.splice(index, 1)\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The removed element is either before the top K or within the top K\n      // If it is before the top K then the first element of the topK moves out of the topK\n      // If it is within the top K then the removed element moves out of the topK\n      result.moveOut = removedElem!\n      if (index < this.#topKStart) {\n        // The removed element is before the topK\n        // so actually, the first element of the topK moves out of the topK\n        // and not the element that we removed\n        // The first element of the topK is now at index topKStart - 1\n        // since we removed an element before the topK\n        const moveOutIndex = this.#topKStart - 1\n        if (moveOutIndex < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[moveOutIndex]!\n        } else {\n          // No value is moving out of the topK\n          // because there are no elements in the topK\n          result.moveOut = null\n        }\n      }\n\n      // Since we removed an element that was before or in the topK\n      // the first element after the topK moved one position to the left\n      // and thus falls into the topK now\n      const moveInIndex = this.#topKEnd - 1\n      if (moveInIndex < this.#sortedValues.length) {\n        result.moveIn = this.#sortedValues[moveInIndex]!\n      }\n    }\n\n    return result\n  }\n\n  // TODO: see if there is a way to refactor the code for insert and delete in the topK above\n  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right\n  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern\n\n  #findIndex(value: V): number {\n    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>\n      this.#comparator(getValue(a), getValue(b)),\n    )\n  }\n}\n"],"names":[],"mappings":";;AAQO,SAAS,aACd,OACA,OACiB;AACjB,SAAO,CAAC,OAAO,KAAK;AACtB;AAEO,SAAS,SAAY,YAAgC;AAC1D,SAAO,WAAW,CAAC;AACrB;AAEO,SAAS,SAAY,YAA8C;AACxE,SAAO,WAAW,CAAC;AACrB;AAMO,SAAS,sBACd,YACkC;AAClC,SAAO,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;AAErC,UAAM,kBAAkB,WAAW,MAAM,IAAI;AAC7C,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAGA,WAAO,YAAY,MAAM,IAAI;AAAA,EAC/B;AACF;AAkCO,MAAM,UAAgC;AAAA,EAC3C,gBAAwC,CAAA;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,QACA,OACA,YACA;AACA,SAAK,aAAa;AAClB,SAAK,WAAW,SAAS;AACzB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,OAAe;AACjB,UAAM,SAAS,KAAK;AACpB,UAAM,QAAQ,KAAK,WAAW,KAAK;AACnC,UAAM,YAAY,KAAK,cAAc,SAAS;AAC9C,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,SAAS,CAAC;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA,EAKA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,EAAA,GAIqB;AACrB,UAAM,YAAY,KAAK;AACvB,UAAM,WAAW,KAAK,WAAW,KAAK;AAKtC,UAAM,WAAmB;AAAA,MACvB,KAAK;AAAA,MACL,KAAK,aAAa,WAAW,KAAK,aAAa,KAAK,OAAO,KAAK;AAAA,IAAA;AAGlE,SAAK,aAAa,UAAU;AAC5B,SAAK,WAAW,KAAK,cAAc,SAAS;AAG5C,UAAM,WAAmB;AAAA,MACvB,KAAK;AAAA,MACL,KAAK,aAAa,WACd,KAAK,IAAI,KAAK,aAAa,KAAK,MAAM,SAAS,CAAC,CAAC,IACjD,KAAK;AAAA,IAAA;AAEX,UAAM,EAAE,SAAS,QAAA,IAAY,aAAa,UAAU,QAAQ;AAE5D,UAAM,UAAkC,CAAA;AACxC,YAAQ,QAAQ,CAAC,UAAU;AACzB,YAAM,QAAQ,KAAK,cAAc,KAAK;AACtC,UAAI,OAAO;AACT,gBAAQ,KAAK,KAAK;AAAA,MACpB;AAAA,IACF,CAAC;AAED,UAAM,WAAmC,CAAA;AACzC,YAAQ,QAAQ,CAAC,UAAU;AACzB,YAAM,QAAQ,KAAK,cAAc,KAAK;AACtC,UAAI,OAAO;AACT,iBAAS,KAAK,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAMD,WAAO,EAAE,SAAS,UAAU,SAAS,QAAQ,SAAS,QAAQ,SAAS,EAAA;AAAA,EACzE;AAAA,EAEA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,cACJ,UAAU,IAAI,OAAO,SAAS,KAAK,cAAc,QAAQ,CAAC,CAAE;AAC9D,UAAM,aACJ,UAAU,KAAK,cAAc,SACzB,OACA,SAAS,KAAK,cAAc,KAAK,CAAE;AACzC,UAAM,kBAAkB,mBAAmB,aAAa,UAAU;AAGlE,UAAM,MAAM,aAAa,OAAO,eAAe;AAE/C,SAAK,cAAc,OAAO,OAAO,GAAG,GAAG;AAGvC,QAAI,QAAQ,KAAK,UAAU;AAKzB,YAAM,cAAc,KAAK,IAAI,OAAO,KAAK,UAAU;AACnD,UAAI,cAAc,KAAK,cAAc,QAAQ;AAI3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAK9C,YAAI,KAAK,WAAW,KAAK,cAAc,QAAQ;AAC7C,iBAAO,UAAU,KAAK,cAAc,KAAK,QAAQ;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,CAAC,WAAW,IAAI,KAAK,cAAc,OAAO,OAAO,CAAC;AAGxD,QAAI,QAAQ,KAAK,UAAU;AAIzB,aAAO,UAAU;AACjB,UAAI,QAAQ,KAAK,YAAY;AAM3B,cAAM,eAAe,KAAK,aAAa;AACvC,YAAI,eAAe,KAAK,cAAc,QAAQ;AAC5C,iBAAO,UAAU,KAAK,cAAc,YAAY;AAAA,QAClD,OAAO;AAGL,iBAAO,UAAU;AAAA,QACnB;AAAA,MACF;AAKA,YAAM,cAAc,KAAK,WAAW;AACpC,UAAI,cAAc,KAAK,cAAc,QAAQ;AAC3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAAA,MAChD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,OAAkB;AAC3B,WAAO;AAAA,MAAa,KAAK;AAAA,MAAe,aAAa,OAAO,EAAE;AAAA,MAAG,CAAC,GAAG,MACnE,KAAK,YAAY,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC;AAAA,IAAA;AAAA,EAE7C;AACF;"}
+{"version":3,"file":"topKArray.js","sources":["../../../src/operators/topKArray.ts"],"sourcesContent":["import { generateKeyBetween } from 'fractional-indexing'\nimport { binarySearch, compareKeys, diffHalfOpen } from '../utils.js'\nimport type { HRange } from '../utils.js'\n\n// Abstraction for fractionally indexed values\nexport type FractionalIndex = string\nexport type IndexedValue<V> = [V, FractionalIndex]\n\nexport function indexedValue<V>(\n  value: V,\n  index: FractionalIndex,\n): IndexedValue<V> {\n  return [value, index]\n}\n\nexport function getValue<V>(indexedVal: IndexedValue<V>): V {\n  return indexedVal[0]\n}\n\nexport function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {\n  return indexedVal[1]\n}\n\n/**\n * Creates a comparator for [key, value] tuples that first compares values,\n * then uses the row key as a stable tie-breaker.\n */\nexport function createKeyedComparator<K extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n): (a: [K, T], b: [K, T]) => number {\n  return ([aKey, aVal], [bKey, bVal]) => {\n    // First compare on the value\n    const valueComparison = comparator(aVal, bVal)\n    if (valueComparison !== 0) {\n      return valueComparison\n    }\n    // If the values are equal, use the row key as tie-breaker\n    // This provides stable, deterministic ordering since keys are string | number\n    return compareKeys(aKey, bKey)\n  }\n}\n\nexport type TopKChanges<V> = {\n  /** Indicates which element moves into the topK (if any) */\n  moveIn: IndexedValue<V> | null\n  /** Indicates which element moves out of the topK (if any) */\n  moveOut: IndexedValue<V> | null\n}\n\nexport type TopKMoveChanges<V> = {\n  /** Flag that marks whether there were any changes to the topK */\n  changes: boolean\n  /** Indicates which elements move into the topK (if any) */\n  moveIns: Array<IndexedValue<V>>\n  /** Indicates which elements move out of the topK (if any) */\n  moveOuts: Array<IndexedValue<V>>\n}\n\n/**\n * A topK data structure that supports insertions and deletions\n * and returns changes to the topK.\n */\nexport interface TopK<V> {\n  size: number\n  insert: (value: V) => TopKChanges<V>\n  delete: (value: V) => TopKChanges<V>\n}\n\n/**\n * Implementation of a topK data structure.\n * Uses a sorted array internally to store the values and keeps a topK window over that array.\n * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted\n * at the start of the array which causes all the elements to shift to the right/left.\n */\nexport class TopKArray<V> implements TopK<V> {\n  #sortedValues: Array<IndexedValue<V>> = []\n  #comparator: (a: V, b: V) => number\n  #topKStart: number\n  #topKEnd: number\n\n  constructor(\n    offset: number,\n    limit: number,\n    comparator: (a: V, b: V) => number,\n  ) {\n    this.#topKStart = offset\n    this.#topKEnd = offset + limit\n    this.#comparator = comparator\n  }\n\n  get size(): number {\n    const offset = this.#topKStart\n    const limit = this.#topKEnd - this.#topKStart\n    const available = this.#sortedValues.length - offset\n    return Math.max(0, Math.min(limit, available))\n  }\n\n  /**\n   * Moves the topK window\n   */\n  move({\n    offset,\n    limit,\n  }: {\n    offset?: number\n    limit?: number\n  }): TopKMoveChanges<V> {\n    const oldOffset = this.#topKStart\n    const oldLimit = this.#topKEnd - this.#topKStart\n\n    // `this.#topKEnd` can be `Infinity` if it has no limit\n    // but `diffHalfOpen` expects a finite range\n    // so we restrict it to the size of the topK if topKEnd is infinite\n    const oldRange: HRange = [\n      this.#topKStart,\n      this.#topKEnd === Infinity ? this.#topKStart + this.size : this.#topKEnd,\n    ]\n\n    this.#topKStart = offset ?? oldOffset\n    this.#topKEnd = this.#topKStart + (limit ?? oldLimit) // can be `Infinity` if limit is `Infinity`\n\n    // Also handle `Infinity` in the newRange\n    const newRange: HRange = [\n      this.#topKStart,\n      this.#topKEnd === Infinity\n        ? Math.max(this.#topKStart + this.size, oldRange[1]) // since the new limit is Infinity we need to take everything (so we need to take the biggest (finite) topKEnd)\n        : this.#topKEnd,\n    ]\n    const { onlyInA, onlyInB } = diffHalfOpen(oldRange, newRange)\n\n    const moveIns: Array<IndexedValue<V>> = []\n    onlyInB.forEach((index) => {\n      const value = this.#sortedValues[index]\n      if (value) {\n        moveIns.push(value)\n      }\n    })\n\n    const moveOuts: Array<IndexedValue<V>> = []\n    onlyInA.forEach((index) => {\n      const value = this.#sortedValues[index]\n      if (value) {\n        moveOuts.push(value)\n      }\n    })\n\n    // It could be that there are changes (i.e. moveIns or moveOuts)\n    // but that the collection is lazy so we don't have the data yet that needs to move in/out\n    // so `moveIns` and `moveOuts` will be empty but `changes` will be true\n    // this will tell the caller that it needs to run the graph to load more data\n    return { moveIns, moveOuts, changes: onlyInA.length + onlyInB.length > 0 }\n  }\n\n  insert(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup insert position\n    const index = this.#findIndex(value)\n    // Generate fractional index based on the fractional indices of the elements before and after it\n    const indexBefore =\n      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)\n    const indexAfter =\n      index === this.#sortedValues.length\n        ? null\n        : getIndex(this.#sortedValues[index]!)\n    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)\n\n    // Insert the value at the correct position\n    const val = indexedValue(value, fractionalIndex)\n    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !\n    this.#sortedValues.splice(index, 0, val)\n\n    if (this.#topKStart === this.#topKEnd) return result\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The inserted element is either before the top K or within the top K\n      // If it is before the top K then it moves the element that was right before the topK into the topK\n      // If it is within the top K then the inserted element moves into the top K\n      // In both cases the last element of the old top K now moves out of the top K\n      const moveInIndex = Math.max(index, this.#topKStart)\n      if (moveInIndex < this.#sortedValues.length) {\n        // We actually have a topK\n        // because in some cases there may not be enough elements in the array to reach the start of the topK\n        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK\n        result.moveIn = this.#sortedValues[moveInIndex]!\n\n        // We need to remove the element that falls out of the top K\n        // The element that falls out of the top K has shifted one to the right\n        // because of the element we inserted, so we find it at index topKEnd\n        if (this.#topKEnd < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[this.#topKEnd]!\n        }\n      }\n    }\n\n    return result\n  }\n\n  /**\n   * Deletes a value that may or may not be in the topK.\n   * IMPORTANT: this assumes that the value is present in the collection\n   *            if it's not the case it will remove the element\n   *            that is on the position where the provided `value` would be.\n   */\n  delete(value: V): TopKChanges<V> {\n    const result: TopKChanges<V> = { moveIn: null, moveOut: null }\n\n    // Lookup delete position\n    const index = this.#findIndex(value)\n    // Remove the value at that position\n    const [removedElem] = this.#sortedValues.splice(index, 1)\n\n    if (this.#topKStart === this.#topKEnd) return result\n\n    // Check if the topK changed\n    if (index < this.#topKEnd) {\n      // The removed element is either before the top K or within the top K\n      // If it is before the top K then the first element of the topK moves out of the topK\n      // If it is within the top K then the removed element moves out of the topK\n      result.moveOut = removedElem!\n      if (index < this.#topKStart) {\n        // The removed element is before the topK\n        // so actually, the first element of the topK moves out of the topK\n        // and not the element that we removed\n        // The first element of the topK is now at index topKStart - 1\n        // since we removed an element before the topK\n        const moveOutIndex = this.#topKStart - 1\n        if (moveOutIndex < this.#sortedValues.length) {\n          result.moveOut = this.#sortedValues[moveOutIndex]!\n        } else {\n          // No value is moving out of the topK\n          // because there are no elements in the topK\n          result.moveOut = null\n        }\n      }\n\n      // Since we removed an element that was before or in the topK\n      // the first element after the topK moved one position to the left\n      // and thus falls into the topK now\n      const moveInIndex = this.#topKEnd - 1\n      if (moveInIndex < this.#sortedValues.length) {\n        result.moveIn = this.#sortedValues[moveInIndex]!\n      }\n    }\n\n    return result\n  }\n\n  // TODO: see if there is a way to refactor the code for insert and delete in the topK above\n  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right\n  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern\n\n  #findIndex(value: V): number {\n    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>\n      this.#comparator(getValue(a), getValue(b)),\n    )\n  }\n}\n"],"names":[],"mappings":";;AAQO,SAAS,aACd,OACA,OACiB;AACjB,SAAO,CAAC,OAAO,KAAK;AACtB;AAEO,SAAS,SAAY,YAAgC;AAC1D,SAAO,WAAW,CAAC;AACrB;AAEO,SAAS,SAAY,YAA8C;AACxE,SAAO,WAAW,CAAC;AACrB;AAMO,SAAS,sBACd,YACkC;AAClC,SAAO,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;AAErC,UAAM,kBAAkB,WAAW,MAAM,IAAI;AAC7C,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAGA,WAAO,YAAY,MAAM,IAAI;AAAA,EAC/B;AACF;AAkCO,MAAM,UAAgC;AAAA,EAC3C,gBAAwC,CAAA;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,QACA,OACA,YACA;AACA,SAAK,aAAa;AAClB,SAAK,WAAW,SAAS;AACzB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,OAAe;AACjB,UAAM,SAAS,KAAK;AACpB,UAAM,QAAQ,KAAK,WAAW,KAAK;AACnC,UAAM,YAAY,KAAK,cAAc,SAAS;AAC9C,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,SAAS,CAAC;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA,EAKA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,EAAA,GAIqB;AACrB,UAAM,YAAY,KAAK;AACvB,UAAM,WAAW,KAAK,WAAW,KAAK;AAKtC,UAAM,WAAmB;AAAA,MACvB,KAAK;AAAA,MACL,KAAK,aAAa,WAAW,KAAK,aAAa,KAAK,OAAO,KAAK;AAAA,IAAA;AAGlE,SAAK,aAAa,UAAU;AAC5B,SAAK,WAAW,KAAK,cAAc,SAAS;AAG5C,UAAM,WAAmB;AAAA,MACvB,KAAK;AAAA,MACL,KAAK,aAAa,WACd,KAAK,IAAI,KAAK,aAAa,KAAK,MAAM,SAAS,CAAC,CAAC,IACjD,KAAK;AAAA,IAAA;AAEX,UAAM,EAAE,SAAS,QAAA,IAAY,aAAa,UAAU,QAAQ;AAE5D,UAAM,UAAkC,CAAA;AACxC,YAAQ,QAAQ,CAAC,UAAU;AACzB,YAAM,QAAQ,KAAK,cAAc,KAAK;AACtC,UAAI,OAAO;AACT,gBAAQ,KAAK,KAAK;AAAA,MACpB;AAAA,IACF,CAAC;AAED,UAAM,WAAmC,CAAA;AACzC,YAAQ,QAAQ,CAAC,UAAU;AACzB,YAAM,QAAQ,KAAK,cAAc,KAAK;AACtC,UAAI,OAAO;AACT,iBAAS,KAAK,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAMD,WAAO,EAAE,SAAS,UAAU,SAAS,QAAQ,SAAS,QAAQ,SAAS,EAAA;AAAA,EACzE;AAAA,EAEA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,cACJ,UAAU,IAAI,OAAO,SAAS,KAAK,cAAc,QAAQ,CAAC,CAAE;AAC9D,UAAM,aACJ,UAAU,KAAK,cAAc,SACzB,OACA,SAAS,KAAK,cAAc,KAAK,CAAE;AACzC,UAAM,kBAAkB,mBAAmB,aAAa,UAAU;AAGlE,UAAM,MAAM,aAAa,OAAO,eAAe;AAE/C,SAAK,cAAc,OAAO,OAAO,GAAG,GAAG;AAEvC,QAAI,KAAK,eAAe,KAAK,SAAU,QAAO;AAG9C,QAAI,QAAQ,KAAK,UAAU;AAKzB,YAAM,cAAc,KAAK,IAAI,OAAO,KAAK,UAAU;AACnD,UAAI,cAAc,KAAK,cAAc,QAAQ;AAI3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAK9C,YAAI,KAAK,WAAW,KAAK,cAAc,QAAQ;AAC7C,iBAAO,UAAU,KAAK,cAAc,KAAK,QAAQ;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,OAAO,OAA0B;AAC/B,UAAM,SAAyB,EAAE,QAAQ,MAAM,SAAS,KAAA;AAGxD,UAAM,QAAQ,KAAK,WAAW,KAAK;AAEnC,UAAM,CAAC,WAAW,IAAI,KAAK,cAAc,OAAO,OAAO,CAAC;AAExD,QAAI,KAAK,eAAe,KAAK,SAAU,QAAO;AAG9C,QAAI,QAAQ,KAAK,UAAU;AAIzB,aAAO,UAAU;AACjB,UAAI,QAAQ,KAAK,YAAY;AAM3B,cAAM,eAAe,KAAK,aAAa;AACvC,YAAI,eAAe,KAAK,cAAc,QAAQ;AAC5C,iBAAO,UAAU,KAAK,cAAc,YAAY;AAAA,QAClD,OAAO;AAGL,iBAAO,UAAU;AAAA,QACnB;AAAA,MACF;AAKA,YAAM,cAAc,KAAK,WAAW;AACpC,UAAI,cAAc,KAAK,cAAc,QAAQ;AAC3C,eAAO,SAAS,KAAK,cAAc,WAAW;AAAA,MAChD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,OAAkB;AAC3B,WAAO;AAAA,MAAa,KAAK;AAAA,MAAe,aAAa,OAAO,EAAE;AAAA,MAAG,CAAC,GAAG,MACnE,KAAK,YAAY,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC;AAAA,IAAA;AAAA,EAE7C;AACF;"}

package/dist/esm/operators/topKState.d.ts

@@ -1,4 +1,13 @@
+import { MultiSet } from '../multiset.js';
 import { IndexedValue, TopK, TopKChanges, TopKMoveChanges } from './topKArray.js';
+/** Apply a keyed batch's retractions before additions. An outer join can emit
+ * a replacement in the opposite order; key multiplicity alone would hide it.
+ * Consolidate first so a transient value added and removed in this same turn
+ * cannot be mistaken for the final replacement. Use structural relation
+ * identity: map can allocate a fresh value for each side of a cancelling pair.
+ * No state survives the run.
+ */
+export declare function topKBatch<K, T>(messages: Array<MultiSet<[K, T]>>): Generator<[[K, T], number], void, unknown>;
 /**
  * Helper class that manages the state for a single topK window.
  * Encapsulates the multiplicity tracking and topK data structure,

package/dist/esm/operators/topKState.js

@@ -1,4 +1,33 @@
+import { equalHashValues } from "../hashing/hash.js";
 import { TopKArray } from "./topKArray.js";
+function* topKBatch(messages) {
+  const keyed = /* @__PURE__ */ new Map();
+  for (const message of messages) {
+    for (const [value, weight] of message.getInner()) {
+      const entries = keyed.get(value[0]);
+      if (entries) entries.push([value, weight]);
+      else keyed.set(value[0], [[value, weight]]);
+    }
+  }
+  const batch = [];
+  for (const entries of keyed.values()) {
+    if (entries.length === 1) {
+      batch.push(entries[0]);
+      continue;
+    }
+    const consolidated = [];
+    for (const [value, weight] of entries) {
+      const previous = consolidated.find(
+        ([candidate]) => equalHashValues(candidate[1], value[1])
+      );
+      if (previous) previous[1] += weight;
+      else consolidated.push([value, weight]);
+    }
+    batch.push(...consolidated);
+  }
+  for (const entry of batch) if (entry[1] < 0) yield entry;
+  for (const entry of batch) if (entry[1] > 0) yield entry;
+}
 class TopKState {
   #multiplicities = /* @__PURE__ */ new Map();
   #topK;
@@ -68,6 +97,7 @@ function handleMoveOut(moveOut, result) {
 export {
   TopKState,
   handleMoveIn,
-  handleMoveOut
+  handleMoveOut,
+  topKBatch
 };
 //# sourceMappingURL=topKState.js.map

package/dist/esm/operators/topKState.js.map

@@ -1 +1 @@
-{"version":3,"file":"topKState.js","sources":["../../../src/operators/topKState.ts"],"sourcesContent":["import { TopKArray } from './topKArray.js'\nimport type {\n  IndexedValue,\n  TopK,\n  TopKChanges,\n  TopKMoveChanges,\n} from './topKArray.js'\n\n/**\n * Helper class that manages the state for a single topK window.\n * Encapsulates the multiplicity tracking and topK data structure,\n * providing a clean interface for processing elements and moving the window.\n *\n * This class is used by both TopKWithFractionalIndexOperator (single instance)\n * and GroupedTopKWithFractionalIndexOperator (one instance per group).\n */\nexport class TopKState<K extends string | number, T> {\n  #multiplicities: Map<K, number> = new Map()\n  #topK: TopK<[K, T]>\n\n  constructor(topK: TopK<[K, T]>) {\n    this.#topK = topK\n  }\n\n  get size(): number {\n    return this.#topK.size\n  }\n\n  get isEmpty(): boolean {\n    return this.#multiplicities.size === 0 && this.#topK.size === 0\n  }\n\n  /**\n   * Process an element update (insert or delete based on multiplicity change).\n   * Returns the changes to the topK window.\n   */\n  processElement(key: K, value: T, multiplicity: number): TopKChanges<[K, T]> {\n    const { oldMultiplicity, newMultiplicity } = this.#updateMultiplicity(\n      key,\n      multiplicity,\n    )\n\n    if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n      // The value was invisible but should now be visible\n      return this.#topK.insert([key, value])\n    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n      // The value was visible but should now be invisible\n      return this.#topK.delete([key, value])\n    }\n    // The value was invisible and remains invisible,\n    // or was visible and remains visible - no topK change\n    return { moveIn: null, moveOut: null }\n  }\n\n  /**\n   * Move the topK window. Only works with TopKArray implementation.\n   */\n  move(options: { offset?: number; limit?: number }): TopKMoveChanges<[K, T]> {\n    if (!(this.#topK instanceof TopKArray)) {\n      throw new Error(\n        `Cannot move B+-tree implementation of TopK with fractional index`,\n      )\n    }\n    return this.#topK.move(options)\n  }\n\n  #updateMultiplicity(\n    key: K,\n    multiplicity: number,\n  ): { oldMultiplicity: number; newMultiplicity: number } {\n    if (multiplicity === 0) {\n      const current = this.#multiplicities.get(key) ?? 0\n      return { oldMultiplicity: current, newMultiplicity: current }\n    }\n\n    const oldMultiplicity = this.#multiplicities.get(key) ?? 0\n    const newMultiplicity = oldMultiplicity + multiplicity\n    if (newMultiplicity === 0) {\n      this.#multiplicities.delete(key)\n    } else {\n      this.#multiplicities.set(key, newMultiplicity)\n    }\n    return { oldMultiplicity, newMultiplicity }\n  }\n}\n\n/**\n * Handles a moveIn change by adding it to the result array.\n */\nexport function handleMoveIn<K extends string | number, T>(\n  moveIn: IndexedValue<[K, T]> | null,\n  result: Array<[[K, IndexedValue<T>], number]>,\n): void {\n  if (moveIn) {\n    const [[key, value], index] = moveIn\n    result.push([[key, [value, index]], 1])\n  }\n}\n\n/**\n * Handles a moveOut change by adding it to the result array.\n */\nexport function handleMoveOut<K extends string | number, T>(\n  moveOut: IndexedValue<[K, T]> | null,\n  result: Array<[[K, IndexedValue<T>], number]>,\n): void {\n  if (moveOut) {\n    const [[key, value], index] = moveOut\n    result.push([[key, [value, index]], -1])\n  }\n}\n"],"names":[],"mappings":";AAgBO,MAAM,UAAwC;AAAA,EACnD,sCAAsC,IAAA;AAAA,EACtC;AAAA,EAEA,YAAY,MAAoB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,IAAI,OAAe;AACjB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK,gBAAgB,SAAS,KAAK,KAAK,MAAM,SAAS;AAAA,EAChE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,eAAe,KAAQ,OAAU,cAA2C;AAC1E,UAAM,EAAE,iBAAiB,gBAAA,IAAoB,KAAK;AAAA,MAChD;AAAA,MACA;AAAA,IAAA;AAGF,QAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAE/C,aAAO,KAAK,MAAM,OAAO,CAAC,KAAK,KAAK,CAAC;AAAA,IACvC,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAEtD,aAAO,KAAK,MAAM,OAAO,CAAC,KAAK,KAAK,CAAC;AAAA,IACvC;AAGA,WAAO,EAAE,QAAQ,MAAM,SAAS,KAAA;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA,EAKA,KAAK,SAAuE;AAC1E,QAAI,EAAE,KAAK,iBAAiB,YAAY;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO,KAAK,MAAM,KAAK,OAAO;AAAA,EAChC;AAAA,EAEA,oBACE,KACA,cACsD;AACtD,QAAI,iBAAiB,GAAG;AACtB,YAAM,UAAU,KAAK,gBAAgB,IAAI,GAAG,KAAK;AACjD,aAAO,EAAE,iBAAiB,SAAS,iBAAiB,QAAA;AAAA,IACtD;AAEA,UAAM,kBAAkB,KAAK,gBAAgB,IAAI,GAAG,KAAK;AACzD,UAAM,kBAAkB,kBAAkB;AAC1C,QAAI,oBAAoB,GAAG;AACzB,WAAK,gBAAgB,OAAO,GAAG;AAAA,IACjC,OAAO;AACL,WAAK,gBAAgB,IAAI,KAAK,eAAe;AAAA,IAC/C;AACA,WAAO,EAAE,iBAAiB,gBAAA;AAAA,EAC5B;AACF;AAKO,SAAS,aACd,QACA,QACM;AACN,MAAI,QAAQ;AACV,UAAM,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,IAAI;AAC9B,WAAO,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,EACxC;AACF;AAKO,SAAS,cACd,SACA,QACM;AACN,MAAI,SAAS;AACX,UAAM,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,IAAI;AAC9B,WAAO,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,EAAE,CAAC;AAAA,EACzC;AACF;"}
+{"version":3,"file":"topKState.js","sources":["../../../src/operators/topKState.ts"],"sourcesContent":["import { equalHashValues } from '../hashing/hash.js'\nimport { TopKArray } from './topKArray.js'\nimport type { MultiSet } from '../multiset.js'\nimport type {\n  IndexedValue,\n  TopK,\n  TopKChanges,\n  TopKMoveChanges,\n} from './topKArray.js'\n\n/** Apply a keyed batch's retractions before additions. An outer join can emit\n * a replacement in the opposite order; key multiplicity alone would hide it.\n * Consolidate first so a transient value added and removed in this same turn\n * cannot be mistaken for the final replacement. Use structural relation\n * identity: map can allocate a fresh value for each side of a cancelling pair.\n * No state survives the run.\n */\nexport function* topKBatch<K, T>(messages: Array<MultiSet<[K, T]>>) {\n  const keyed = new Map<K, Array<[[K, T], number]>>()\n  for (const message of messages) {\n    for (const [value, weight] of message.getInner()) {\n      const entries = keyed.get(value[0])\n      if (entries) entries.push([value, weight])\n      else keyed.set(value[0], [[value, weight]])\n    }\n  }\n  const batch: Array<[[K, T], number]> = []\n  for (const entries of keyed.values()) {\n    // Ordering distinct keys needs only the caller's comparator. In particular,\n    // do not traverse irrelevant payloads or merge keys on a hash collision.\n    if (entries.length === 1) {\n      batch.push(entries[0]!)\n      continue\n    }\n    // A per-key scan keeps normal two-version replacements cheap and permits\n    // cycles, at the cost of quadratic comparisons for long same-key histories.\n    const consolidated: Array<[[K, T], number]> = []\n    for (const [value, weight] of entries) {\n      const previous = consolidated.find(([candidate]) =>\n        equalHashValues(candidate[1], value[1]),\n      )\n      if (previous) previous[1] += weight\n      else consolidated.push([value, weight])\n    }\n    batch.push(...consolidated)\n  }\n  for (const entry of batch) if (entry[1] < 0) yield entry\n  for (const entry of batch) if (entry[1] > 0) yield entry\n}\n\n/**\n * Helper class that manages the state for a single topK window.\n * Encapsulates the multiplicity tracking and topK data structure,\n * providing a clean interface for processing elements and moving the window.\n *\n * This class is used by both TopKWithFractionalIndexOperator (single instance)\n * and GroupedTopKWithFractionalIndexOperator (one instance per group).\n */\nexport class TopKState<K extends string | number, T> {\n  #multiplicities: Map<K, number> = new Map()\n  #topK: TopK<[K, T]>\n\n  constructor(topK: TopK<[K, T]>) {\n    this.#topK = topK\n  }\n\n  get size(): number {\n    return this.#topK.size\n  }\n\n  get isEmpty(): boolean {\n    return this.#multiplicities.size === 0 && this.#topK.size === 0\n  }\n\n  /**\n   * Process an element update (insert or delete based on multiplicity change).\n   * Returns the changes to the topK window.\n   */\n  processElement(key: K, value: T, multiplicity: number): TopKChanges<[K, T]> {\n    const { oldMultiplicity, newMultiplicity } = this.#updateMultiplicity(\n      key,\n      multiplicity,\n    )\n\n    if (oldMultiplicity <= 0 && newMultiplicity > 0) {\n      // The value was invisible but should now be visible\n      return this.#topK.insert([key, value])\n    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {\n      // The value was visible but should now be invisible\n      return this.#topK.delete([key, value])\n    }\n    // The value was invisible and remains invisible,\n    // or was visible and remains visible - no topK change\n    return { moveIn: null, moveOut: null }\n  }\n\n  /**\n   * Move the topK window. Only works with TopKArray implementation.\n   */\n  move(options: { offset?: number; limit?: number }): TopKMoveChanges<[K, T]> {\n    if (!(this.#topK instanceof TopKArray)) {\n      throw new Error(\n        `Cannot move B+-tree implementation of TopK with fractional index`,\n      )\n    }\n    return this.#topK.move(options)\n  }\n\n  #updateMultiplicity(\n    key: K,\n    multiplicity: number,\n  ): { oldMultiplicity: number; newMultiplicity: number } {\n    if (multiplicity === 0) {\n      const current = this.#multiplicities.get(key) ?? 0\n      return { oldMultiplicity: current, newMultiplicity: current }\n    }\n\n    const oldMultiplicity = this.#multiplicities.get(key) ?? 0\n    const newMultiplicity = oldMultiplicity + multiplicity\n    if (newMultiplicity === 0) {\n      this.#multiplicities.delete(key)\n    } else {\n      this.#multiplicities.set(key, newMultiplicity)\n    }\n    return { oldMultiplicity, newMultiplicity }\n  }\n}\n\n/**\n * Handles a moveIn change by adding it to the result array.\n */\nexport function handleMoveIn<K extends string | number, T>(\n  moveIn: IndexedValue<[K, T]> | null,\n  result: Array<[[K, IndexedValue<T>], number]>,\n): void {\n  if (moveIn) {\n    const [[key, value], index] = moveIn\n    result.push([[key, [value, index]], 1])\n  }\n}\n\n/**\n * Handles a moveOut change by adding it to the result array.\n */\nexport function handleMoveOut<K extends string | number, T>(\n  moveOut: IndexedValue<[K, T]> | null,\n  result: Array<[[K, IndexedValue<T>], number]>,\n): void {\n  if (moveOut) {\n    const [[key, value], index] = moveOut\n    result.push([[key, [value, index]], -1])\n  }\n}\n"],"names":[],"mappings":";;AAiBO,UAAU,UAAgB,UAAmC;AAClE,QAAM,4BAAY,IAAA;AAClB,aAAW,WAAW,UAAU;AAC9B,eAAW,CAAC,OAAO,MAAM,KAAK,QAAQ,YAAY;AAChD,YAAM,UAAU,MAAM,IAAI,MAAM,CAAC,CAAC;AAClC,UAAI,QAAS,SAAQ,KAAK,CAAC,OAAO,MAAM,CAAC;AAAA,UACpC,OAAM,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC;AAAA,IAC5C;AAAA,EACF;AACA,QAAM,QAAiC,CAAA;AACvC,aAAW,WAAW,MAAM,UAAU;AAGpC,QAAI,QAAQ,WAAW,GAAG;AACxB,YAAM,KAAK,QAAQ,CAAC,CAAE;AACtB;AAAA,IACF;AAGA,UAAM,eAAwC,CAAA;AAC9C,eAAW,CAAC,OAAO,MAAM,KAAK,SAAS;AACrC,YAAM,WAAW,aAAa;AAAA,QAAK,CAAC,CAAC,SAAS,MAC5C,gBAAgB,UAAU,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,MAAA;AAExC,UAAI,SAAU,UAAS,CAAC,KAAK;AAAA,UACxB,cAAa,KAAK,CAAC,OAAO,MAAM,CAAC;AAAA,IACxC;AACA,UAAM,KAAK,GAAG,YAAY;AAAA,EAC5B;AACA,aAAW,SAAS,MAAO,KAAI,MAAM,CAAC,IAAI,EAAG,OAAM;AACnD,aAAW,SAAS,MAAO,KAAI,MAAM,CAAC,IAAI,EAAG,OAAM;AACrD;AAUO,MAAM,UAAwC;AAAA,EACnD,sCAAsC,IAAA;AAAA,EACtC;AAAA,EAEA,YAAY,MAAoB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,IAAI,OAAe;AACjB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK,gBAAgB,SAAS,KAAK,KAAK,MAAM,SAAS;AAAA,EAChE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,eAAe,KAAQ,OAAU,cAA2C;AAC1E,UAAM,EAAE,iBAAiB,gBAAA,IAAoB,KAAK;AAAA,MAChD;AAAA,MACA;AAAA,IAAA;AAGF,QAAI,mBAAmB,KAAK,kBAAkB,GAAG;AAE/C,aAAO,KAAK,MAAM,OAAO,CAAC,KAAK,KAAK,CAAC;AAAA,IACvC,WAAW,kBAAkB,KAAK,mBAAmB,GAAG;AAEtD,aAAO,KAAK,MAAM,OAAO,CAAC,KAAK,KAAK,CAAC;AAAA,IACvC;AAGA,WAAO,EAAE,QAAQ,MAAM,SAAS,KAAA;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA,EAKA,KAAK,SAAuE;AAC1E,QAAI,EAAE,KAAK,iBAAiB,YAAY;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO,KAAK,MAAM,KAAK,OAAO;AAAA,EAChC;AAAA,EAEA,oBACE,KACA,cACsD;AACtD,QAAI,iBAAiB,GAAG;AACtB,YAAM,UAAU,KAAK,gBAAgB,IAAI,GAAG,KAAK;AACjD,aAAO,EAAE,iBAAiB,SAAS,iBAAiB,QAAA;AAAA,IACtD;AAEA,UAAM,kBAAkB,KAAK,gBAAgB,IAAI,GAAG,KAAK;AACzD,UAAM,kBAAkB,kBAAkB;AAC1C,QAAI,oBAAoB,GAAG;AACzB,WAAK,gBAAgB,OAAO,GAAG;AAAA,IACjC,OAAO;AACL,WAAK,gBAAgB,IAAI,KAAK,eAAe;AAAA,IAC/C;AACA,WAAO,EAAE,iBAAiB,gBAAA;AAAA,EAC5B;AACF;AAKO,SAAS,aACd,QACA,QACM;AACN,MAAI,QAAQ;AACV,UAAM,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,IAAI;AAC9B,WAAO,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;AAAA,EACxC;AACF;AAKO,SAAS,cACd,SACA,QACM;AACN,MAAI,SAAS;AACX,UAAM,CAAC,CAAC,KAAK,KAAK,GAAG,KAAK,IAAI;AAC9B,WAAO,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,GAAG,EAAE,CAAC;AAAA,EACzC;AACF;"}

package/dist/esm/operators/topKWithFractionalIndex.js

@@ -1,7 +1,7 @@
 import { UnaryOperator, DifferenceStreamWriter } from "../graph.js";
 import { StreamBuilder } from "../d2.js";
 import { MultiSet } from "../multiset.js";
-import { TopKState, handleMoveIn, handleMoveOut } from "./topKState.js";
+import { TopKState, handleMoveIn, handleMoveOut, topKBatch } from "./topKState.js";
 import { createKeyedComparator, TopKArray } from "./topKArray.js";
 class TopKWithFractionalIndexOperator extends UnaryOperator {
   #state;
@@ -36,11 +36,10 @@ class TopKWithFractionalIndexOperator extends UnaryOperator {
   }
   run() {
     const result = [];
-    for (const message of this.inputMessages()) {
-      for (const [item, multiplicity] of message.getInner()) {
-        const [key, value] = item;
-        this.processElement(key, value, multiplicity, result);
-      }
+    for (const [[key, value], multiplicity] of topKBatch(
+      this.inputMessages()
+    )) {
+      this.processElement(key, value, multiplicity, result);
     }
     if (result.length > 0) {
       this.output.sendData(new MultiSet(result));

package/dist/esm/operators/topKWithFractionalIndex.js.map

@@ -1 +1 @@
-{"version":3,"file":"topKWithFractionalIndex.js","sources":["../../../src/operators/topKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue, TopK } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface TopKWithFractionalIndexOptions {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n}\n\n/**\n * Operator for fractional indexed topK operations\n * This operator maintains fractional indices for sorted elements\n * and only updates indices when elements move position\n */\nexport class TopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #state: TopKState<K, T>\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: TopKWithFractionalIndexOptions,\n  ) {\n    super(id, inputA, output)\n    const limit = options.limit ?? Infinity\n    const offset = options.offset ?? 0\n    const topK = this.createTopK(\n      offset,\n      limit,\n      createKeyedComparator(comparator),\n    )\n    this.#state = new TopKState(topK)\n    options.setSizeCallback?.(() => this.#state.size)\n    options.setWindowFn?.(this.moveTopK.bind(this))\n  }\n\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: [K, T], b: [K, T]) => number,\n  ): TopK<[K, T]> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  /**\n   * Moves the topK window based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  moveTopK({ offset, limit }: { offset?: number; limit?: number }) {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    const diff = this.#state.move({ offset, limit })\n\n    diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n    diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n    if (diff.changes) {\n      // There are changes to the topK\n      // it could be that moveIns and moveOuts are empty\n      // because the collection is lazy, so we will run the graph again to load the data\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const message of this.inputMessages()) {\n      for (const [item, multiplicity] of message.getInner()) {\n        const [key, value] = item\n        this.processElement(key, value, multiplicity, result)\n      }\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const changes = this.#state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithFractionalIndex<KType extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options?: TopKWithFractionalIndexOptions,\n): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {\n  const opts = options || {}\n\n  return (\n    stream: IStreamBuilder<[KType, T]>,\n  ): IStreamBuilder<[KType, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[KType, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[KType, IndexedValue<T>]>(),\n    )\n    const operator = new TopKWithFractionalIndexOperator<KType, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      opts,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;AAuBO,MAAM,wCAGH,cAA4C;AAAA,EACpD;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,UAAM,QAAQ,QAAQ,SAAS;AAC/B,UAAM,SAAS,QAAQ,UAAU;AACjC,UAAM,OAAO,KAAK;AAAA,MAChB;AAAA,MACA;AAAA,MACA,sBAAsB,UAAU;AAAA,IAAA;AAElC,SAAK,SAAS,IAAI,UAAU,IAAI;AAChC,YAAQ,kBAAkB,MAAM,KAAK,OAAO,IAAI;AAChD,YAAQ,cAAc,KAAK,SAAS,KAAK,IAAI,CAAC;AAAA,EAChD;AAAA,EAEU,WACR,QACA,OACA,YACc;AACd,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,EAAE,QAAQ,SAA8C;AAC/D,UAAM,SAAgD,CAAA;AACtD,UAAM,OAAO,KAAK,OAAO,KAAK,EAAE,QAAQ,OAAO;AAE/C,SAAK,QAAQ,QAAQ,CAAC,WAAW,aAAa,QAAQ,MAAM,CAAC;AAC7D,SAAK,SAAS,QAAQ,CAAC,YAAY,cAAc,SAAS,MAAM,CAAC;AAEjE,QAAI,KAAK,SAAS;AAIhB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,WAAW,KAAK,iBAAiB;AAC1C,iBAAW,CAAC,MAAM,YAAY,KAAK,QAAQ,YAAY;AACrD,cAAM,CAAC,KAAK,KAAK,IAAI;AACrB,aAAK,eAAe,KAAK,OAAO,cAAc,MAAM;AAAA,MACtD;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,eACE,KACA,OACA,cACA,QACM;AACN,UAAM,UAAU,KAAK,OAAO,eAAe,KAAK,OAAO,YAAY;AACnE,iBAAa,QAAQ,QAAQ,MAAM;AACnC,kBAAc,QAAQ,SAAS,MAAM;AAAA,EACvC;AACF;AAYO,SAAS,wBACd,YACA,SACqD;AACrD,QAAM,OAAO,WAAW,CAAA;AAExB,SAAO,CACL,WAC6C;AAC7C,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAAiD;AAEvD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"topKWithFractionalIndex.js","sources":["../../../src/operators/topKWithFractionalIndex.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { MultiSet } from '../multiset.js'\nimport {\n  TopKState,\n  handleMoveIn,\n  handleMoveOut,\n  topKBatch,\n} from './topKState.js'\nimport { TopKArray, createKeyedComparator } from './topKArray.js'\nimport type { IndexedValue, TopK } from './topKArray.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\n\nexport interface TopKWithFractionalIndexOptions {\n  limit?: number\n  offset?: number\n  setSizeCallback?: (getSize: () => number) => void\n  setWindowFn?: (\n    windowFn: (options: { offset?: number; limit?: number }) => void,\n  ) => void\n}\n\n/**\n * Operator for fractional indexed topK operations\n * This operator maintains fractional indices for sorted elements\n * and only updates indices when elements move position\n */\nexport class TopKWithFractionalIndexOperator<\n  K extends string | number,\n  T,\n> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {\n  #state: TopKState<K, T>\n\n  constructor(\n    id: number,\n    inputA: DifferenceStreamReader<[K, T]>,\n    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,\n    comparator: (a: T, b: T) => number,\n    options: TopKWithFractionalIndexOptions,\n  ) {\n    super(id, inputA, output)\n    const limit = options.limit ?? Infinity\n    const offset = options.offset ?? 0\n    const topK = this.createTopK(\n      offset,\n      limit,\n      createKeyedComparator(comparator),\n    )\n    this.#state = new TopKState(topK)\n    options.setSizeCallback?.(() => this.#state.size)\n    options.setWindowFn?.(this.moveTopK.bind(this))\n  }\n\n  protected createTopK(\n    offset: number,\n    limit: number,\n    comparator: (a: [K, T], b: [K, T]) => number,\n  ): TopK<[K, T]> {\n    return new TopKArray(offset, limit, comparator)\n  }\n\n  /**\n   * Moves the topK window based on the provided offset and limit.\n   * Any changes to the topK are sent to the output.\n   */\n  moveTopK({ offset, limit }: { offset?: number; limit?: number }) {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    const diff = this.#state.move({ offset, limit })\n\n    diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))\n    diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))\n\n    if (diff.changes) {\n      // There are changes to the topK\n      // it could be that moveIns and moveOuts are empty\n      // because the collection is lazy, so we will run the graph again to load the data\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  run(): void {\n    const result: Array<[[K, IndexedValue<T>], number]> = []\n    for (const [[key, value], multiplicity] of topKBatch(\n      this.inputMessages(),\n    )) {\n      this.processElement(key, value, multiplicity, result)\n    }\n\n    if (result.length > 0) {\n      this.output.sendData(new MultiSet(result))\n    }\n  }\n\n  processElement(\n    key: K,\n    value: T,\n    multiplicity: number,\n    result: Array<[[K, IndexedValue<T>], number]>,\n  ): void {\n    const changes = this.#state.processElement(key, value, multiplicity)\n    handleMoveIn(changes.moveIn, result)\n    handleMoveOut(changes.moveOut, result)\n  }\n}\n\n/**\n * Limits the number of results based on a comparator, with optional offset.\n * Uses fractional indexing to minimize the number of changes when elements move positions.\n * Each element is assigned a fractional index that is lexicographically sortable.\n * When elements move, only the indices of the moved elements are updated, not all elements.\n *\n * @param comparator - A function that compares two elements\n * @param options - An optional object containing limit and offset properties\n * @returns A piped operator that orders the elements and limits the number of results\n */\nexport function topKWithFractionalIndex<KType extends string | number, T>(\n  comparator: (a: T, b: T) => number,\n  options?: TopKWithFractionalIndexOptions,\n): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {\n  const opts = options || {}\n\n  return (\n    stream: IStreamBuilder<[KType, T]>,\n  ): IStreamBuilder<[KType, IndexedValue<T>]> => {\n    const output = new StreamBuilder<[KType, IndexedValue<T>]>(\n      stream.graph,\n      new DifferenceStreamWriter<[KType, IndexedValue<T>]>(),\n    )\n    const operator = new TopKWithFractionalIndexOperator<KType, T>(\n      stream.graph.getNextOperatorId(),\n      stream.connectReader(),\n      output.writer,\n      comparator,\n      opts,\n    )\n    stream.graph.addOperator(operator)\n    return output\n  }\n}\n"],"names":[],"mappings":";;;;;AA4BO,MAAM,wCAGH,cAA4C;AAAA,EACpD;AAAA,EAEA,YACE,IACA,QACA,QACA,YACA,SACA;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,UAAM,QAAQ,QAAQ,SAAS;AAC/B,UAAM,SAAS,QAAQ,UAAU;AACjC,UAAM,OAAO,KAAK;AAAA,MAChB;AAAA,MACA;AAAA,MACA,sBAAsB,UAAU;AAAA,IAAA;AAElC,SAAK,SAAS,IAAI,UAAU,IAAI;AAChC,YAAQ,kBAAkB,MAAM,KAAK,OAAO,IAAI;AAChD,YAAQ,cAAc,KAAK,SAAS,KAAK,IAAI,CAAC;AAAA,EAChD;AAAA,EAEU,WACR,QACA,OACA,YACc;AACd,WAAO,IAAI,UAAU,QAAQ,OAAO,UAAU;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,EAAE,QAAQ,SAA8C;AAC/D,UAAM,SAAgD,CAAA;AACtD,UAAM,OAAO,KAAK,OAAO,KAAK,EAAE,QAAQ,OAAO;AAE/C,SAAK,QAAQ,QAAQ,CAAC,WAAW,aAAa,QAAQ,MAAM,CAAC;AAC7D,SAAK,SAAS,QAAQ,CAAC,YAAY,cAAc,SAAS,MAAM,CAAC;AAEjE,QAAI,KAAK,SAAS;AAIhB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,MAAY;AACV,UAAM,SAAgD,CAAA;AACtD,eAAW,CAAC,CAAC,KAAK,KAAK,GAAG,YAAY,KAAK;AAAA,MACzC,KAAK,cAAA;AAAA,IAAc,GAClB;AACD,WAAK,eAAe,KAAK,OAAO,cAAc,MAAM;AAAA,IACtD;AAEA,QAAI,OAAO,SAAS,GAAG;AACrB,WAAK,OAAO,SAAS,IAAI,SAAS,MAAM,CAAC;AAAA,IAC3C;AAAA,EACF;AAAA,EAEA,eACE,KACA,OACA,cACA,QACM;AACN,UAAM,UAAU,KAAK,OAAO,eAAe,KAAK,OAAO,YAAY;AACnE,iBAAa,QAAQ,QAAQ,MAAM;AACnC,kBAAc,QAAQ,SAAS,MAAM;AAAA,EACvC;AACF;AAYO,SAAS,wBACd,YACA,SACqD;AACrD,QAAM,OAAO,WAAW,CAAA;AAExB,SAAO,CACL,WAC6C;AAC7C,UAAM,SAAS,IAAI;AAAA,MACjB,OAAO;AAAA,MACP,IAAI,uBAAA;AAAA,IAAiD;AAEvD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/db-ivm",
-  "version": "0.1.20",
+  "version": "0.1.22",
   "description": "Incremental View Maintenance for TanStack DB based on Differential Dataflow",
   "author": "Sam Willis",
   "license": "MIT",
@@ -57,6 +57,8 @@
     "dev": "vite build --watch",
     "lint": "eslint . --fix",
     "test": "vitest --run",
+    "test:incrementalization": "vitest --run tests/incrementalization-law.property.test.ts",
+    "typecheck": "tsc --noEmit",
     "bench:hash": "vitest bench --run tests/hash.bench.ts --coverage.enabled=false"
   }
 }