@tanstack/db-ivm 0.1.14 → 0.1.15

package/dist/esm/utils.d.ts

@@ -57,3 +57,9 @@ export declare function diffHalfOpen(a: HRange, b: HRange): {
  * Handles mixed types by ordering strings before numbers.
  */
 export declare function compareKeys(a: string | number, b: string | number): number;
+/**
+ * Serializes a value for use as a key, handling BigInt and Date values that JSON.stringify cannot handle.
+ * Uses JSON.stringify with a replacer function to convert BigInt values to strings and Date values to ISO strings.
+ * This is used for creating string keys in groupBy operations.
+ */
+export declare function serializeValue(value: unknown): string;

package/dist/esm/utils.js

@@ -111,6 +111,17 @@ function compareKeys(a, b) {
   }
   return typeof a === `string` ? -1 : 1;
 }
+function serializeValue(value) {
+  return JSON.stringify(value, (_, val) => {
+    if (typeof val === "bigint") {
+      return val.toString();
+    }
+    if (val instanceof Date) {
+      return val.toISOString();
+    }
+    return val;
+  });
+}
 export {
   DefaultMap,
   ObjectIdGenerator,
@@ -118,6 +129,7 @@ export {
   chunkedArrayPush,
   compareKeys,
   diffHalfOpen,
-  globalObjectIdGenerator
+  globalObjectIdGenerator,
+  serializeValue
 };
 //# sourceMappingURL=utils.js.map

package/dist/esm/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sources":["../../src/utils.ts"],"sourcesContent":["/**\n * Simple assertion function for runtime checks.\n * Throws an error if the condition is false.\n */\nexport function assert(\n  condition: unknown,\n  message?: string,\n): asserts condition {\n  if (!condition) {\n    throw new Error(message || `Assertion failed`)\n  }\n}\n\n/**\n * A map that returns a default value for keys that are not present.\n */\nexport class DefaultMap<K, V> extends Map<K, V> {\n  constructor(\n    private defaultValue: () => V,\n    entries?: Iterable<[K, V]>,\n  ) {\n    super(entries)\n  }\n\n  get(key: K): V {\n    if (!this.has(key)) {\n      // this.set(key, this.defaultValue())\n      return this.defaultValue()\n    }\n    return super.get(key)!\n  }\n\n  /**\n   * Update the value for a key using a function.\n   */\n  update(key: K, updater: (value: V) => V): V {\n    const value = this.get(key)\n    const newValue = updater(value)\n    this.set(key, newValue)\n    return newValue\n  }\n}\n\n// JS engines have various limits on how many args can be passed to a function\n// with a spread operator, so we need to split the operation into chunks\n// 32767 is the max for Chrome 14, all others are higher\n// TODO: investigate the performance of this and other approaches\nconst chunkSize = 30000\nexport function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {\n  if (other.length <= chunkSize) {\n    array.push(...other)\n  } else {\n    for (let i = 0; i < other.length; i += chunkSize) {\n      const chunk = other.slice(i, i + chunkSize)\n      array.push(...chunk)\n    }\n  }\n}\n\nexport function binarySearch<T>(\n  array: Array<T>,\n  value: T,\n  comparator: (a: T, b: T) => number,\n): number {\n  let low = 0\n  let high = array.length\n  while (low < high) {\n    const mid = Math.floor((low + high) / 2)\n    const comparison = comparator(array[mid]!, value)\n    if (comparison < 0) {\n      low = mid + 1\n    } else if (comparison > 0) {\n      high = mid\n    } else {\n      return mid\n    }\n  }\n  return low\n}\n\n/**\n * Utility for generating unique IDs for objects and values.\n * Uses WeakMap for object reference tracking and consistent hashing for primitives.\n */\nexport class ObjectIdGenerator {\n  private objectIds = new WeakMap<object, number>()\n  private nextId = 0\n\n  /**\n   * Get a unique identifier for any value.\n   * - Objects: Uses WeakMap for reference-based identity\n   * - Primitives: Uses consistent string-based hashing\n   */\n  getId(value: any): number {\n    // For primitives, use a simple hash of their string representation\n    if (typeof value !== `object` || value === null) {\n      const str = String(value)\n      let hashValue = 0\n      for (let i = 0; i < str.length; i++) {\n        const char = str.charCodeAt(i)\n        hashValue = (hashValue << 5) - hashValue + char\n        hashValue = hashValue & hashValue // Convert to 32-bit integer\n      }\n      return hashValue\n    }\n\n    // For objects, use WeakMap to assign unique IDs\n    if (!this.objectIds.has(value)) {\n      this.objectIds.set(value, this.nextId++)\n    }\n    return this.objectIds.get(value)!\n  }\n\n  /**\n   * Get a string representation of the ID for use in composite keys.\n   */\n  getStringId(value: any): string {\n    if (value === null) return `null`\n    if (value === undefined) return `undefined`\n    if (typeof value !== `object`) return `str_${String(value)}`\n\n    return `obj_${this.getId(value)}`\n  }\n}\n\n/**\n * Global instance for cases where a shared object ID space is needed.\n */\nexport const globalObjectIdGenerator = new ObjectIdGenerator()\n\nexport function* concatIterable<T>(\n  ...iterables: Array<Iterable<T>>\n): Iterable<T> {\n  for (const iterable of iterables) {\n    yield* iterable\n  }\n}\n\nexport function* mapIterable<T, U>(\n  it: Iterable<T>,\n  fn: (t: T) => U,\n): Iterable<U> {\n  for (const t of it) {\n    yield fn(t)\n  }\n}\n\nexport type HRange = [number, number] // half-open [start, end[ i.e. end is exclusive\n\n/**\n * Computes the difference between two half-open ranges.\n * @param a - The first half-open range\n * @param b - The second half-open range\n * @returns The difference between the two ranges\n */\nexport function diffHalfOpen(a: HRange, b: HRange) {\n  const [a1, a2] = a\n  const [b1, b2] = b\n\n  // A \\ B can be up to two segments (left and right of the overlap)\n  const onlyInA: Array<number> = [\n    ...range(a1, Math.min(a2, b1)), // left side of A outside B\n    ...range(Math.max(a1, b2), a2), // right side of A outside B\n  ]\n\n  // B \\ A similarly\n  const onlyInB: Array<number> = [\n    ...range(b1, Math.min(b2, a1)),\n    ...range(Math.max(b1, a2), b2),\n  ]\n\n  return { onlyInA, onlyInB }\n}\n\nfunction range(start: number, end: number): Array<number> {\n  const out: Array<number> = []\n  for (let i = start; i < end; i++) out.push(i)\n  return out\n}\n\n/**\n * Compares two keys (string | number) in a consistent, deterministic way.\n * Handles mixed types by ordering strings before numbers.\n */\nexport function compareKeys(a: string | number, b: string | number): number {\n  // Same type: compare directly\n  if (typeof a === typeof b) {\n    if (a < b) return -1\n    if (a > b) return 1\n    return 0\n  }\n  // Different types: strings come before numbers\n  return typeof a === `string` ? -1 : 1\n}\n"],"names":[],"mappings":"AAgBO,MAAM,mBAAyB,IAAU;AAAA,EAC9C,YACU,cACR,SACA;AACA,UAAM,OAAO;AAHL,SAAA,eAAA;AAAA,EAIV;AAAA,EAEA,IAAI,KAAW;AACb,QAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAElB,aAAO,KAAK,aAAA;AAAA,IACd;AACA,WAAO,MAAM,IAAI,GAAG;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,KAAQ,SAA6B;AAC1C,UAAM,QAAQ,KAAK,IAAI,GAAG;AAC1B,UAAM,WAAW,QAAQ,KAAK;AAC9B,SAAK,IAAI,KAAK,QAAQ;AACtB,WAAO;AAAA,EACT;AACF;AAMA,MAAM,YAAY;AACX,SAAS,iBAAiB,OAAuB,OAAuB;AAC7E,MAAI,MAAM,UAAU,WAAW;AAC7B,UAAM,KAAK,GAAG,KAAK;AAAA,EACrB,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,SAAS;AAC1C,YAAM,KAAK,GAAG,KAAK;AAAA,IACrB;AAAA,EACF;AACF;AAEO,SAAS,aACd,OACA,OACA,YACQ;AACR,MAAI,MAAM;AACV,MAAI,OAAO,MAAM;AACjB,SAAO,MAAM,MAAM;AACjB,UAAM,MAAM,KAAK,OAAO,MAAM,QAAQ,CAAC;AACvC,UAAM,aAAa,WAAW,MAAM,GAAG,GAAI,KAAK;AAChD,QAAI,aAAa,GAAG;AAClB,YAAM,MAAM;AAAA,IACd,WAAW,aAAa,GAAG;AACzB,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAMO,MAAM,kBAAkB;AAAA,EAAxB,cAAA;AACL,SAAQ,gCAAgB,QAAA;AACxB,SAAQ,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOjB,MAAM,OAAoB;AAExB,QAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,YAAM,MAAM,OAAO,KAAK;AACxB,UAAI,YAAY;AAChB,eAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,cAAM,OAAO,IAAI,WAAW,CAAC;AAC7B,qBAAa,aAAa,KAAK,YAAY;AAC3C,oBAAY,YAAY;AAAA,MAC1B;AACA,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,UAAU,IAAI,KAAK,GAAG;AAC9B,WAAK,UAAU,IAAI,OAAO,KAAK,QAAQ;AAAA,IACzC;AACA,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,OAAoB;AAC9B,QAAI,UAAU,KAAM,QAAO;AAC3B,QAAI,UAAU,OAAW,QAAO;AAChC,QAAI,OAAO,UAAU,iBAAiB,OAAO,OAAO,KAAK,CAAC;AAE1D,WAAO,OAAO,KAAK,MAAM,KAAK,CAAC;AAAA,EACjC;AACF;AAKO,MAAM,0BAA0B,IAAI,kBAAA;AA2BpC,SAAS,aAAa,GAAW,GAAW;AACjD,QAAM,CAAC,IAAI,EAAE,IAAI;AACjB,QAAM,CAAC,IAAI,EAAE,IAAI;AAGjB,QAAM,UAAyB;AAAA,IAC7B,GAAG,MAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAAA;AAAA,IAC7B,GAAG,MAAM,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA;AAAA,EAAA;AAI/B,QAAM,UAAyB;AAAA,IAC7B,GAAG,MAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAAA,IAC7B,GAAG,MAAM,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA,EAAA;AAG/B,SAAO,EAAE,SAAS,QAAA;AACpB;AAEA,SAAS,MAAM,OAAe,KAA4B;AACxD,QAAM,MAAqB,CAAA;AAC3B,WAAS,IAAI,OAAO,IAAI,KAAK,IAAK,KAAI,KAAK,CAAC;AAC5C,SAAO;AACT;AAMO,SAAS,YAAY,GAAoB,GAA4B;AAE1E,MAAI,OAAO,MAAM,OAAO,GAAG;AACzB,QAAI,IAAI,EAAG,QAAO;AAClB,QAAI,IAAI,EAAG,QAAO;AAClB,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,MAAM,WAAW,KAAK;AACtC;"}
+{"version":3,"file":"utils.js","sources":["../../src/utils.ts"],"sourcesContent":["/**\n * Simple assertion function for runtime checks.\n * Throws an error if the condition is false.\n */\nexport function assert(\n  condition: unknown,\n  message?: string,\n): asserts condition {\n  if (!condition) {\n    throw new Error(message || `Assertion failed`)\n  }\n}\n\n/**\n * A map that returns a default value for keys that are not present.\n */\nexport class DefaultMap<K, V> extends Map<K, V> {\n  constructor(\n    private defaultValue: () => V,\n    entries?: Iterable<[K, V]>,\n  ) {\n    super(entries)\n  }\n\n  get(key: K): V {\n    if (!this.has(key)) {\n      // this.set(key, this.defaultValue())\n      return this.defaultValue()\n    }\n    return super.get(key)!\n  }\n\n  /**\n   * Update the value for a key using a function.\n   */\n  update(key: K, updater: (value: V) => V): V {\n    const value = this.get(key)\n    const newValue = updater(value)\n    this.set(key, newValue)\n    return newValue\n  }\n}\n\n// JS engines have various limits on how many args can be passed to a function\n// with a spread operator, so we need to split the operation into chunks\n// 32767 is the max for Chrome 14, all others are higher\n// TODO: investigate the performance of this and other approaches\nconst chunkSize = 30000\nexport function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {\n  if (other.length <= chunkSize) {\n    array.push(...other)\n  } else {\n    for (let i = 0; i < other.length; i += chunkSize) {\n      const chunk = other.slice(i, i + chunkSize)\n      array.push(...chunk)\n    }\n  }\n}\n\nexport function binarySearch<T>(\n  array: Array<T>,\n  value: T,\n  comparator: (a: T, b: T) => number,\n): number {\n  let low = 0\n  let high = array.length\n  while (low < high) {\n    const mid = Math.floor((low + high) / 2)\n    const comparison = comparator(array[mid]!, value)\n    if (comparison < 0) {\n      low = mid + 1\n    } else if (comparison > 0) {\n      high = mid\n    } else {\n      return mid\n    }\n  }\n  return low\n}\n\n/**\n * Utility for generating unique IDs for objects and values.\n * Uses WeakMap for object reference tracking and consistent hashing for primitives.\n */\nexport class ObjectIdGenerator {\n  private objectIds = new WeakMap<object, number>()\n  private nextId = 0\n\n  /**\n   * Get a unique identifier for any value.\n   * - Objects: Uses WeakMap for reference-based identity\n   * - Primitives: Uses consistent string-based hashing\n   */\n  getId(value: any): number {\n    // For primitives, use a simple hash of their string representation\n    if (typeof value !== `object` || value === null) {\n      const str = String(value)\n      let hashValue = 0\n      for (let i = 0; i < str.length; i++) {\n        const char = str.charCodeAt(i)\n        hashValue = (hashValue << 5) - hashValue + char\n        hashValue = hashValue & hashValue // Convert to 32-bit integer\n      }\n      return hashValue\n    }\n\n    // For objects, use WeakMap to assign unique IDs\n    if (!this.objectIds.has(value)) {\n      this.objectIds.set(value, this.nextId++)\n    }\n    return this.objectIds.get(value)!\n  }\n\n  /**\n   * Get a string representation of the ID for use in composite keys.\n   */\n  getStringId(value: any): string {\n    if (value === null) return `null`\n    if (value === undefined) return `undefined`\n    if (typeof value !== `object`) return `str_${String(value)}`\n\n    return `obj_${this.getId(value)}`\n  }\n}\n\n/**\n * Global instance for cases where a shared object ID space is needed.\n */\nexport const globalObjectIdGenerator = new ObjectIdGenerator()\n\nexport function* concatIterable<T>(\n  ...iterables: Array<Iterable<T>>\n): Iterable<T> {\n  for (const iterable of iterables) {\n    yield* iterable\n  }\n}\n\nexport function* mapIterable<T, U>(\n  it: Iterable<T>,\n  fn: (t: T) => U,\n): Iterable<U> {\n  for (const t of it) {\n    yield fn(t)\n  }\n}\n\nexport type HRange = [number, number] // half-open [start, end[ i.e. end is exclusive\n\n/**\n * Computes the difference between two half-open ranges.\n * @param a - The first half-open range\n * @param b - The second half-open range\n * @returns The difference between the two ranges\n */\nexport function diffHalfOpen(a: HRange, b: HRange) {\n  const [a1, a2] = a\n  const [b1, b2] = b\n\n  // A \\ B can be up to two segments (left and right of the overlap)\n  const onlyInA: Array<number> = [\n    ...range(a1, Math.min(a2, b1)), // left side of A outside B\n    ...range(Math.max(a1, b2), a2), // right side of A outside B\n  ]\n\n  // B \\ A similarly\n  const onlyInB: Array<number> = [\n    ...range(b1, Math.min(b2, a1)),\n    ...range(Math.max(b1, a2), b2),\n  ]\n\n  return { onlyInA, onlyInB }\n}\n\nfunction range(start: number, end: number): Array<number> {\n  const out: Array<number> = []\n  for (let i = start; i < end; i++) out.push(i)\n  return out\n}\n\n/**\n * Compares two keys (string | number) in a consistent, deterministic way.\n * Handles mixed types by ordering strings before numbers.\n */\nexport function compareKeys(a: string | number, b: string | number): number {\n  // Same type: compare directly\n  if (typeof a === typeof b) {\n    if (a < b) return -1\n    if (a > b) return 1\n    return 0\n  }\n  // Different types: strings come before numbers\n  return typeof a === `string` ? -1 : 1\n}\n\n/**\n * Serializes a value for use as a key, handling BigInt and Date values that JSON.stringify cannot handle.\n * Uses JSON.stringify with a replacer function to convert BigInt values to strings and Date values to ISO strings.\n * This is used for creating string keys in groupBy operations.\n */\nexport function serializeValue(value: unknown): string {\n  return JSON.stringify(value, (_, val) => {\n    if (typeof val === 'bigint') {\n      return val.toString()\n    }\n    if (val instanceof Date) {\n      return val.toISOString()\n    }\n    return val\n  })\n}\n"],"names":[],"mappings":"AAgBO,MAAM,mBAAyB,IAAU;AAAA,EAC9C,YACU,cACR,SACA;AACA,UAAM,OAAO;AAHL,SAAA,eAAA;AAAA,EAIV;AAAA,EAEA,IAAI,KAAW;AACb,QAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAElB,aAAO,KAAK,aAAA;AAAA,IACd;AACA,WAAO,MAAM,IAAI,GAAG;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA,EAKA,OAAO,KAAQ,SAA6B;AAC1C,UAAM,QAAQ,KAAK,IAAI,GAAG;AAC1B,UAAM,WAAW,QAAQ,KAAK;AAC9B,SAAK,IAAI,KAAK,QAAQ;AACtB,WAAO;AAAA,EACT;AACF;AAMA,MAAM,YAAY;AACX,SAAS,iBAAiB,OAAuB,OAAuB;AAC7E,MAAI,MAAM,UAAU,WAAW;AAC7B,UAAM,KAAK,GAAG,KAAK;AAAA,EACrB,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,YAAM,QAAQ,MAAM,MAAM,GAAG,IAAI,SAAS;AAC1C,YAAM,KAAK,GAAG,KAAK;AAAA,IACrB;AAAA,EACF;AACF;AAEO,SAAS,aACd,OACA,OACA,YACQ;AACR,MAAI,MAAM;AACV,MAAI,OAAO,MAAM;AACjB,SAAO,MAAM,MAAM;AACjB,UAAM,MAAM,KAAK,OAAO,MAAM,QAAQ,CAAC;AACvC,UAAM,aAAa,WAAW,MAAM,GAAG,GAAI,KAAK;AAChD,QAAI,aAAa,GAAG;AAClB,YAAM,MAAM;AAAA,IACd,WAAW,aAAa,GAAG;AACzB,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAMO,MAAM,kBAAkB;AAAA,EAAxB,cAAA;AACL,SAAQ,gCAAgB,QAAA;AACxB,SAAQ,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOjB,MAAM,OAAoB;AAExB,QAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,YAAM,MAAM,OAAO,KAAK;AACxB,UAAI,YAAY;AAChB,eAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,cAAM,OAAO,IAAI,WAAW,CAAC;AAC7B,qBAAa,aAAa,KAAK,YAAY;AAC3C,oBAAY,YAAY;AAAA,MAC1B;AACA,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,UAAU,IAAI,KAAK,GAAG;AAC9B,WAAK,UAAU,IAAI,OAAO,KAAK,QAAQ;AAAA,IACzC;AACA,WAAO,KAAK,UAAU,IAAI,KAAK;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,OAAoB;AAC9B,QAAI,UAAU,KAAM,QAAO;AAC3B,QAAI,UAAU,OAAW,QAAO;AAChC,QAAI,OAAO,UAAU,iBAAiB,OAAO,OAAO,KAAK,CAAC;AAE1D,WAAO,OAAO,KAAK,MAAM,KAAK,CAAC;AAAA,EACjC;AACF;AAKO,MAAM,0BAA0B,IAAI,kBAAA;AA2BpC,SAAS,aAAa,GAAW,GAAW;AACjD,QAAM,CAAC,IAAI,EAAE,IAAI;AACjB,QAAM,CAAC,IAAI,EAAE,IAAI;AAGjB,QAAM,UAAyB;AAAA,IAC7B,GAAG,MAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAAA;AAAA,IAC7B,GAAG,MAAM,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA;AAAA,EAAA;AAI/B,QAAM,UAAyB;AAAA,IAC7B,GAAG,MAAM,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;AAAA,IAC7B,GAAG,MAAM,KAAK,IAAI,IAAI,EAAE,GAAG,EAAE;AAAA,EAAA;AAG/B,SAAO,EAAE,SAAS,QAAA;AACpB;AAEA,SAAS,MAAM,OAAe,KAA4B;AACxD,QAAM,MAAqB,CAAA;AAC3B,WAAS,IAAI,OAAO,IAAI,KAAK,IAAK,KAAI,KAAK,CAAC;AAC5C,SAAO;AACT;AAMO,SAAS,YAAY,GAAoB,GAA4B;AAE1E,MAAI,OAAO,MAAM,OAAO,GAAG;AACzB,QAAI,IAAI,EAAG,QAAO;AAClB,QAAI,IAAI,EAAG,QAAO;AAClB,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,MAAM,WAAW,KAAK;AACtC;AAOO,SAAS,eAAe,OAAwB;AACrD,SAAO,KAAK,UAAU,OAAO,CAAC,GAAG,QAAQ;AACvC,QAAI,OAAO,QAAQ,UAAU;AAC3B,aAAO,IAAI,SAAA;AAAA,IACb;AACA,QAAI,eAAe,MAAM;AACvB,aAAO,IAAI,YAAA;AAAA,IACb;AACA,WAAO;AAAA,EACT,CAAC;AACH;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/db-ivm",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "description": "Incremental View Maintenance for TanStack DB based on Differential Dataflow",
   "author": "Sam Willis",
   "license": "MIT",

package/src/index.ts

@@ -2,4 +2,4 @@ export * from './d2.js'
 export * from './multiset.js'
 export * from './operators/index.js'
 export * from './types.js'
-export { compareKeys } from './utils.js'
+export { compareKeys, serializeValue } from './utils.js'

package/src/operators/groupBy.ts

@@ -1,3 +1,4 @@
+import { serializeValue } from '../utils.js'
 import { map } from './map.js'
 import { reduce } from './reduce.js'
 import type { IStreamBuilder, KeyValue } from '../types.js'
@@ -67,7 +68,7 @@ export function groupBy<
     const withKeysAndValues = stream.pipe(
       map((data) => {
         const key = keyExtractor(data)
-        const keyString = JSON.stringify(key)
+        const keyString = serializeValue(key)
 
         // Create values object with pre-aggregated values
         const values: Record<string, unknown> = {}

package/src/operators/groupedTopKWithFractionalIndex.ts

@@ -0,0 +1,192 @@
+import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
+import { StreamBuilder } from '../d2.js'
+import { MultiSet } from '../multiset.js'
+import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
+import { TopKArray, createKeyedComparator } from './topKArray.js'
+import type { IndexedValue, TopK } from './topKArray.js'
+import type { DifferenceStreamReader } from '../graph.js'
+import type { IStreamBuilder, PipedOperator } from '../types.js'
+
+export interface GroupedTopKWithFractionalIndexOptions<K, T> {
+  limit?: number
+  offset?: number
+  setSizeCallback?: (getSize: () => number) => void
+  setWindowFn?: (
+    windowFn: (options: { offset?: number; limit?: number }) => void,
+  ) => void
+  /**
+   * Function to extract a group key from the element's key and value.
+   * Elements with the same group key will be sorted and limited together.
+   */
+  groupKeyFn: (key: K, value: T) => unknown
+}
+
+/**
+ * Operator for grouped fractional indexed topK operations.
+ * This operator maintains separate topK windows for each group,
+ * allowing per-group limits and ordering.
+ *
+ * The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].
+ * Elements are grouped by the groupKeyFn, and each group maintains
+ * its own sorted collection with independent limit/offset.
+ */
+export class GroupedTopKWithFractionalIndexOperator<
+  K extends string | number,
+  T,
+> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {
+  #groupStates: Map<unknown, TopKState<K, T>> = new Map()
+  #groupKeyFn: (key: K, value: T) => unknown
+  #comparator: (a: [K, T], b: [K, T]) => number
+  #offset: number
+  #limit: number
+
+  constructor(
+    id: number,
+    inputA: DifferenceStreamReader<[K, T]>,
+    output: DifferenceStreamWriter<[K, IndexedValue<T>]>,
+    comparator: (a: T, b: T) => number,
+    options: GroupedTopKWithFractionalIndexOptions<K, T>,
+  ) {
+    super(id, inputA, output)
+    this.#groupKeyFn = options.groupKeyFn
+    this.#limit = options.limit ?? Infinity
+    this.#offset = options.offset ?? 0
+    this.#comparator = createKeyedComparator(comparator)
+    options.setSizeCallback?.(() => this.#getTotalSize())
+    options.setWindowFn?.(this.#moveTopK.bind(this))
+  }
+
+  /**
+   * Creates a new TopK data structure for a group.
+   * Can be overridden in subclasses to use different implementations (e.g., B+ tree).
+   */
+  protected createTopK(
+    offset: number,
+    limit: number,
+    comparator: (a: [K, T], b: [K, T]) => number,
+  ): TopK<[K, T]> {
+    return new TopKArray(offset, limit, comparator)
+  }
+
+  #getTotalSize(): number {
+    let size = 0
+    for (const state of this.#groupStates.values()) {
+      size += state.size
+    }
+    return size
+  }
+
+  #getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {
+    let state = this.#groupStates.get(groupKey)
+    if (!state) {
+      const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)
+      state = new TopKState(topK)
+      this.#groupStates.set(groupKey, state)
+    }
+    return state
+  }
+
+  #cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {
+    if (state.isEmpty) {
+      this.#groupStates.delete(groupKey)
+    }
+  }
+
+  /**
+   * Moves the topK window for all groups based on the provided offset and limit.
+   * Any changes to the topK are sent to the output.
+   */
+  #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {
+    if (offset !== undefined) {
+      this.#offset = offset
+    }
+    if (limit !== undefined) {
+      this.#limit = limit
+    }
+
+    const result: Array<[[K, IndexedValue<T>], number]> = []
+    let hasChanges = false
+
+    for (const state of this.#groupStates.values()) {
+      const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit
+
+      diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))
+      diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))
+
+      if (diff.changes) {
+        hasChanges = true
+      }
+    }
+
+    if (hasChanges) {
+      this.output.sendData(new MultiSet(result))
+    }
+  }
+
+  run(): void {
+    const result: Array<[[K, IndexedValue<T>], number]> = []
+    for (const message of this.inputMessages()) {
+      for (const [item, multiplicity] of message.getInner()) {
+        const [key, value] = item
+        this.#processElement(key, value, multiplicity, result)
+      }
+    }
+
+    if (result.length > 0) {
+      this.output.sendData(new MultiSet(result))
+    }
+  }
+
+  #processElement(
+    key: K,
+    value: T,
+    multiplicity: number,
+    result: Array<[[K, IndexedValue<T>], number]>,
+  ): void {
+    const groupKey = this.#groupKeyFn(key, value)
+    const state = this.#getOrCreateGroupState(groupKey)
+
+    const changes = state.processElement(key, value, multiplicity)
+    handleMoveIn(changes.moveIn, result)
+    handleMoveOut(changes.moveOut, result)
+
+    // Cleanup empty groups to prevent memory leaks
+    this.#cleanupGroupIfEmpty(groupKey, state)
+  }
+}
+
+/**
+ * Limits the number of results per group based on a comparator, with optional offset.
+ * Uses fractional indexing to minimize the number of changes when elements move positions.
+ * Each element is assigned a fractional index that is lexicographically sortable.
+ * When elements move, only the indices of the moved elements are updated, not all elements.
+ *
+ * This operator groups elements by the provided groupKeyFn and applies the limit/offset
+ * independently to each group.
+ *
+ * @param comparator - A function that compares two elements for ordering
+ * @param options - Configuration including groupKeyFn, limit, and offset
+ * @returns A piped operator that orders elements per group and limits results per group
+ */
+export function groupedTopKWithFractionalIndex<K extends string | number, T>(
+  comparator: (a: T, b: T) => number,
+  options: GroupedTopKWithFractionalIndexOptions<K, T>,
+): PipedOperator<[K, T], [K, IndexedValue<T>]> {
+  return (
+    stream: IStreamBuilder<[K, T]>,
+  ): IStreamBuilder<[K, IndexedValue<T>]> => {
+    const output = new StreamBuilder<[K, IndexedValue<T>]>(
+      stream.graph,
+      new DifferenceStreamWriter<[K, IndexedValue<T>]>(),
+    )
+    const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer,
+      comparator,
+      options,
+    )
+    stream.graph.addOperator(operator)
+    return output
+  }
+}

package/src/operators/index.ts

@@ -14,6 +14,7 @@ export * from './distinct.js'
 export * from './keying.js'
 export * from './topK.js'
 export * from './topKWithFractionalIndex.js'
+export * from './groupedTopKWithFractionalIndex.js'
 export * from './orderBy.js'
 export * from './filterBy.js'
 export { groupBy, groupByOperators } from './groupBy.js'

package/src/operators/topKArray.ts

@@ -0,0 +1,255 @@
+import { generateKeyBetween } from 'fractional-indexing'
+import { binarySearch, compareKeys, diffHalfOpen } from '../utils.js'
+import type { HRange } from '../utils.js'
+
+// Abstraction for fractionally indexed values
+export type FractionalIndex = string
+export type IndexedValue<V> = [V, FractionalIndex]
+
+export function indexedValue<V>(
+  value: V,
+  index: FractionalIndex,
+): IndexedValue<V> {
+  return [value, index]
+}
+
+export function getValue<V>(indexedVal: IndexedValue<V>): V {
+  return indexedVal[0]
+}
+
+export function getIndex<V>(indexedVal: IndexedValue<V>): FractionalIndex {
+  return indexedVal[1]
+}
+
+/**
+ * Creates a comparator for [key, value] tuples that first compares values,
+ * then uses the row key as a stable tie-breaker.
+ */
+export function createKeyedComparator<K extends string | number, T>(
+  comparator: (a: T, b: T) => number,
+): (a: [K, T], b: [K, T]) => number {
+  return ([aKey, aVal], [bKey, bVal]) => {
+    // First compare on the value
+    const valueComparison = comparator(aVal, bVal)
+    if (valueComparison !== 0) {
+      return valueComparison
+    }
+    // If the values are equal, use the row key as tie-breaker
+    // This provides stable, deterministic ordering since keys are string | number
+    return compareKeys(aKey, bKey)
+  }
+}
+
+export type TopKChanges<V> = {
+  /** Indicates which element moves into the topK (if any) */
+  moveIn: IndexedValue<V> | null
+  /** Indicates which element moves out of the topK (if any) */
+  moveOut: IndexedValue<V> | null
+}
+
+export type TopKMoveChanges<V> = {
+  /** Flag that marks whether there were any changes to the topK */
+  changes: boolean
+  /** Indicates which elements move into the topK (if any) */
+  moveIns: Array<IndexedValue<V>>
+  /** Indicates which elements move out of the topK (if any) */
+  moveOuts: Array<IndexedValue<V>>
+}
+
+/**
+ * A topK data structure that supports insertions and deletions
+ * and returns changes to the topK.
+ */
+export interface TopK<V> {
+  size: number
+  insert: (value: V) => TopKChanges<V>
+  delete: (value: V) => TopKChanges<V>
+}
+
+/**
+ * Implementation of a topK data structure.
+ * Uses a sorted array internally to store the values and keeps a topK window over that array.
+ * Inserts and deletes are O(n) operations because worst case an element is inserted/deleted
+ * at the start of the array which causes all the elements to shift to the right/left.
+ */
+export class TopKArray<V> implements TopK<V> {
+  #sortedValues: Array<IndexedValue<V>> = []
+  #comparator: (a: V, b: V) => number
+  #topKStart: number
+  #topKEnd: number
+
+  constructor(
+    offset: number,
+    limit: number,
+    comparator: (a: V, b: V) => number,
+  ) {
+    this.#topKStart = offset
+    this.#topKEnd = offset + limit
+    this.#comparator = comparator
+  }
+
+  get size(): number {
+    const offset = this.#topKStart
+    const limit = this.#topKEnd - this.#topKStart
+    const available = this.#sortedValues.length - offset
+    return Math.max(0, Math.min(limit, available))
+  }
+
+  /**
+   * Moves the topK window
+   */
+  move({
+    offset,
+    limit,
+  }: {
+    offset?: number
+    limit?: number
+  }): TopKMoveChanges<V> {
+    const oldOffset = this.#topKStart
+    const oldLimit = this.#topKEnd - this.#topKStart
+
+    // `this.#topKEnd` can be `Infinity` if it has no limit
+    // but `diffHalfOpen` expects a finite range
+    // so we restrict it to the size of the topK if topKEnd is infinite
+    const oldRange: HRange = [
+      this.#topKStart,
+      this.#topKEnd === Infinity ? this.#topKStart + this.size : this.#topKEnd,
+    ]
+
+    this.#topKStart = offset ?? oldOffset
+    this.#topKEnd = this.#topKStart + (limit ?? oldLimit) // can be `Infinity` if limit is `Infinity`
+
+    // Also handle `Infinity` in the newRange
+    const newRange: HRange = [
+      this.#topKStart,
+      this.#topKEnd === Infinity
+        ? Math.max(this.#topKStart + this.size, oldRange[1]) // since the new limit is Infinity we need to take everything (so we need to take the biggest (finite) topKEnd)
+        : this.#topKEnd,
+    ]
+    const { onlyInA, onlyInB } = diffHalfOpen(oldRange, newRange)
+
+    const moveIns: Array<IndexedValue<V>> = []
+    onlyInB.forEach((index) => {
+      const value = this.#sortedValues[index]
+      if (value) {
+        moveIns.push(value)
+      }
+    })
+
+    const moveOuts: Array<IndexedValue<V>> = []
+    onlyInA.forEach((index) => {
+      const value = this.#sortedValues[index]
+      if (value) {
+        moveOuts.push(value)
+      }
+    })
+
+    // It could be that there are changes (i.e. moveIns or moveOuts)
+    // but that the collection is lazy so we don't have the data yet that needs to move in/out
+    // so `moveIns` and `moveOuts` will be empty but `changes` will be true
+    // this will tell the caller that it needs to run the graph to load more data
+    return { moveIns, moveOuts, changes: onlyInA.length + onlyInB.length > 0 }
+  }
+
+  insert(value: V): TopKChanges<V> {
+    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
+
+    // Lookup insert position
+    const index = this.#findIndex(value)
+    // Generate fractional index based on the fractional indices of the elements before and after it
+    const indexBefore =
+      index === 0 ? null : getIndex(this.#sortedValues[index - 1]!)
+    const indexAfter =
+      index === this.#sortedValues.length
+        ? null
+        : getIndex(this.#sortedValues[index]!)
+    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)
+
+    // Insert the value at the correct position
+    const val = indexedValue(value, fractionalIndex)
+    // Splice is O(n) where n = all elements in the collection (i.e. n >= k) !
+    this.#sortedValues.splice(index, 0, val)
+
+    // Check if the topK changed
+    if (index < this.#topKEnd) {
+      // The inserted element is either before the top K or within the top K
+      // If it is before the top K then it moves the element that was right before the topK into the topK
+      // If it is within the top K then the inserted element moves into the top K
+      // In both cases the last element of the old top K now moves out of the top K
+      const moveInIndex = Math.max(index, this.#topKStart)
+      if (moveInIndex < this.#sortedValues.length) {
+        // We actually have a topK
+        // because in some cases there may not be enough elements in the array to reach the start of the topK
+        // e.g. [1, 2, 3] with K = 2 and offset = 3 does not have a topK
+        result.moveIn = this.#sortedValues[moveInIndex]!
+
+        // We need to remove the element that falls out of the top K
+        // The element that falls out of the top K has shifted one to the right
+        // because of the element we inserted, so we find it at index topKEnd
+        if (this.#topKEnd < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[this.#topKEnd]!
+        }
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Deletes a value that may or may not be in the topK.
+   * IMPORTANT: this assumes that the value is present in the collection
+   *            if it's not the case it will remove the element
+   *            that is on the position where the provided `value` would be.
+   */
+  delete(value: V): TopKChanges<V> {
+    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
+
+    // Lookup delete position
+    const index = this.#findIndex(value)
+    // Remove the value at that position
+    const [removedElem] = this.#sortedValues.splice(index, 1)
+
+    // Check if the topK changed
+    if (index < this.#topKEnd) {
+      // The removed element is either before the top K or within the top K
+      // If it is before the top K then the first element of the topK moves out of the topK
+      // If it is within the top K then the removed element moves out of the topK
+      result.moveOut = removedElem!
+      if (index < this.#topKStart) {
+        // The removed element is before the topK
+        // so actually, the first element of the topK moves out of the topK
+        // and not the element that we removed
+        // The first element of the topK is now at index topKStart - 1
+        // since we removed an element before the topK
+        const moveOutIndex = this.#topKStart - 1
+        if (moveOutIndex < this.#sortedValues.length) {
+          result.moveOut = this.#sortedValues[moveOutIndex]!
+        } else {
+          // No value is moving out of the topK
+          // because there are no elements in the topK
+          result.moveOut = null
+        }
+      }
+
+      // Since we removed an element that was before or in the topK
+      // the first element after the topK moved one position to the left
+      // and thus falls into the topK now
+      const moveInIndex = this.#topKEnd - 1
+      if (moveInIndex < this.#sortedValues.length) {
+        result.moveIn = this.#sortedValues[moveInIndex]!
+      }
+    }
+
+    return result
+  }
+
+  // TODO: see if there is a way to refactor the code for insert and delete in the topK above
+  //       because they are very similar, one is shifting the topK window to the left and the other is shifting it to the right
+  //       so i have the feeling there is a common pattern here and we can implement both cases using that pattern
+
+  #findIndex(value: V): number {
+    return binarySearch(this.#sortedValues, indexedValue(value, ``), (a, b) =>
+      this.#comparator(getValue(a), getValue(b)),
+    )
+  }
+}

package/src/operators/topKState.ts

@@ -0,0 +1,111 @@
+import { TopKArray } from './topKArray.js'
+import type {
+  IndexedValue,
+  TopK,
+  TopKChanges,
+  TopKMoveChanges,
+} from './topKArray.js'
+
+/**
+ * Helper class that manages the state for a single topK window.
+ * Encapsulates the multiplicity tracking and topK data structure,
+ * providing a clean interface for processing elements and moving the window.
+ *
+ * This class is used by both TopKWithFractionalIndexOperator (single instance)
+ * and GroupedTopKWithFractionalIndexOperator (one instance per group).
+ */
+export class TopKState<K extends string | number, T> {
+  #multiplicities: Map<K, number> = new Map()
+  #topK: TopK<[K, T]>
+
+  constructor(topK: TopK<[K, T]>) {
+    this.#topK = topK
+  }
+
+  get size(): number {
+    return this.#topK.size
+  }
+
+  get isEmpty(): boolean {
+    return this.#multiplicities.size === 0 && this.#topK.size === 0
+  }
+
+  /**
+   * Process an element update (insert or delete based on multiplicity change).
+   * Returns the changes to the topK window.
+   */
+  processElement(key: K, value: T, multiplicity: number): TopKChanges<[K, T]> {
+    const { oldMultiplicity, newMultiplicity } = this.#updateMultiplicity(
+      key,
+      multiplicity,
+    )
+
+    if (oldMultiplicity <= 0 && newMultiplicity > 0) {
+      // The value was invisible but should now be visible
+      return this.#topK.insert([key, value])
+    } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
+      // The value was visible but should now be invisible
+      return this.#topK.delete([key, value])
+    }
+    // The value was invisible and remains invisible,
+    // or was visible and remains visible - no topK change
+    return { moveIn: null, moveOut: null }
+  }
+
+  /**
+   * Move the topK window. Only works with TopKArray implementation.
+   */
+  move(options: { offset?: number; limit?: number }): TopKMoveChanges<[K, T]> {
+    if (!(this.#topK instanceof TopKArray)) {
+      throw new Error(
+        `Cannot move B+-tree implementation of TopK with fractional index`,
+      )
+    }
+    return this.#topK.move(options)
+  }
+
+  #updateMultiplicity(
+    key: K,
+    multiplicity: number,
+  ): { oldMultiplicity: number; newMultiplicity: number } {
+    if (multiplicity === 0) {
+      const current = this.#multiplicities.get(key) ?? 0
+      return { oldMultiplicity: current, newMultiplicity: current }
+    }
+
+    const oldMultiplicity = this.#multiplicities.get(key) ?? 0
+    const newMultiplicity = oldMultiplicity + multiplicity
+    if (newMultiplicity === 0) {
+      this.#multiplicities.delete(key)
+    } else {
+      this.#multiplicities.set(key, newMultiplicity)
+    }
+    return { oldMultiplicity, newMultiplicity }
+  }
+}
+
+/**
+ * Handles a moveIn change by adding it to the result array.
+ */
+export function handleMoveIn<K extends string | number, T>(
+  moveIn: IndexedValue<[K, T]> | null,
+  result: Array<[[K, IndexedValue<T>], number]>,
+): void {
+  if (moveIn) {
+    const [[key, value], index] = moveIn
+    result.push([[key, [value, index]], 1])
+  }
+}
+
+/**
+ * Handles a moveOut change by adding it to the result array.
+ */
+export function handleMoveOut<K extends string | number, T>(
+  moveOut: IndexedValue<[K, T]> | null,
+  result: Array<[[K, IndexedValue<T>], number]>,
+): void {
+  if (moveOut) {
+    const [[key, value], index] = moveOut
+    result.push([[key, [value, index]], -1])
+  }
+}