@tanstack/db-ivm 0.1.1 → 0.1.3

package/src/operators/orderBy.ts

@@ -12,6 +12,10 @@ export interface OrderByOptions<Ve> {
   offset?: number
 }
 
+type OrderByWithFractionalIndexOptions<Ve> = OrderByOptions<Ve> & {
+  setSizeCallback?: (getSize: () => number) => void
+}
+
 /**
  * Orders the elements and limits the number of results, with optional offset
  * This requires a keyed stream, and uses the `topK` operator to order all the elements.
@@ -136,13 +140,14 @@ export function orderByWithFractionalIndexBase<
   valueExtractor: (
     value: T extends KeyValue<unknown, infer V> ? V : never
   ) => Ve,
-  options?: OrderByOptions<Ve>
+  options?: OrderByWithFractionalIndexOptions<Ve>
 ) {
   type KeyType = T extends KeyValue<infer K, unknown> ? K : never
   type ValueType = T extends KeyValue<unknown, infer V> ? V : never
 
   const limit = options?.limit ?? Infinity
   const offset = options?.offset ?? 0
+  const setSizeCallback = options?.setSizeCallback
   const comparator =
     options?.comparator ??
     ((a, b) => {
@@ -162,6 +167,7 @@ export function orderByWithFractionalIndexBase<
         {
           limit,
           offset,
+          setSizeCallback,
         }
       ),
       consolidate()
@@ -185,7 +191,7 @@ export function orderByWithFractionalIndex<
   valueExtractor: (
     value: T extends KeyValue<unknown, infer V> ? V : never
   ) => Ve,
-  options?: OrderByOptions<Ve>
+  options?: OrderByWithFractionalIndexOptions<Ve>
 ) {
   return orderByWithFractionalIndexBase(
     topKWithFractionalIndex,

package/src/operators/tap.ts

@@ -0,0 +1,53 @@
+import { DifferenceStreamWriter, LinearUnaryOperator } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import type { IStreamBuilder, PipedOperator } from "../types.js"
+import type { DifferenceStreamReader } from "../graph.js"
+import type { MultiSet } from "../multiset.js"
+
+/**
+ * Operator that applies a function to each element in the input stream
+ */
+export class TapOperator<T> extends LinearUnaryOperator<T, T> {
+  #f: (data: T) => void
+
+  constructor(
+    id: number,
+    inputA: DifferenceStreamReader<T>,
+    output: DifferenceStreamWriter<T>,
+    f: (data: T) => void
+  ) {
+    super(id, inputA, output)
+    this.#f = f
+  }
+
+  inner(collection: MultiSet<T>): MultiSet<T> {
+    return collection.map((data) => {
+      this.#f(data)
+      return data
+    })
+  }
+}
+
+/**
+ * Invokes a function for each element in the input stream.
+ * This operator doesn't modify the stream and is used to perform side effects.
+ * @param f - The function to invoke on each element
+ * @returns The input stream
+ */
+export function tap<T>(f: (data: T) => void): PipedOperator<T, T> {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
+    const output = new StreamBuilder<T>(
+      stream.graph,
+      new DifferenceStreamWriter<T>()
+    )
+    const operator = new TapOperator<T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer,
+      f
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/topKWithFractionalIndex.ts

@@ -9,6 +9,7 @@ import type { IStreamBuilder, PipedOperator } from "../types.js"
 export interface TopKWithFractionalIndexOptions {
   limit?: number
   offset?: number
+  setSizeCallback?: (getSize: () => number) => void
 }
 
 export type TopKChanges<V> = {
@@ -23,6 +24,7 @@ export type TopKChanges<V> = {
  * and returns changes to the topK.
  */
 export interface TopK<V> {
+  size: number
   insert: (value: V) => TopKChanges<V>
   delete: (value: V) => TopKChanges<V>
 }
@@ -49,6 +51,13 @@ class TopKArray<V> implements TopK<V> {
     this.#comparator = comparator
   }
 
+  get size(): number {
+    const offset = this.#topKStart
+    const limit = this.#topKEnd - this.#topKStart
+    const available = this.#sortedValues.length - offset
+    return Math.max(0, Math.min(limit, available))
+  }
+
   insert(value: V): TopKChanges<V> {
     const result: TopKChanges<V> = { moveIn: null, moveOut: null }
 
@@ -169,6 +178,8 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
    */
   #topK: TopK<TaggedValue<K, T>>
 
+  #limit: number
+
   constructor(
     id: number,
     inputA: DifferenceStreamReader<[K, T]>,
@@ -177,7 +188,7 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
     options: TopKWithFractionalIndexOptions
   ) {
     super(id, inputA, output)
-    const limit = options.limit ?? Infinity
+    this.#limit = options.limit ?? Infinity
     const offset = options.offset ?? 0
     const compareTaggedValues = (
       a: TaggedValue<K, T>,
@@ -193,7 +204,8 @@ export class TopKWithFractionalIndexOperator<K, T> extends UnaryOperator<
       const tieBreakerB = getTag(b)
       return tieBreakerA - tieBreakerB
     }
-    this.#topK = this.createTopK(offset, limit, compareTaggedValues)
+    this.#topK = this.createTopK(offset, this.#limit, compareTaggedValues)
+    options.setSizeCallback?.(() => this.#topK.size)
   }
 
   protected createTopK(

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -75,6 +75,13 @@ class TopKTree<V> implements TopK<V> {
     this.#tree = new BTree(undefined, comparator)
   }
 
+  get size(): number {
+    const offset = this.#topKStart
+    const limit = this.#topKEnd - this.#topKStart
+    const available = this.#tree.size - offset
+    return Math.max(0, Math.min(limit, available))
+  }
+
   /**
    * Insert a *new* value.
    * Ignores the value if it is already present.

package/src/utils.ts

@@ -1,4 +1,15 @@
-import murmurhash from "murmurhash-js"
+/**
+ * Simple assertion function for runtime checks.
+ * Throws an error if the condition is false.
+ */
+export function assert(
+  condition: unknown,
+  message?: string
+): asserts condition {
+  if (!condition) {
+    throw new Error(message || `Assertion failed`)
+  }
+}
 
 /**
  * A map that returns a default value for keys that are not present.
@@ -13,7 +24,8 @@ export class DefaultMap<K, V> extends Map<K, V> {
 
   get(key: K): V {
     if (!this.has(key)) {
-      this.set(key, this.defaultValue())
+      // this.set(key, this.defaultValue())
+      return this.defaultValue()
     }
     return super.get(key)!
   }
@@ -45,52 +57,6 @@ export function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {
   }
 }
 
-const hashCache = new WeakMap()
-
-/**
- * Replacer function for JSON.stringify that converts unsupported types to strings
- */
-function hashReplacer(_key: string, value: any): any {
-  if (typeof value === `bigint`) {
-    return String(value)
-  } else if (typeof value === `symbol`) {
-    return String(value)
-  } else if (typeof value === `function`) {
-    return String(value)
-  } else if (value === undefined) {
-    return `undefined`
-  } else if (value instanceof Map) {
-    return `Map(${JSON.stringify(Array.from(value.entries()), hashReplacer)})`
-  } else if (value instanceof Set) {
-    return `Set(${JSON.stringify(Array.from(value.values()), hashReplacer)})`
-  }
-  return value
-}
-
-/**
- * A hash method that caches the hash of a value in a week map
- */
-export function hash(data: any): string {
-  if (
-    data === null ||
-    data === undefined ||
-    (typeof data !== `object` && typeof data !== `function`)
-  ) {
-    // Can't be cached in the weak map because it's not an object
-    const serialized = JSON.stringify(data, hashReplacer)
-    return murmurhash.murmur3(serialized).toString(16)
-  }
-
-  if (hashCache.has(data)) {
-    return hashCache.get(data)
-  }
-
-  const serialized = JSON.stringify(data, hashReplacer)
-  const hashValue = murmurhash.murmur3(serialized).toString(16)
-  hashCache.set(data, hashValue)
-  return hashValue
-}
-
 export function binarySearch<T>(
   array: Array<T>,
   value: T,
@@ -161,3 +127,20 @@ export class ObjectIdGenerator {
  * Global instance for cases where a shared object ID space is needed.
  */
 export const globalObjectIdGenerator = new ObjectIdGenerator()
+
+export function* concatIterable<T>(
+  ...iterables: Array<Iterable<T>>
+): Iterable<T> {
+  for (const iterable of iterables) {
+    yield* iterable
+  }
+}
+
+export function* mapIterable<T, U>(
+  it: Iterable<T>,
+  fn: (t: T) => U
+): Iterable<U> {
+  for (const t of it) {
+    yield fn(t)
+  }
+}

package/src/valueIndex.ts

@@ -0,0 +1,78 @@
+import { hash } from "./hashing/index.js"
+
+/**
+ * A map from a difference collection trace's keys -> (value, multiplicities) that changed.
+ * Used in operations like join and reduce where the operation needs to
+ * exploit the key-value structure of the data to run efficiently.
+ */
+export class ValueIndex<K, V> {
+  #inner: Map<K, [V, number]> // Maps key to the value and its multiplicity
+
+  constructor() {
+    this.#inner = new Map()
+  }
+
+  toString(indent = false): string {
+    return `ValueIndex(${JSON.stringify(
+      [...this.#inner.entries()],
+      undefined,
+      indent ? 2 : undefined
+    )})`
+  }
+
+  get(key: K): [V, number] | undefined {
+    return this.#inner.get(key)
+  }
+
+  getMultiplicity(key: K): number {
+    return this.get(key)?.[1] ?? 0
+  }
+
+  entries() {
+    return this.#inner.entries()
+  }
+
+  has(key: K): boolean {
+    return this.#inner.has(key)
+  }
+
+  delete(key: K): void {
+    this.#inner.delete(key)
+  }
+
+  get size(): number {
+    return this.#inner.size
+  }
+
+  addValue(key: K, v: [V, number]): void {
+    const [value, multiplicity] = v
+
+    if (multiplicity === 0) {
+      return
+    }
+
+    if (this.has(key)) {
+      const [currValue, currMultiplicity] = this.get(key)!
+      if (hash(value) === hash(currValue)) {
+        // Update the multiplicity
+        this.#setMultiplicity(key, value, currMultiplicity + multiplicity)
+        return
+      }
+      // Different value, not allowed.
+      // ValueIndex only supports one value per key.
+      throw new Error(
+        `Cannot add value for key ${key} because it already exists in ValueIndex with a different value`
+      )
+    }
+
+    this.#inner.set(key, [value, multiplicity])
+  }
+
+  #setMultiplicity(key: K, value: V, multiplicity: number): void {
+    if (multiplicity === 0) {
+      this.#inner.delete(key)
+    } else {
+      this.#inner.set(key, [value, multiplicity])
+    }
+  }
+}