@tanstack/db-ivm 0.0.0

package/src/multiset.ts

@@ -0,0 +1,215 @@
+import {
+  DefaultMap,
+  chunkedArrayPush,
+  globalObjectIdGenerator,
+  hash,
+} from "./utils.js"
+
+export type MultiSetArray<T> = Array<[T, number]>
+export type KeyedData<T> = [key: string, value: T]
+
+/**
+ * A multiset of data.
+ */
+export class MultiSet<T> {
+  #inner: MultiSetArray<T>
+
+  constructor(data: MultiSetArray<T> = []) {
+    this.#inner = data
+  }
+
+  toString(indent = false): string {
+    return `MultiSet(${JSON.stringify(this.#inner, null, indent ? 2 : undefined)})`
+  }
+
+  toJSON(): string {
+    return JSON.stringify(Array.from(this.getInner()))
+  }
+
+  static fromJSON<U>(json: string): MultiSet<U> {
+    return new MultiSet(JSON.parse(json))
+  }
+
+  /**
+   * Apply a function to all records in the collection.
+   */
+  map<U>(f: (data: T) => U): MultiSet<U> {
+    return new MultiSet(
+      this.#inner.map(([data, multiplicity]) => [f(data), multiplicity])
+    )
+  }
+
+  /**
+   * Filter out records for which a function f(record) evaluates to False.
+   */
+  filter(f: (data: T) => boolean): MultiSet<T> {
+    return new MultiSet(this.#inner.filter(([data, _]) => f(data)))
+  }
+
+  /**
+   * Negate all multiplicities in the collection.
+   */
+  negate(): MultiSet<T> {
+    return new MultiSet(
+      this.#inner.map(([data, multiplicity]) => [data, -multiplicity])
+    )
+  }
+
+  /**
+   * Concatenate two collections together.
+   */
+  concat(other: MultiSet<T>): MultiSet<T> {
+    const out: MultiSetArray<T> = []
+    chunkedArrayPush(out, this.#inner)
+    chunkedArrayPush(out, other.getInner())
+    return new MultiSet(out)
+  }
+
+  /**
+   * Produce as output a collection that is logically equivalent to the input
+   * but which combines identical instances of the same record into one
+   * (record, multiplicity) pair.
+   */
+  consolidate(): MultiSet<T> {
+    // Check if this looks like a keyed multiset (first item is a tuple of length 2)
+    if (this.#inner.length > 0) {
+      const firstItem = this.#inner[0]?.[0]
+      if (Array.isArray(firstItem) && firstItem.length === 2) {
+        return this.#consolidateKeyed()
+      }
+    }
+
+    // Fall back to original method for unkeyed data
+    return this.#consolidateUnkeyed()
+  }
+
+  /**
+   * Private method for consolidating keyed multisets where keys are strings/numbers
+   * and values are compared by reference equality.
+   *
+   * This method provides significant performance improvements over the hash-based approach
+   * by using WeakMap for object reference tracking and avoiding expensive serialization.
+   *
+   * Special handling for join operations: When values are tuples of length 2 (common in joins),
+   * we unpack them and compare each element individually to maintain proper equality semantics.
+   */
+  #consolidateKeyed(): MultiSet<T> {
+    const consolidated = new Map<string, number>()
+    const values = new Map<string, T>()
+
+    // Use global object ID generator for consistent reference equality
+
+    /**
+     * Special handler for tuples (arrays of length 2) commonly produced by join operations.
+     * Unpacks the tuple and generates an ID based on both elements to ensure proper
+     * consolidation of join results like ['A', null] and [null, 'X'].
+     */
+    const getTupleId = (tuple: Array<any>): string => {
+      if (tuple.length !== 2) {
+        throw new Error(`Expected tuple of length 2`)
+      }
+      const [first, second] = tuple
+      return `${globalObjectIdGenerator.getStringId(first)}|${globalObjectIdGenerator.getStringId(second)}`
+    }
+
+    // Process each item in the multiset
+    for (const [data, multiplicity] of this.#inner) {
+      // Verify this is still a keyed item (should be [key, value] pair)
+      if (!Array.isArray(data) || data.length !== 2) {
+        // Found non-keyed item, fall back to unkeyed consolidation
+        return this.#consolidateUnkeyed()
+      }
+
+      const [key, value] = data
+
+      // Verify key is string or number as expected for keyed multisets
+      if (typeof key !== `string` && typeof key !== `number`) {
+        // Found non-string/number key, fall back to unkeyed consolidation
+        return this.#consolidateUnkeyed()
+      }
+
+      // Generate value ID with special handling for join tuples
+      let valueId: string
+      if (Array.isArray(value) && value.length === 2) {
+        // Special case: value is a tuple from join operations
+        valueId = getTupleId(value)
+      } else {
+        // Regular case: use reference/value equality
+        valueId = globalObjectIdGenerator.getStringId(value)
+      }
+
+      // Create composite key and consolidate
+      const compositeKey = key + `|` + valueId
+      consolidated.set(
+        compositeKey,
+        (consolidated.get(compositeKey) || 0) + multiplicity
+      )
+
+      // Store the original data for the first occurrence
+      if (!values.has(compositeKey)) {
+        values.set(compositeKey, data as T)
+      }
+    }
+
+    // Build result array, filtering out zero multiplicities
+    const result: MultiSetArray<T> = []
+    for (const [compositeKey, multiplicity] of consolidated) {
+      if (multiplicity !== 0) {
+        result.push([values.get(compositeKey)!, multiplicity])
+      }
+    }
+
+    return new MultiSet(result)
+  }
+
+  /**
+   * Private method for consolidating unkeyed multisets using the original approach.
+   */
+  #consolidateUnkeyed(): MultiSet<T> {
+    const consolidated = new DefaultMap<string | number, number>(() => 0)
+    const values = new Map<string, any>()
+
+    let hasString = false
+    let hasNumber = false
+    let hasOther = false
+    for (const [data, _] of this.#inner) {
+      if (typeof data === `string`) {
+        hasString = true
+      } else if (typeof data === `number`) {
+        hasNumber = true
+      } else {
+        hasOther = true
+        break
+      }
+    }
+
+    const requireJson = hasOther || (hasString && hasNumber)
+
+    for (const [data, multiplicity] of this.#inner) {
+      const key = requireJson ? hash(data) : (data as string | number)
+      if (requireJson && !values.has(key as string)) {
+        values.set(key as string, data)
+      }
+      consolidated.update(key, (count) => count + multiplicity)
+    }
+
+    const result: MultiSetArray<T> = []
+    for (const [key, multiplicity] of consolidated.entries()) {
+      if (multiplicity !== 0) {
+        const parsedKey = requireJson ? values.get(key as string) : key
+        result.push([parsedKey as T, multiplicity])
+      }
+    }
+
+    return new MultiSet(result)
+  }
+
+  extend(other: MultiSet<T> | MultiSetArray<T>): void {
+    const otherArray = other instanceof MultiSet ? other.getInner() : other
+    chunkedArrayPush(this.#inner, otherArray)
+  }
+
+  getInner(): MultiSetArray<T> {
+    return this.#inner
+  }
+}

package/src/operators/concat.ts

@@ -0,0 +1,45 @@
+import { BinaryOperator, DifferenceStreamWriter } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import type { IStreamBuilder, PipedOperator } from "../types.js"
+
+/**
+ * Operator that concatenates two input streams
+ */
+export class ConcatOperator<T, T2> extends BinaryOperator<T | T2> {
+  run(): void {
+    for (const message of this.inputAMessages()) {
+      this.output.sendData(message)
+    }
+
+    for (const message of this.inputBMessages()) {
+      this.output.sendData(message)
+    }
+  }
+}
+
+/**
+ * Concatenates two input streams
+ * @param other - The other stream to concatenate
+ */
+export function concat<T, T2>(
+  other: IStreamBuilder<T2>
+): PipedOperator<T, T | T2> {
+  return (stream: IStreamBuilder<T | T2>): IStreamBuilder<T | T2> => {
+    if (stream.graph !== other.graph) {
+      throw new Error(`Cannot concat streams from different graphs`)
+    }
+    const output = new StreamBuilder<T | T2>(
+      stream.graph,
+      new DifferenceStreamWriter<T | T2>()
+    )
+    const operator = new ConcatOperator<T, T2>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      other.connectReader(),
+      output.writer
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/consolidate.ts

@@ -0,0 +1,50 @@
+import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import { MultiSet } from "../multiset.js"
+import type { IStreamBuilder, PipedOperator } from "../types.js"
+
+/**
+ * Operator that consolidates collections
+ */
+export class ConsolidateOperator<T> extends UnaryOperator<T> {
+  run(): void {
+    const messages = this.inputMessages()
+    if (messages.length === 0) {
+      return
+    }
+
+    // Combine all messages into a single MultiSet
+    const combined = new MultiSet<T>()
+    for (const message of messages) {
+      combined.extend(message)
+    }
+
+    // Consolidate the combined MultiSet
+    const consolidated = combined.consolidate()
+
+    // Only send if there are results
+    if (consolidated.getInner().length > 0) {
+      this.output.sendData(consolidated)
+    }
+  }
+}
+
+/**
+ * Consolidates the elements in the stream
+ */
+export function consolidate<T>(): PipedOperator<T, T> {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
+    const output = new StreamBuilder<T>(
+      stream.graph,
+      new DifferenceStreamWriter<T>()
+    )
+    const operator = new ConsolidateOperator<T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/count.ts

@@ -0,0 +1,52 @@
+import { DifferenceStreamWriter } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import { ReduceOperator } from "./reduce.js"
+import type { DifferenceStreamReader } from "../graph.js"
+import type { IStreamBuilder, KeyValue } from "../types.js"
+
+/**
+ * Operator that counts elements by key (version-free)
+ */
+export class CountOperator<K, V> extends ReduceOperator<K, V, number> {
+  constructor(
+    id: number,
+    inputA: DifferenceStreamReader<[K, V]>,
+    output: DifferenceStreamWriter<[K, number]>
+  ) {
+    const countInner = (vals: Array<[V, number]>): Array<[number, number]> => {
+      let totalCount = 0
+      for (const [_, diff] of vals) {
+        totalCount += diff
+      }
+      return [[totalCount, 1]]
+    }
+
+    super(id, inputA, output, countInner)
+  }
+}
+
+/**
+ * Counts the number of elements by key (version-free)
+ */
+export function count<
+  KType extends T extends KeyValue<infer K, infer _V> ? K : never,
+  VType extends T extends KeyValue<KType, infer V> ? V : never,
+  T,
+>() {
+  return (
+    stream: IStreamBuilder<T>
+  ): IStreamBuilder<KeyValue<KType, number>> => {
+    const output = new StreamBuilder<KeyValue<KType, number>>(
+      stream.graph,
+      new DifferenceStreamWriter<KeyValue<KType, number>>()
+    )
+    const operator = new CountOperator<KType, VType>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader() as DifferenceStreamReader<KeyValue<KType, VType>>,
+      output.writer
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/debug.ts

@@ -0,0 +1,58 @@
+import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import type { IStreamBuilder, PipedOperator } from "../types.js"
+import type { DifferenceStreamReader } from "../graph.js"
+
+/**
+ * Operator that logs debug information about the stream
+ */
+export class DebugOperator<T> extends UnaryOperator<T> {
+  #name: string
+  #indent: boolean
+
+  constructor(
+    id: number,
+    inputA: DifferenceStreamReader<T>,
+    output: DifferenceStreamWriter<T>,
+    name: string,
+    indent: boolean = false
+  ) {
+    super(id, inputA, output)
+    this.#name = name
+    this.#indent = indent
+  }
+
+  run(): void {
+    for (const message of this.inputMessages()) {
+      console.log(`debug ${this.#name} data: ${message.toString(this.#indent)}`)
+      this.output.sendData(message)
+    }
+  }
+}
+
+/**
+ * Logs debug information about the stream using console.log
+ * @param name - The name to prefix debug messages with
+ * @param indent - Whether to indent the debug output
+ */
+export function debug<T>(
+  name: string,
+  indent: boolean = false
+): PipedOperator<T, T> {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
+    const output = new StreamBuilder<T>(
+      stream.graph,
+      new DifferenceStreamWriter<T>()
+    )
+    const operator = new DebugOperator<T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer,
+      name,
+      indent
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/distinct.ts

@@ -0,0 +1,98 @@
+import { DifferenceStreamWriter, UnaryOperator } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import { hash } from "../utils.js"
+import { MultiSet } from "../multiset.js"
+import type { DifferenceStreamReader } from "../graph.js"
+import type { IStreamBuilder } from "../types.js"
+
+type HashedValue = string
+type Multiplicity = number
+
+/**
+ * Operator that removes duplicates
+ */
+export class DistinctOperator<T> extends UnaryOperator<T> {
+  #by: (value: T) => any
+  #values: Map<HashedValue, Multiplicity> // keeps track of the number of times each value has been seen
+
+  constructor(
+    id: number,
+    input: DifferenceStreamReader<T>,
+    output: DifferenceStreamWriter<T>,
+    by: (value: T) => any = (value: T) => value
+  ) {
+    super(id, input, output)
+    this.#by = by
+    this.#values = new Map()
+  }
+
+  run(): void {
+    const updatedValues = new Map<HashedValue, [Multiplicity, T]>()
+
+    // Compute the new multiplicity for each value
+    for (const message of this.inputMessages()) {
+      for (const [value, diff] of message.getInner()) {
+        const hashedValue = hash(this.#by(value))
+
+        const oldMultiplicity =
+          updatedValues.get(hashedValue)?.[0] ??
+          this.#values.get(hashedValue) ??
+          0
+        const newMultiplicity = oldMultiplicity + diff
+
+        updatedValues.set(hashedValue, [newMultiplicity, value])
+      }
+    }
+
+    const result: Array<[T, number]> = []
+
+    // Check which values became visible or disappeared
+    for (const [
+      hashedValue,
+      [newMultiplicity, value],
+    ] of updatedValues.entries()) {
+      const oldMultiplicity = this.#values.get(hashedValue) ?? 0
+
+      if (newMultiplicity === 0) {
+        this.#values.delete(hashedValue)
+      } else {
+        this.#values.set(hashedValue, newMultiplicity)
+      }
+
+      if (oldMultiplicity <= 0 && newMultiplicity > 0) {
+        // The value wasn't present in the stream
+        // but with this change it is now present in the stream
+        result.push([value, 1])
+      } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
+        // The value was present in the stream
+        // but with this change it is no longer present in the stream
+        result.push([value, -1])
+      }
+    }
+
+    if (result.length > 0) {
+      this.output.sendData(new MultiSet(result))
+    }
+  }
+}
+
+/**
+ * Removes duplicate values
+ */
+export function distinct<T>(by: (value: T) => any = (value: T) => value) {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
+    const output = new StreamBuilder<T>(
+      stream.graph,
+      new DifferenceStreamWriter<T>()
+    )
+    const operator = new DistinctOperator<T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer,
+      by
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/filter.ts

@@ -0,0 +1,48 @@
+import { DifferenceStreamWriter, LinearUnaryOperator } from "../graph.js"
+import { StreamBuilder } from "../d2.js"
+import type { IStreamBuilder, PipedOperator } from "../types.js"
+import type { DifferenceStreamReader } from "../graph.js"
+import type { MultiSet } from "../multiset.js"
+
+/**
+ * Operator that filters elements from the input stream
+ */
+export class FilterOperator<T> extends LinearUnaryOperator<T, T> {
+  #f: (data: T) => boolean
+
+  constructor(
+    id: number,
+    inputA: DifferenceStreamReader<T>,
+    output: DifferenceStreamWriter<T>,
+    f: (data: T) => boolean
+  ) {
+    super(id, inputA, output)
+    this.#f = f
+  }
+
+  inner(collection: MultiSet<T>): MultiSet<T> {
+    return collection.filter(this.#f)
+  }
+}
+
+/**
+ * Filters elements from the input stream
+ * @param f - The predicate to filter elements
+ */
+export function filter<T>(f: (data: T) => boolean): PipedOperator<T, T> {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
+    const output = new StreamBuilder<T>(
+      stream.graph,
+      new DifferenceStreamWriter<T>()
+    )
+    const operator = new FilterOperator<T>(
+      stream.graph.getNextOperatorId(),
+      stream.connectReader(),
+      output.writer,
+      f
+    )
+    stream.graph.addOperator(operator)
+    stream.graph.addStream(output.connectReader())
+    return output
+  }
+}

package/src/operators/filterBy.ts

@@ -0,0 +1,31 @@
+import { map } from "./map.js"
+import { innerJoin } from "./join.js"
+import { consolidate } from "./consolidate.js"
+import type { KeyValue } from "../types.js"
+import type { IStreamBuilder, PipedOperator } from "../types"
+
+/**
+ * Filters the elements of a keyed stream, by keys of another stream.
+ * This allows you to build pipelies where you have multiple outputs that are related,
+ * such as a streams of issues and comments for a project.
+ *
+ * @param other - The other stream to filter by, which must have the same key type as the input stream
+ */
+export function filterBy<
+  K,
+  V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
+  T,
+>(
+  other: IStreamBuilder<KeyValue<K, unknown>>
+): PipedOperator<T, KeyValue<K, V1>> {
+  return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<K, V1>> => {
+    const otherKeys = other.pipe(
+      map(([key, _]) => [key, null] as KeyValue<K, null>)
+    )
+    return stream.pipe(
+      innerJoin(otherKeys),
+      map(([key, [value, _]]) => [key, value] as KeyValue<K, V1>),
+      consolidate()
+    )
+  }
+}