@tanstack/db-ivm 0.0.0

package/src/types.ts

@@ -0,0 +1,376 @@
+import type { MultiSet, MultiSetArray } from "./multiset.js"
+import type {
+  BinaryOperator,
+  DifferenceStreamReader,
+  DifferenceStreamWriter,
+  UnaryOperator,
+} from "./graph.js"
+
+export type KeyValue<K, V> = [K, V]
+
+export interface IOperator<_T> {
+  run: () => void
+  hasPendingWork: () => boolean
+}
+
+export interface IDifferenceStreamReader<T> {
+  drain: () => Array<MultiSet<T>>
+  isEmpty: () => boolean
+}
+
+export interface IDifferenceStreamWriter<T> {
+  sendData: (collection: MultiSet<T> | MultiSetArray<T>) => void
+  newReader: () => IDifferenceStreamReader<T>
+}
+
+export interface ID2 {
+  getNextOperatorId: () => number
+  newInput: <T>() => IStreamBuilder<T>
+  addOperator: (operator: UnaryOperator<any> | BinaryOperator<any>) => void
+  addStream: (stream: DifferenceStreamReader<any>) => void
+  finalize: () => void
+  step: () => void
+}
+
+export interface IStreamBuilder<T> {
+  writer: DifferenceStreamWriter<T>
+  connectReader: () => DifferenceStreamReader<T>
+  graph: ID2
+
+  // Don't judge, this is the only way to type this function.
+  // rxjs has very similar code to type its pipe function
+  // https://github.com/ReactiveX/rxjs/blob/master/packages/rxjs/src/internal/util/pipe.ts
+  // We go to 20 operators deep, because surly that's enough for anyone...
+  // A user can always split the pipe into multiple pipes to get around this.
+  pipe: (<O>(o1: PipedOperator<T, O>) => IStreamBuilder<O>) &
+    (<T2, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, O>
+    ) => IStreamBuilder<O>) &
+    (<T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, O>(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, T16>,
+      o16: PipedOperator<T16, O>
+    ) => IStreamBuilder<O>) &
+    (<
+      T2,
+      T3,
+      T4,
+      T5,
+      T6,
+      T7,
+      T8,
+      T9,
+      T10,
+      T11,
+      T12,
+      T13,
+      T14,
+      T15,
+      T16,
+      T17,
+      O,
+    >(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, T16>,
+      o16: PipedOperator<T16, T17>,
+      o17: PipedOperator<T17, O>
+    ) => IStreamBuilder<O>) &
+    (<
+      T2,
+      T3,
+      T4,
+      T5,
+      T6,
+      T7,
+      T8,
+      T9,
+      T10,
+      T11,
+      T12,
+      T13,
+      T14,
+      T15,
+      T16,
+      T17,
+      T18,
+      O,
+    >(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, T16>,
+      o16: PipedOperator<T16, T17>,
+      o17: PipedOperator<T17, T18>,
+      o18: PipedOperator<T18, O>
+    ) => IStreamBuilder<O>) &
+    (<
+      T2,
+      T3,
+      T4,
+      T5,
+      T6,
+      T7,
+      T8,
+      T9,
+      T10,
+      T11,
+      T12,
+      T13,
+      T14,
+      T15,
+      T16,
+      T17,
+      T18,
+      T19,
+      O,
+    >(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, T16>,
+      o16: PipedOperator<T16, T17>,
+      o17: PipedOperator<T17, T18>,
+      o18: PipedOperator<T18, T19>,
+      o19: PipedOperator<T19, O>
+    ) => IStreamBuilder<O>) &
+    (<
+      T2,
+      T3,
+      T4,
+      T5,
+      T6,
+      T7,
+      T8,
+      T9,
+      T10,
+      T11,
+      T12,
+      T13,
+      T14,
+      T15,
+      T16,
+      T17,
+      T18,
+      T19,
+      T20,
+      O,
+    >(
+      o1: PipedOperator<T, T2>,
+      o2: PipedOperator<T2, T3>,
+      o3: PipedOperator<T3, T4>,
+      o4: PipedOperator<T4, T5>,
+      o5: PipedOperator<T5, T6>,
+      o6: PipedOperator<T6, T7>,
+      o7: PipedOperator<T7, T8>,
+      o8: PipedOperator<T8, T9>,
+      o9: PipedOperator<T9, T10>,
+      o10: PipedOperator<T10, T11>,
+      o11: PipedOperator<T11, T12>,
+      o12: PipedOperator<T12, T13>,
+      o13: PipedOperator<T13, T14>,
+      o14: PipedOperator<T14, T15>,
+      o15: PipedOperator<T15, T16>,
+      o16: PipedOperator<T16, T17>,
+      o17: PipedOperator<T17, T18>,
+      o18: PipedOperator<T18, T19>,
+      o19: PipedOperator<T19, T20>,
+      o20: PipedOperator<T20, O>
+    ) => IStreamBuilder<O>) &
+    (<O>(...operators: Array<PipedOperator<any, any>>) => IStreamBuilder<O>)
+}
+
+export type PipedOperator<I, O> = (
+  stream: IStreamBuilder<I>
+) => IStreamBuilder<O>

package/src/utils.ts

@@ -0,0 +1,163 @@
+import murmurhash from "murmurhash-js"
+
+/**
+ * A map that returns a default value for keys that are not present.
+ */
+export class DefaultMap<K, V> extends Map<K, V> {
+  constructor(
+    private defaultValue: () => V,
+    entries?: Iterable<[K, V]>
+  ) {
+    super(entries)
+  }
+
+  get(key: K): V {
+    if (!this.has(key)) {
+      this.set(key, this.defaultValue())
+    }
+    return super.get(key)!
+  }
+
+  /**
+   * Update the value for a key using a function.
+   */
+  update(key: K, updater: (value: V) => V): V {
+    const value = this.get(key)
+    const newValue = updater(value)
+    this.set(key, newValue)
+    return newValue
+  }
+}
+
+// JS engines have various limits on how many args can be passed to a function
+// with a spread operator, so we need to split the operation into chunks
+// 32767 is the max for Chrome 14, all others are higher
+// TODO: investigate the performance of this and other approaches
+const chunkSize = 30000
+export function chunkedArrayPush(array: Array<unknown>, other: Array<unknown>) {
+  if (other.length <= chunkSize) {
+    array.push(...other)
+  } else {
+    for (let i = 0; i < other.length; i += chunkSize) {
+      const chunk = other.slice(i, i + chunkSize)
+      array.push(...chunk)
+    }
+  }
+}
+
+const hashCache = new WeakMap()
+
+/**
+ * Replacer function for JSON.stringify that converts unsupported types to strings
+ */
+function hashReplacer(_key: string, value: any): any {
+  if (typeof value === `bigint`) {
+    return String(value)
+  } else if (typeof value === `symbol`) {
+    return String(value)
+  } else if (typeof value === `function`) {
+    return String(value)
+  } else if (value === undefined) {
+    return `undefined`
+  } else if (value instanceof Map) {
+    return `Map(${JSON.stringify(Array.from(value.entries()), hashReplacer)})`
+  } else if (value instanceof Set) {
+    return `Set(${JSON.stringify(Array.from(value.values()), hashReplacer)})`
+  }
+  return value
+}
+
+/**
+ * A hash method that caches the hash of a value in a week map
+ */
+export function hash(data: any): string {
+  if (
+    data === null ||
+    data === undefined ||
+    (typeof data !== `object` && typeof data !== `function`)
+  ) {
+    // Can't be cached in the weak map because it's not an object
+    const serialized = JSON.stringify(data, hashReplacer)
+    return murmurhash.murmur3(serialized).toString(16)
+  }
+
+  if (hashCache.has(data)) {
+    return hashCache.get(data)
+  }
+
+  const serialized = JSON.stringify(data, hashReplacer)
+  const hashValue = murmurhash.murmur3(serialized).toString(16)
+  hashCache.set(data, hashValue)
+  return hashValue
+}
+
+export function binarySearch<T>(
+  array: Array<T>,
+  value: T,
+  comparator: (a: T, b: T) => number
+): number {
+  let low = 0
+  let high = array.length
+  while (low < high) {
+    const mid = Math.floor((low + high) / 2)
+    const comparison = comparator(array[mid]!, value)
+    if (comparison < 0) {
+      low = mid + 1
+    } else if (comparison > 0) {
+      high = mid
+    } else {
+      return mid
+    }
+  }
+  return low
+}
+
+/**
+ * Utility for generating unique IDs for objects and values.
+ * Uses WeakMap for object reference tracking and consistent hashing for primitives.
+ */
+export class ObjectIdGenerator {
+  private objectIds = new WeakMap<object, number>()
+  private nextId = 0
+
+  /**
+   * Get a unique identifier for any value.
+   * - Objects: Uses WeakMap for reference-based identity
+   * - Primitives: Uses consistent string-based hashing
+   */
+  getId(value: any): number {
+    // For primitives, use a simple hash of their string representation
+    if (typeof value !== `object` || value === null) {
+      const str = String(value)
+      let hashValue = 0
+      for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i)
+        hashValue = (hashValue << 5) - hashValue + char
+        hashValue = hashValue & hashValue // Convert to 32-bit integer
+      }
+      return hashValue
+    }
+
+    // For objects, use WeakMap to assign unique IDs
+    if (!this.objectIds.has(value)) {
+      this.objectIds.set(value, this.nextId++)
+    }
+    return this.objectIds.get(value)!
+  }
+
+  /**
+   * Get a string representation of the ID for use in composite keys.
+   */
+  getStringId(value: any): string {
+    if (value === null) return `null`
+    if (value === undefined) return `undefined`
+    if (typeof value !== `object`) return `str_${String(value)}`
+
+    return `obj_${this.getId(value)}`
+  }
+}
+
+/**
+ * Global instance for cases where a shared object ID space is needed.
+ */
+export const globalObjectIdGenerator = new ObjectIdGenerator()