@tanstack/db-ivm 0.1.26 → 0.2.0

package/src/operators/count.ts

@@ -1,51 +0,0 @@
-import { DifferenceStreamWriter } from '../graph.js'
-import { StreamBuilder } from '../d2.js'
-import { ReduceOperator } from './reduce.js'
-import type { DifferenceStreamReader } from '../graph.js'
-import type { IStreamBuilder, KeyValue } from '../types.js'
-
-/**
- * Operator that counts elements by key (version-free)
- */
-export class CountOperator<K, V> extends ReduceOperator<K, V, number> {
-  constructor(
-    id: number,
-    inputA: DifferenceStreamReader<[K, V]>,
-    output: DifferenceStreamWriter<[K, number]>,
-  ) {
-    const countInner = (vals: Array<[V, number]>): Array<[number, number]> => {
-      let totalCount = 0
-      for (const [_, diff] of vals) {
-        totalCount += diff
-      }
-      return [[totalCount, 1]]
-    }
-
-    super(id, inputA, output, countInner)
-  }
-}
-
-/**
- * Counts the number of elements by key (version-free)
- */
-export function count<
-  KType extends (T extends KeyValue<infer K, infer _V> ? K : never),
-  VType extends (T extends KeyValue<KType, infer V> ? V : never),
-  T,
->() {
-  return (
-    stream: IStreamBuilder<T>,
-  ): IStreamBuilder<KeyValue<KType, number>> => {
-    const output = new StreamBuilder<KeyValue<KType, number>>(
-      stream.graph,
-      new DifferenceStreamWriter<KeyValue<KType, number>>(),
-    )
-    const operator = new CountOperator<KType, VType>(
-      stream.graph.getNextOperatorId(),
-      stream.connectReader() as DifferenceStreamReader<KeyValue<KType, VType>>,
-      output.writer,
-    )
-    stream.graph.addOperator(operator)
-    return output
-  }
-}

package/src/operators/debug.ts

@@ -1,57 +0,0 @@
-import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
-import { StreamBuilder } from '../d2.js'
-import type { IStreamBuilder, PipedOperator } from '../types.js'
-import type { DifferenceStreamReader } from '../graph.js'
-
-/**
- * Operator that logs debug information about the stream
- */
-export class DebugOperator<T> extends UnaryOperator<T> {
-  #name: string
-  #indent: boolean
-
-  constructor(
-    id: number,
-    inputA: DifferenceStreamReader<T>,
-    output: DifferenceStreamWriter<T>,
-    name: string,
-    indent: boolean = false,
-  ) {
-    super(id, inputA, output)
-    this.#name = name
-    this.#indent = indent
-  }
-
-  run(): void {
-    for (const message of this.inputMessages()) {
-      console.log(`debug ${this.#name} data: ${message.toString(this.#indent)}`)
-      this.output.sendData(message)
-    }
-  }
-}
-
-/**
- * Logs debug information about the stream using console.log
- * @param name - The name to prefix debug messages with
- * @param indent - Whether to indent the debug output
- */
-export function debug<T>(
-  name: string,
-  indent: boolean = false,
-): PipedOperator<T, T> {
-  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
-    const output = new StreamBuilder<T>(
-      stream.graph,
-      new DifferenceStreamWriter<T>(),
-    )
-    const operator = new DebugOperator<T>(
-      stream.graph.getNextOperatorId(),
-      stream.connectReader(),
-      output.writer,
-      name,
-      indent,
-    )
-    stream.graph.addOperator(operator)
-    return output
-  }
-}

package/src/operators/filterBy.ts

@@ -1,30 +0,0 @@
-import { map } from './map.js'
-import { innerJoin } from './join.js'
-import { consolidate } from './consolidate.js'
-import type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'
-
-/**
- * Filters the elements of a keyed stream, by keys of another stream.
- * This allows you to build pipelies where you have multiple outputs that are related,
- * such as a streams of issues and comments for a project.
- *
- * @param other - The other stream to filter by, which must have the same key type as the input stream
- */
-export function filterBy<
-  K,
-  V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
-  T,
->(
-  other: IStreamBuilder<KeyValue<K, unknown>>,
-): PipedOperator<T, KeyValue<K, V1>> {
-  return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<K, V1>> => {
-    const otherKeys = other.pipe(
-      map(([key, _]) => [key, null] as KeyValue<K, null>),
-    )
-    return stream.pipe(
-      innerJoin(otherKeys),
-      map(([key, [value, _]]) => [key, value] as KeyValue<K, V1>),
-      consolidate(),
-    )
-  }
-}

package/src/operators/keying.ts

@@ -1,34 +0,0 @@
-import { map } from './map.js'
-import type { PipedOperator } from '../types.js'
-
-/**
- * Type for a keyed value
- */
-export type Keyed<K, V> = [K, V]
-
-/**
- * Takes an unkeyed input and returns a keyed result
- * @param keyFn - Function to generate the key for each value
- */
-export function keyBy<T, K>(
-  keyFn: (value: T) => K,
-): PipedOperator<T, Keyed<K, T>> {
-  return map((value: T): Keyed<K, T> => [keyFn(value), value])
-}
-
-/**
- * Removes the key from a keyed stream
- */
-export function unkey<K, V>(): PipedOperator<Keyed<K, V>, V> {
-  return map(([_, value]: Keyed<K, V>): V => value)
-}
-
-/**
- * Takes a keyed input and rekeys it with a new key
- * @param keyFn - Function to generate the new key for each value
- */
-export function rekey<K1, K2, V>(
-  keyFn: (value: V) => K2,
-): PipedOperator<Keyed<K1, V>, Keyed<K2, V>> {
-  return map(([_, value]: Keyed<K1, V>): Keyed<K2, V> => [keyFn(value), value])
-}

package/src/operators/negate.ts

@@ -1,32 +0,0 @@
-import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
-import { StreamBuilder } from '../d2.js'
-import type { IStreamBuilder, PipedOperator } from '../types.js'
-import type { MultiSet } from '../multiset.js'
-
-/**
- * Operator that negates the multiplicities in the input stream
- */
-export class NegateOperator<T> extends LinearUnaryOperator<T, T> {
-  inner(collection: MultiSet<T>): MultiSet<T> {
-    return collection.negate()
-  }
-}
-
-/**
- * Negates the multiplicities in the input stream
- */
-export function negate<T>(): PipedOperator<T, T> {
-  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
-    const output = new StreamBuilder<T>(
-      stream.graph,
-      new DifferenceStreamWriter<T>(),
-    )
-    const operator = new NegateOperator<T>(
-      stream.graph.getNextOperatorId(),
-      stream.connectReader(),
-      output.writer,
-    )
-    stream.graph.addOperator(operator)
-    return output
-  }
-}

package/src/operators/topK.ts

@@ -1,91 +0,0 @@
-import { MultiSet } from '../multiset.js'
-import { reduce } from './reduce.js'
-import type { IStreamBuilder, KeyValue, PipedOperator } from '../types'
-
-interface TopKOptions {
-  limit?: number
-  offset?: number
-}
-
-/**
- * Limits the number of results based on a comparator, with optional offset.
- * This works on a keyed stream, where the key is the first element of the tuple
- * The ordering is withing a key group, i.e. elements are sorted within a key group
- * and the limit + offset is applied to that sorted group.
- * To order the entire stream, key by the same value for all elements such as null.
- *
- * @param comparator - A function that compares two elements
- * @param options - An optional object containing limit and offset properties
- * @returns A piped operator that limits the number of results
- */
-export function topK<
-  KType extends (T extends KeyValue<infer K, infer _V> ? K : never),
-  V1Type extends (T extends KeyValue<KType, infer V> ? V : never),
-  T,
->(
-  comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions,
-): PipedOperator<T, T> {
-  const limit = options?.limit ?? Infinity
-  const offset = options?.offset ?? 0
-
-  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
-    const reduced = stream.pipe(
-      reduce((values) => {
-        // `values` is a list of tuples, first element is the value, second is the multiplicity
-        const consolidated = new MultiSet(values).consolidate()
-        const sortedValues = consolidated
-          .getInner()
-          .sort((a, b) => comparator(a[0] as V1Type, b[0] as V1Type))
-        return sortedValues.slice(offset, offset + limit)
-      }),
-    )
-    return reduced as IStreamBuilder<T>
-  }
-}
-
-/**
- * Limits the number of results based on a comparator, with optional offset.
- * This works on a keyed stream, where the key is the first element of the tuple
- * The ordering is withing a key group, i.e. elements are sorted within a key group
- * and the limit + offset is applied to that sorted group.
- * To order the entire stream, key by the same value for all elements such as null.
- * Adds the index of the element to the result as [key, [value, index]]
- *
- * @param comparator - A function that compares two elements
- * @param options - An optional object containing limit and offset properties
- * @returns A piped operator that orders the elements and limits the number of results
- */
-export function topKWithIndex<
-  KType extends (T extends KeyValue<infer K, infer _V> ? K : never),
-  V1Type extends (T extends KeyValue<KType, infer V> ? V : never),
-  T,
->(
-  comparator: (a: V1Type, b: V1Type) => number,
-  options?: TopKOptions,
-): PipedOperator<T, KeyValue<KType, [V1Type, number]>> {
-  const limit = options?.limit ?? Infinity
-  const offset = options?.offset ?? 0
-
-  return (
-    stream: IStreamBuilder<T>,
-  ): IStreamBuilder<KeyValue<KType, [V1Type, number]>> => {
-    const reduced = stream.pipe(
-      reduce<KType, V1Type, [V1Type, number], T>((values) => {
-        // `values` is a list of tuples, first element is the value, second is the multiplicity
-        const consolidated = new MultiSet(values).consolidate()
-        let i = offset
-        const sortedValues = consolidated
-          .getInner()
-          .sort((a, b) => comparator(a[0], b[0]))
-          .slice(offset, offset + limit)
-          .map(([value, multiplicity]): [[V1Type, number], number] => [
-            [value, i++],
-            multiplicity,
-          ])
-        return sortedValues
-      }),
-    )
-    return reduced
-  }
-}

package/src/operators/topKWithFractionalIndexBTree.ts

@@ -1,297 +0,0 @@
-import { generateKeyBetween } from 'fractional-indexing'
-import { DifferenceStreamWriter } from '../graph.js'
-import { StreamBuilder } from '../d2.js'
-import { TopKWithFractionalIndexOperator } from './topKWithFractionalIndex.js'
-import { getIndex, getValue, indexedValue } from './topKArray.js'
-import type { IndexedValue, TopK, TopKChanges } from './topKArray.js'
-import type { IStreamBuilder, PipedOperator } from '../types.js'
-import type { TopKWithFractionalIndexOptions } from './topKWithFractionalIndex.js'
-
-interface BTree<Key, Value> {
-  nextLowerPair: (key: Key) => [Key, Value] | undefined
-  nextHigherPair: (key: Key) => [Key, Value] | undefined
-  set: (key: Key, value: Value, overwrite?: boolean) => boolean
-  maxKey: () => Key | undefined
-  get: (key: Key, defaultValue?: Value) => Value | undefined
-  delete: (key: Key) => boolean
-  size: number
-}
-
-interface BTreeClass {
-  new <Key, Value>(
-    entries?: Array<[Key, Value]>,
-    compare?: (a: Key, b: Key) => number,
-    maxNodeSize?: number,
-  ): BTree<Key, Value>
-}
-
-let BTree: BTreeClass | undefined
-
-export async function loadBTree() {
-  if (BTree === undefined) {
-    const { default: _BTreeClass } = await import(`sorted-btree`)
-    BTree = _BTreeClass
-  }
-}
-
-/**
- * Implementation of a topK data structure that uses a B+ tree.
- * The tree allows for logarithmic time insertions and deletions.
- */
-class TopKTree<V> implements TopK<V> {
-  #comparator: (a: V, b: V) => number
-  // topK is a window at position [topKStart, topKEnd[
-  // i.e. `topKStart` is inclusive and `topKEnd` is exclusive
-  #topKStart: number
-  #topKEnd: number
-
-  #tree: BTree<V, IndexedValue<V>>
-  #topKFirstElem: IndexedValue<V> | null = null // inclusive
-  #topKLastElem: IndexedValue<V> | null = null // inclusive
-
-  constructor(
-    offset: number,
-    limit: number,
-    comparator: (a: V, b: V) => number,
-  ) {
-    if (BTree === undefined) {
-      throw new Error(
-        `B+ tree not loaded. You need to call loadBTree() before using TopKTree.`,
-      )
-    }
-
-    this.#topKStart = offset
-    this.#topKEnd = offset + limit
-    this.#comparator = comparator
-    this.#tree = new BTree(undefined, comparator)
-  }
-
-  get size(): number {
-    const offset = this.#topKStart
-    const limit = this.#topKEnd - this.#topKStart
-    const available = this.#tree.size - offset
-    return Math.max(0, Math.min(limit, available))
-  }
-
-  /**
-   * Insert a *new* value.
-   * Ignores the value if it is already present.
-   */
-  insert(value: V): TopKChanges<V> {
-    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
-
-    // Get the elements before and after the value
-    const [, indexedValueBefore] = this.#tree.nextLowerPair(value) ?? [
-      null,
-      null,
-    ]
-    const [, indexedValueAfter] = this.#tree.nextHigherPair(value) ?? [
-      null,
-      null,
-    ]
-
-    const indexBefore = indexedValueBefore ? getIndex(indexedValueBefore) : null
-    const indexAfter = indexedValueAfter ? getIndex(indexedValueAfter) : null
-
-    // Generate a fractional index for the value
-    // based on the fractional indices of the elements before and after it
-    const fractionalIndex = generateKeyBetween(indexBefore, indexAfter)
-    const insertedElem = indexedValue(value, fractionalIndex)
-
-    // Insert the value into the tree
-    const inserted = this.#tree.set(value, insertedElem, false)
-    if (!inserted) {
-      // The value was already present in the tree
-      // ignore this insertions since we don't support overwrites!
-      return result
-    }
-
-    if (
-      this.#topKStart === this.#topKEnd ||
-      this.#tree.size - 1 < this.#topKStart
-    ) {
-      // The window is empty, or there aren't enough rows to reach its offset.
-      return result
-    }
-
-    if (this.#topKFirstElem) {
-      // We have a topK containing at least 1 element
-      if (this.#comparator(value, getValue(this.#topKFirstElem)) < 0) {
-        // The element was inserted before the topK
-        // so it moves the element that is right before the topK into the topK
-        const firstElem = getValue(this.#topKFirstElem)
-        const [, newFirstElem] = this.#tree.nextLowerPair(firstElem)!
-        this.#topKFirstElem = newFirstElem
-        result.moveIn = this.#topKFirstElem
-      } else if (
-        !this.#topKLastElem ||
-        this.#comparator(value, getValue(this.#topKLastElem)) < 0
-      ) {
-        // The element was inserted within the topK
-        result.moveIn = insertedElem
-      }
-
-      if (
-        this.#topKLastElem &&
-        this.#comparator(value, getValue(this.#topKLastElem)) < 0
-      ) {
-        // The element was inserted before or within the topK
-        // the newly inserted element pushes the last element of the topK out of the topK
-        // so the one before that becomes the new last element of the topK
-        const lastElem = this.#topKLastElem
-        const lastValue = getValue(lastElem)
-        const [, newLastElem] = this.#tree.nextLowerPair(lastValue)!
-        this.#topKLastElem = newLastElem
-        result.moveOut = lastElem
-      }
-    }
-
-    // If the tree has as many elements as the offset (i.e. #topKStart)
-    // then the insertion shifted the elements 1 position to the right
-    // and the last element in the tree is now the first element of the topK
-    if (this.#tree.size - 1 === this.#topKStart) {
-      const topKFirstKey = this.#tree.maxKey()!
-      this.#topKFirstElem = this.#tree.get(topKFirstKey)!
-      result.moveIn = this.#topKFirstElem
-    }
-
-    // By inserting this new element we now have a complete topK
-    // store the last element of the topK
-    if (this.#tree.size === this.#topKEnd) {
-      const topKLastKey = this.#tree.maxKey()!
-      this.#topKLastElem = this.#tree.get(topKLastKey)!
-    }
-
-    return result
-  }
-
-  delete(value: V): TopKChanges<V> {
-    const result: TopKChanges<V> = { moveIn: null, moveOut: null }
-
-    const deletedElem = this.#tree.get(value)
-    const deleted = this.#tree.delete(value)
-    if (!deleted) {
-      return result
-    }
-
-    if (!this.#topKFirstElem) {
-      // We didn't have a topK before the delete
-      // so we still can't have a topK after the delete
-      return result
-    }
-
-    if (this.#comparator(value, getValue(this.#topKFirstElem)) <= 0) {
-      // Deleting at or before the first selected value advances that boundary.
-      result.moveOut = this.#topKFirstElem
-      // the element that was right after the first element of the topK
-      // is now the new first element of the topK
-      const firstElem = getValue(this.#topKFirstElem)
-      const [, newFirstElem] = this.#tree.nextHigherPair(firstElem) ?? [
-        null,
-        null,
-      ]
-      this.#topKFirstElem = newFirstElem
-    } else if (
-      !this.#topKLastElem ||
-      // TODO: if on equal order the element is inserted *after* the already existing one
-      //       then this check should become < 0
-      this.#comparator(value, getValue(this.#topKLastElem)) <= 0
-    ) {
-      // The element we deleted was within the topK
-      // so we need to signal that that element is no longer in the topK
-      result.moveOut = deletedElem!
-    }
-
-    if (
-      this.#topKLastElem &&
-      // TODO: if on equal order the element is inserted *after* the already existing one
-      //       then this check should become < 0
-      this.#comparator(value, getValue(this.#topKLastElem)) <= 0
-    ) {
-      // The element we deleted was before or within the topK
-      // So the first element after the topK moved one position to the left
-      // and thus falls into the topK now
-      const lastElem = this.#topKLastElem
-      const lastValue = getValue(lastElem)
-      const [, newLastElem] = this.#tree.nextHigherPair(lastValue) ?? [
-        null,
-        null,
-      ]
-      this.#topKLastElem = newLastElem
-      if (newLastElem) {
-        result.moveIn = newLastElem
-      }
-    }
-
-    return result
-  }
-}
-
-/**
- * Operator for fractional indexed topK operations
- * This operator maintains fractional indices for sorted elements
- * and only updates indices when elements move position
- */
-export class TopKWithFractionalIndexBTreeOperator<
-  K extends string | number,
-  T,
-> extends TopKWithFractionalIndexOperator<K, T> {
-  protected override createTopK(
-    offset: number,
-    limit: number,
-    comparator: (a: [K, T], b: [K, T]) => number,
-  ): TopK<[K, T]> {
-    if (BTree === undefined) {
-      throw new Error(
-        `B+ tree not loaded. You need to call loadBTree() before using TopKWithFractionalIndexBTreeOperator.`,
-      )
-    }
-    return new TopKTree(offset, limit, comparator)
-  }
-}
-
-/**
- * Limits the number of results based on a comparator, with optional offset.
- * This works on a keyed stream, where the key is the first element of the tuple.
- * The ordering is within a key group, i.e. elements are sorted within a key group
- * and the limit + offset is applied to that sorted group.
- * To order the entire stream, key by the same value for all elements such as null.
- *
- * Uses fractional indexing to minimize the number of changes when elements move positions.
- * Each element is assigned a fractional index that is lexicographically sortable.
- * When elements move, only the indices of the moved elements are updated, not all elements.
- *
- * @param comparator - A function that compares two elements
- * @param options - An optional object containing limit and offset properties
- * @returns A piped operator that orders the elements and limits the number of results
- */
-export function topKWithFractionalIndexBTree<KType extends string | number, T>(
-  comparator: (a: T, b: T) => number,
-  options?: TopKWithFractionalIndexOptions,
-): PipedOperator<[KType, T], [KType, IndexedValue<T>]> {
-  const opts = options || {}
-
-  if (BTree === undefined) {
-    throw new Error(
-      `B+ tree not loaded. You need to call loadBTree() before using topKWithFractionalIndexBTree.`,
-    )
-  }
-
-  return (
-    stream: IStreamBuilder<[KType, T]>,
-  ): IStreamBuilder<[KType, IndexedValue<T>]> => {
-    const output = new StreamBuilder<[KType, IndexedValue<T>]>(
-      stream.graph,
-      new DifferenceStreamWriter<[KType, IndexedValue<T>]>(),
-    )
-    const operator = new TopKWithFractionalIndexBTreeOperator<KType, T>(
-      stream.graph.getNextOperatorId(),
-      stream.connectReader(),
-      output.writer,
-      comparator,
-      opts,
-    )
-    stream.graph.addOperator(operator)
-    return output
-  }
-}