@tanstack/db-ivm 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/cjs/indexes.cjs +243 -85
  2. package/dist/cjs/indexes.cjs.map +1 -1
  3. package/dist/cjs/indexes.d.cts +58 -7
  4. package/dist/cjs/operators/distinct.cjs +2 -2
  5. package/dist/cjs/operators/distinct.cjs.map +1 -1
  6. package/dist/cjs/operators/distinct.d.cts +6 -4
  7. package/dist/cjs/operators/groupBy.cjs +12 -10
  8. package/dist/cjs/operators/groupBy.cjs.map +1 -1
  9. package/dist/cjs/operators/groupBy.d.cts +7 -4
  10. package/dist/cjs/utils.cjs +0 -12
  11. package/dist/cjs/utils.cjs.map +1 -1
  12. package/dist/esm/indexes.d.ts +58 -7
  13. package/dist/esm/indexes.js +243 -85
  14. package/dist/esm/indexes.js.map +1 -1
  15. package/dist/esm/operators/distinct.d.ts +6 -4
  16. package/dist/esm/operators/distinct.js +2 -2
  17. package/dist/esm/operators/distinct.js.map +1 -1
  18. package/dist/esm/operators/groupBy.d.ts +7 -4
  19. package/dist/esm/operators/groupBy.js +12 -10
  20. package/dist/esm/operators/groupBy.js.map +1 -1
  21. package/dist/esm/utils.js +1 -13
  22. package/dist/esm/utils.js.map +1 -1
  23. package/package.json +1 -1
  24. package/src/indexes.ts +365 -90
  25. package/src/operators/distinct.ts +13 -8
  26. package/src/operators/groupBy.ts +38 -18
  27. package/dist/cjs/hashIndex.cjs +0 -83
  28. package/dist/cjs/hashIndex.cjs.map +0 -1
  29. package/dist/cjs/hashIndex.d.cts +0 -25
  30. package/dist/cjs/valueIndex.cjs +0 -74
  31. package/dist/cjs/valueIndex.cjs.map +0 -1
  32. package/dist/cjs/valueIndex.d.cts +0 -17
  33. package/dist/esm/hashIndex.d.ts +0 -25
  34. package/dist/esm/hashIndex.js +0 -83
  35. package/dist/esm/hashIndex.js.map +0 -1
  36. package/dist/esm/valueIndex.d.ts +0 -17
  37. package/dist/esm/valueIndex.js +0 -74
  38. package/dist/esm/valueIndex.js.map +0 -1
  39. package/src/hashIndex.ts +0 -94
  40. package/src/valueIndex.ts +0 -78
package/src/indexes.ts CHANGED
@@ -1,60 +1,238 @@
1
+ /**
2
+ * # Optimized Index Data Structure
3
+ *
4
+ * Multi-level index that adapts storage strategy based on data patterns to minimize memory
5
+ * usage, eliminate wasteful lookups, and avoid hashing whenever possible.
6
+ *
7
+ * ## Storage Strategy
8
+ *
9
+ * **Single value**: `IndexMap['key'] → [value, multiplicity]` (no hashing needed)
10
+ *
11
+ * **Multiple unprefixed values**: Direct ValueMap (avoids NO_PREFIX lookup)
12
+ * ```
13
+ * IndexMap['key'] → ValueMap { hash(value1) → [value1, mult1], ... }
14
+ * ```
15
+ *
16
+ * **Values with prefixes**: PrefixMap uses prefix keys directly (no hashing)
17
+ * ```
18
+ * IndexMap['key'] → PrefixMap { 'prefix1' → [value1, mult1], NO_PREFIX → ValueMap{...} }
19
+ * ```
20
+ *
21
+ * **Multiple values per prefix**: ValueMap within PrefixMap (hash only suffixes)
22
+ * ```
23
+ * PrefixMap['prefix'] → ValueMap { hash(suffix1) → [full_value1, mult1], ... }
24
+ * ```
25
+ *
26
+ * ## Dynamic Evolution
27
+ *
28
+ * Structure automatically evolves as data is added:
29
+ * - Single → ValueMap (when both values unprefixed)
30
+ * - Single → PrefixMap (when at least one prefixed)
31
+ * - ValueMap → PrefixMap (adding prefixed value to unprefixed)
32
+ *
33
+ * Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'
34
+ */
35
+
1
36
  import { MultiSet } from "./multiset.js"
2
- import { HashIndex } from "./hashIndex.js"
3
- import { ValueIndex } from "./valueIndex.js"
4
- import { concatIterable, mapIterable } from "./utils.js"
37
+ import { hash } from "./hashing/index.js"
38
+ import type { Hash } from "./hashing/index.js"
39
+
40
+ // We use a symbol to represent the absence of a prefix, unprefixed values a stored
41
+ // against this key.
42
+ const NO_PREFIX = Symbol(`NO_PREFIX`)
43
+ type NO_PREFIX = typeof NO_PREFIX
44
+
45
+ // A single value is a tuple of the value and the multiplicity.
46
+ type SingleValue<TValue> = [TValue, number]
47
+
48
+ // Base map type for the index. Stores single values, prefix maps, or value maps against a key.
49
+ type IndexMap<TKey, TValue, TPrefix> = Map<
50
+ TKey,
51
+ SingleValue<TValue> | PrefixMap<TValue, TPrefix> | ValueMap<TValue>
52
+ >
53
+
54
+ // Second level map type for the index, stores single values or value maps against a prefix.
55
+ class PrefixMap<TValue, TPrefix> extends Map<
56
+ TPrefix | NO_PREFIX,
57
+ SingleValue<TValue> | ValueMap<TValue>
58
+ > {
59
+ /**
60
+ * Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.
61
+ */
62
+ addValue(value: TValue, multiplicity: number): boolean {
63
+ if (multiplicity === 0) return this.size === 0
64
+
65
+ const prefix = getPrefix<TValue, TPrefix>(value)
66
+ const valueMapOrSingleValue = this.get(prefix)
67
+
68
+ if (isSingleValue(valueMapOrSingleValue)) {
69
+ const [currentValue, currentMultiplicity] = valueMapOrSingleValue
70
+ const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
71
+
72
+ if (currentPrefix !== prefix) {
73
+ throw new Error(`Mismatching prefixes, this should never happen`)
74
+ }
75
+
76
+ if (currentValue === value || hash(currentValue) === hash(value)) {
77
+ // Same value, update multiplicity
78
+ const newMultiplicity = currentMultiplicity + multiplicity
79
+ if (newMultiplicity === 0) {
80
+ this.delete(prefix)
81
+ } else {
82
+ this.set(prefix, [value, newMultiplicity])
83
+ }
84
+ } else {
85
+ // Different suffixes, need to create ValueMap
86
+ const valueMap = new ValueMap<TValue>()
87
+ valueMap.set(hash(currentValue), valueMapOrSingleValue)
88
+ valueMap.set(hash(value), [value, multiplicity])
89
+ this.set(prefix, valueMap)
90
+ }
91
+ } else if (valueMapOrSingleValue === undefined) {
92
+ // No existing value for this prefix
93
+ this.set(prefix, [value, multiplicity])
94
+ } else {
95
+ // Existing ValueMap
96
+ const isEmpty = valueMapOrSingleValue.addValue(value, multiplicity)
97
+ if (isEmpty) {
98
+ this.delete(prefix)
99
+ }
100
+ }
101
+
102
+ return this.size === 0
103
+ }
104
+ }
105
+
106
+ // Third level map type for the index, stores single values or value maps against a hash.
107
+ class ValueMap<TValue> extends Map<Hash, [TValue, number]> {
108
+ /**
109
+ * Add a value to the ValueMap. Returns true if the map becomes empty after the operation.
110
+ * @param value - The full value to store
111
+ * @param multiplicity - The multiplicity to add
112
+ * @param hashKey - Optional hash key to use instead of hashing the full value (used when in PrefixMap context)
113
+ */
114
+ addValue(value: TValue, multiplicity: number): boolean {
115
+ if (multiplicity === 0) return this.size === 0
116
+
117
+ const key = hash(value)
118
+ const currentValue = this.get(key)
119
+
120
+ if (currentValue) {
121
+ const [, currentMultiplicity] = currentValue
122
+ const newMultiplicity = currentMultiplicity + multiplicity
123
+ if (newMultiplicity === 0) {
124
+ this.delete(key)
125
+ } else {
126
+ this.set(key, [value, newMultiplicity])
127
+ }
128
+ } else {
129
+ this.set(key, [value, multiplicity])
130
+ }
131
+
132
+ return this.size === 0
133
+ }
134
+ }
5
135
 
6
136
  /**
7
137
  * A map from a difference collection trace's keys -> (value, multiplicities) that changed.
8
138
  * Used in operations like join and reduce where the operation needs to
9
139
  * exploit the key-value structure of the data to run efficiently.
10
140
  */
11
- export class Index<K, V> {
141
+ export class Index<TKey, TValue, TPrefix = any> {
12
142
  /*
13
- * This is a hybrid Index that composes a ValueIndex and a HashIndex.
14
- * Keys that have only one value are stored in the ValueIndex.
15
- * Keys that have multiple values are stored in the HashIndex, the hash distinguishes between the values.
16
- * This reduces the amount of hashes we need to compute since often times only a small portion of the keys are updated
17
- * so we don't have to hash the keys that are never updated.
18
- *
19
- * Note: The `valueIndex` and `hashIndex` have disjoint keys.
20
- * When a key that has only one value gets a new distinct value,
21
- * it is added to the `hashIndex` and removed from the `valueIndex` and vice versa.
143
+ * This index maintains a nested map of keys -> (value, multiplicities), where:
144
+ * - initially the values are stored against the key as a single value tuple
145
+ * - when a key gets additional values, the values are stored against the key in a
146
+ * prefix map
147
+ * - the prefix is extract where possible from values that are structured as
148
+ * [rowPrimaryKey, rowValue], as they are in the Tanstack DB query pipeline.
149
+ * - only when there are multiple values for a given prefix do we fall back to a
150
+ * hash to identify identical values, storing them in a third level value map.
22
151
  */
23
- #valueIndex: ValueIndex<K, V>
24
- #hashIndex: HashIndex<K, V>
152
+ #inner: IndexMap<TKey, TValue, TPrefix>
25
153
 
26
154
  constructor() {
27
- this.#valueIndex = new ValueIndex<K, V>()
28
- this.#hashIndex = new HashIndex<K, V>()
155
+ this.#inner = new Map()
29
156
  }
30
157
 
158
+ /**
159
+ * This method returns a string representation of the index.
160
+ * @param indent - Whether to indent the string representation.
161
+ * @returns A string representation of the index.
162
+ */
31
163
  toString(indent = false): string {
32
- return `Index(\n ${this.#valueIndex.toString(indent)},\n ${this.#hashIndex.toString(indent)}\n)`
164
+ return `Index(${JSON.stringify(
165
+ [...this.entries()],
166
+ undefined,
167
+ indent ? 2 : undefined
168
+ )})`
33
169
  }
34
170
 
35
- get(key: K): Array<[V, number]> {
36
- if (this.#valueIndex.has(key)) {
37
- return [this.#valueIndex.get(key)!]
38
- }
39
- return this.#hashIndex.get(key)
171
+ /**
172
+ * The size of the index.
173
+ */
174
+ get size(): number {
175
+ return this.#inner.size
176
+ }
177
+
178
+ /**
179
+ * This method checks if the index has a given key.
180
+ * @param key - The key to check.
181
+ * @returns True if the index has the key, false otherwise.
182
+ */
183
+ has(key: TKey): boolean {
184
+ return this.#inner.has(key)
40
185
  }
41
186
 
42
- getMultiplicity(key: K, value: V): number {
43
- if (this.#valueIndex.has(key)) {
44
- return this.#valueIndex.getMultiplicity(key)
187
+ /**
188
+ * This method returns all values for a given key.
189
+ * @param key - The key to get the values for.
190
+ * @returns An array of value tuples [value, multiplicity].
191
+ */
192
+ get(key: TKey): Array<[TValue, number]> {
193
+ return [...this.getIterator(key)]
194
+ }
195
+
196
+ /**
197
+ * This method returns an iterator over all values for a given key.
198
+ * @param key - The key to get the values for.
199
+ * @returns An iterator of value tuples [value, multiplicity].
200
+ */
201
+ *getIterator(key: TKey): Iterable<[TValue, number]> {
202
+ const mapOrSingleValue = this.#inner.get(key)
203
+ if (isSingleValue(mapOrSingleValue)) {
204
+ yield mapOrSingleValue
205
+ } else if (mapOrSingleValue === undefined) {
206
+ return
207
+ } else if (mapOrSingleValue instanceof ValueMap) {
208
+ // Direct ValueMap - all values have NO_PREFIX
209
+ for (const valueTuple of mapOrSingleValue.values()) {
210
+ yield valueTuple
211
+ }
212
+ } else {
213
+ // PrefixMap - iterate through all prefixes
214
+ for (const singleValueOrValueMap of mapOrSingleValue.values()) {
215
+ if (isSingleValue(singleValueOrValueMap)) {
216
+ yield singleValueOrValueMap
217
+ } else {
218
+ for (const valueTuple of singleValueOrValueMap.values()) {
219
+ yield valueTuple
220
+ }
221
+ }
222
+ }
45
223
  }
46
- return this.#hashIndex.getMultiplicity(key, value)
47
224
  }
48
225
 
49
226
  /**
50
227
  * This returns an iterator that iterates over all key-value pairs.
51
228
  * @returns An iterable of all key-value pairs (and their multiplicities) in the index.
52
229
  */
53
- #entries(): Iterable<[K, [V, number]]> {
54
- return concatIterable(
55
- this.#valueIndex.entries(),
56
- this.#hashIndex.entriesIterator()
57
- )
230
+ *entries(): Iterable<[TKey, [TValue, number]]> {
231
+ for (const key of this.#inner.keys()) {
232
+ for (const valueTuple of this.getIterator(key)) {
233
+ yield [key, valueTuple]
234
+ }
235
+ }
58
236
  }
59
237
 
60
238
  /**
@@ -63,90 +241,156 @@ export class Index<K, V> {
63
241
  * It returns an iterator that you can use if you need to iterate over the values for a given key.
64
242
  * @returns An iterator of all *keys* in the index and their corresponding value iterator.
65
243
  */
66
- *#entriesIterators(): Iterable<[K, Iterable<[V, number]>]> {
67
- for (const [key, [value, multiplicity]] of this.#valueIndex.entries()) {
68
- yield [key, new Map<V, number>([[value, multiplicity]])]
69
- }
70
- for (const [key, valueMap] of this.#hashIndex.entries()) {
71
- yield [
72
- key,
73
- mapIterable(valueMap, ([_hash, [value, multiplicity]]) => [
74
- value,
75
- multiplicity,
76
- ]),
77
- ]
244
+ *entriesIterators(): Iterable<[TKey, Iterable<[TValue, number]>]> {
245
+ for (const key of this.#inner.keys()) {
246
+ yield [key, this.getIterator(key)]
78
247
  }
79
248
  }
80
249
 
81
- has(key: K): boolean {
82
- return this.#valueIndex.has(key) || this.#hashIndex.has(key)
83
- }
250
+ /**
251
+ * This method adds a value to the index.
252
+ * @param key - The key to add the value to.
253
+ * @param valueTuple - The value tuple [value, multiplicity] to add to the index.
254
+ */
255
+ addValue(key: TKey, valueTuple: SingleValue<TValue>) {
256
+ const [value, multiplicity] = valueTuple
257
+ // If the multiplicity is 0, do nothing
258
+ if (multiplicity === 0) return
84
259
 
85
- get size(): number {
86
- return this.#valueIndex.size + this.#hashIndex.size
87
- }
260
+ const mapOrSingleValue = this.#inner.get(key)
88
261
 
89
- addValue(key: K, value: [V, number]): void {
90
- const containedInValueIndex = this.#valueIndex.has(key)
91
- const containedInHashIndex = this.#hashIndex.has(key)
262
+ if (mapOrSingleValue === undefined) {
263
+ // First value for this key
264
+ this.#inner.set(key, valueTuple)
265
+ return
266
+ }
92
267
 
93
- if (containedInHashIndex && containedInValueIndex) {
94
- throw new Error(
95
- `Key ${key} is contained in both the value index and the hash index. This should never happen because they should have disjoint keysets.`
268
+ if (isSingleValue(mapOrSingleValue)) {
269
+ // Handle transition from single value to map
270
+ this.#handleSingleValueTransition(
271
+ key,
272
+ mapOrSingleValue,
273
+ value,
274
+ multiplicity
96
275
  )
276
+ return
97
277
  }
98
278
 
99
- if (!containedInValueIndex && !containedInHashIndex) {
100
- // This is the first time we see the key
101
- // Add it to the value index
102
- this.#valueIndex.addValue(key, value)
103
- return
279
+ if (mapOrSingleValue instanceof ValueMap) {
280
+ // Handle existing ValueMap
281
+ const prefix = getPrefix<TValue, TPrefix>(value)
282
+ if (prefix !== NO_PREFIX) {
283
+ // Convert ValueMap to PrefixMap since we have a prefixed value
284
+ const prefixMap = new PrefixMap<TValue, TPrefix>()
285
+ prefixMap.set(NO_PREFIX, mapOrSingleValue)
286
+ prefixMap.set(prefix, valueTuple)
287
+ this.#inner.set(key, prefixMap)
288
+ } else {
289
+ // Add to existing ValueMap
290
+ const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
291
+ if (isEmpty) {
292
+ this.#inner.delete(key)
293
+ }
294
+ }
295
+ } else {
296
+ // Handle existing PrefixMap
297
+ const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
298
+ if (isEmpty) {
299
+ this.#inner.delete(key)
300
+ }
104
301
  }
302
+ }
105
303
 
106
- if (containedInValueIndex) {
107
- // This key is already in the value index
108
- // It could be that it's the same value or a different one
109
- // If it's a different value we will need to remove the key from the value index
110
- // and add the key and its two values to the hash index
111
- try {
112
- this.#valueIndex.addValue(key, value)
113
- } catch {
114
- // This is a different value, need to move the key to the hash index
115
- const existingValue = this.#valueIndex.get(key)!
116
- this.#valueIndex.delete(key)
117
- this.#hashIndex.addValue(key, existingValue)
118
- this.#hashIndex.addValue(key, value)
304
+ /**
305
+ * Handle the transition from a single value to either a ValueMap or PrefixMap
306
+ */
307
+ #handleSingleValueTransition(
308
+ key: TKey,
309
+ currentSingleValue: SingleValue<TValue>,
310
+ newValue: TValue,
311
+ multiplicity: number
312
+ ) {
313
+ const [currentValue, currentMultiplicity] = currentSingleValue
314
+
315
+ // Check for exact same value (reference equality)
316
+ if (currentValue === newValue) {
317
+ const newMultiplicity = currentMultiplicity + multiplicity
318
+ if (newMultiplicity === 0) {
319
+ this.#inner.delete(key)
320
+ } else {
321
+ this.#inner.set(key, [newValue, newMultiplicity])
119
322
  }
120
323
  return
121
324
  }
122
325
 
123
- if (containedInHashIndex) {
124
- // This key is already in the hash index so it already has two or more values.
125
- // However, this new value and multiplicity could cause an existing value to be removed
126
- // and lead to the key having only a single value in which case we need to move it back to the value index
127
- const singleRemainingValue = this.#hashIndex.addValue(key, value)
128
- if (singleRemainingValue) {
129
- // The key only has a single remaining value so we need to move it back to the value index
130
- this.#hashIndex.delete(key)
131
- this.#valueIndex.addValue(key, singleRemainingValue)
326
+ // Get prefixes for both values
327
+ const newPrefix = getPrefix<TValue, TPrefix>(newValue)
328
+ const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
329
+
330
+ // Check if they're the same value by prefix/suffix comparison
331
+ if (
332
+ currentPrefix === newPrefix &&
333
+ (currentValue === newValue || hash(currentValue) === hash(newValue))
334
+ ) {
335
+ const newMultiplicity = currentMultiplicity + multiplicity
336
+ if (newMultiplicity === 0) {
337
+ this.#inner.delete(key)
338
+ } else {
339
+ this.#inner.set(key, [newValue, newMultiplicity])
132
340
  }
133
341
  return
134
342
  }
343
+
344
+ // Different values - choose appropriate map type
345
+ if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) {
346
+ // Both have NO_PREFIX, use ValueMap directly
347
+ const valueMap = new ValueMap<TValue>()
348
+ valueMap.set(hash(currentValue), currentSingleValue)
349
+ valueMap.set(hash(newValue), [newValue, multiplicity])
350
+ this.#inner.set(key, valueMap)
351
+ } else {
352
+ // At least one has a prefix, use PrefixMap
353
+ const prefixMap = new PrefixMap<TValue, TPrefix>()
354
+
355
+ if (currentPrefix === newPrefix) {
356
+ // Same prefix, different suffixes - need ValueMap within PrefixMap
357
+ const valueMap = new ValueMap<TValue>()
358
+ valueMap.set(hash(currentValue), currentSingleValue)
359
+ valueMap.set(hash(newValue), [newValue, multiplicity])
360
+ prefixMap.set(currentPrefix, valueMap)
361
+ } else {
362
+ // Different prefixes - store as separate single values
363
+ prefixMap.set(currentPrefix, currentSingleValue)
364
+ prefixMap.set(newPrefix, [newValue, multiplicity])
365
+ }
366
+
367
+ this.#inner.set(key, prefixMap)
368
+ }
135
369
  }
136
370
 
137
- append(other: Index<K, V>): void {
138
- for (const [key, value] of other.#entries()) {
371
+ /**
372
+ * This method appends another index to the current index.
373
+ * @param other - The index to append to the current index.
374
+ */
375
+ append(other: Index<TKey, TValue>): void {
376
+ for (const [key, value] of other.entries()) {
139
377
  this.addValue(key, value)
140
378
  }
141
379
  }
142
380
 
143
- join<V2>(other: Index<K, V2>): MultiSet<[K, [V, V2]]> {
144
- const result: Array<[[K, [V, V2]], number]> = []
145
-
381
+ /**
382
+ * This method joins two indexes.
383
+ * @param other - The index to join with the current index.
384
+ * @returns A multiset of the joined values.
385
+ */
386
+ join<TValue2>(
387
+ other: Index<TKey, TValue2>
388
+ ): MultiSet<[TKey, [TValue, TValue2]]> {
389
+ const result: Array<[[TKey, [TValue, TValue2]], number]> = []
146
390
  // We want to iterate over the smaller of the two indexes to reduce the
147
391
  // number of operations we need to do.
148
392
  if (this.size <= other.size) {
149
- for (const [key, valueIt] of this.#entriesIterators()) {
393
+ for (const [key, valueIt] of this.entriesIterators()) {
150
394
  if (!other.has(key)) continue
151
395
  const otherValues = other.get(key)
152
396
  for (const [val1, mul1] of valueIt) {
@@ -158,7 +402,7 @@ export class Index<K, V> {
158
402
  }
159
403
  }
160
404
  } else {
161
- for (const [key, otherValueIt] of other.#entriesIterators()) {
405
+ for (const [key, otherValueIt] of other.entriesIterators()) {
162
406
  if (!this.has(key)) continue
163
407
  const values = this.get(key)
164
408
  for (const [val2, mul2] of otherValueIt) {
@@ -174,3 +418,34 @@ export class Index<K, V> {
174
418
  return new MultiSet(result)
175
419
  }
176
420
  }
421
+
422
+ /**
423
+ * This function extracts the prefix from a value.
424
+ * @param value - The value to extract the prefix from.
425
+ * @returns The prefix and the suffix.
426
+ */
427
+ function getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {
428
+ // If the value is an array and the first element is a string or number, then the
429
+ // first element is the prefix. This is used to distinguish between values without
430
+ // the need for hashing unless there are multiple values for the same prefix.
431
+ if (
432
+ Array.isArray(value) &&
433
+ (typeof value[0] === `string` ||
434
+ typeof value[0] === `number` ||
435
+ typeof value[0] === `bigint`)
436
+ ) {
437
+ return value[0] as TPrefix
438
+ }
439
+ return NO_PREFIX
440
+ }
441
+
442
+ /**
443
+ * This function checks if a value is a single value.
444
+ * @param value - The value to check.
445
+ * @returns True if the value is a single value, false otherwise.
446
+ */
447
+ function isSingleValue<TValue>(
448
+ value: SingleValue<TValue> | unknown
449
+ ): value is SingleValue<TValue> {
450
+ return Array.isArray(value)
451
+ }
@@ -4,21 +4,25 @@ import { hash } from "../hashing/index.js"
4
4
  import { MultiSet } from "../multiset.js"
5
5
  import type { Hash } from "../hashing/index.js"
6
6
  import type { DifferenceStreamReader } from "../graph.js"
7
- import type { IStreamBuilder } from "../types.js"
7
+ import type { IStreamBuilder, KeyValue } from "../types.js"
8
8
 
9
9
  type Multiplicity = number
10
10
 
11
+ type GetValue<T> = T extends KeyValue<any, infer V> ? V : never
12
+
11
13
  /**
12
14
  * Operator that removes duplicates
13
15
  */
14
- export class DistinctOperator<T> extends UnaryOperator<T> {
16
+ export class DistinctOperator<
17
+ T extends KeyValue<any, any>,
18
+ > extends UnaryOperator<T, KeyValue<number, GetValue<T>>> {
15
19
  #by: (value: T) => any
16
20
  #values: Map<Hash, Multiplicity> // keeps track of the number of times each value has been seen
17
21
 
18
22
  constructor(
19
23
  id: number,
20
24
  input: DifferenceStreamReader<T>,
21
- output: DifferenceStreamWriter<T>,
25
+ output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,
22
26
  by: (value: T) => any = (value: T) => value
23
27
  ) {
24
28
  super(id, input, output)
@@ -39,12 +43,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
39
43
  this.#values.get(hashedValue) ??
40
44
  0
41
45
  const newMultiplicity = oldMultiplicity + diff
42
-
43
46
  updatedValues.set(hashedValue, [newMultiplicity, value])
44
47
  }
45
48
  }
46
49
 
47
- const result: Array<[T, number]> = []
50
+ const result: Array<[KeyValue<number, GetValue<T>>, number]> = []
48
51
 
49
52
  // Check which values became visible or disappeared
50
53
  for (const [
@@ -62,11 +65,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
62
65
  if (oldMultiplicity <= 0 && newMultiplicity > 0) {
63
66
  // The value wasn't present in the stream
64
67
  // but with this change it is now present in the stream
65
- result.push([value, 1])
68
+ result.push([[hash(this.#by(value)), value[1]], 1])
66
69
  } else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
67
70
  // The value was present in the stream
68
71
  // but with this change it is no longer present in the stream
69
- result.push([value, -1])
72
+ result.push([[hash(this.#by(value)), value[1]], -1])
70
73
  }
71
74
  }
72
75
 
@@ -79,7 +82,9 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
79
82
  /**
80
83
  * Removes duplicate values
81
84
  */
82
- export function distinct<T>(by: (value: T) => any = (value: T) => value) {
85
+ export function distinct<T extends KeyValue<any, any>>(
86
+ by: (value: T) => any = (value: T) => value
87
+ ) {
83
88
  return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
84
89
  const output = new StreamBuilder<T>(
85
90
  stream.graph,
@@ -210,44 +210,64 @@ export function avg<T>(
210
210
  }
211
211
  }
212
212
 
213
+ type CanMinMax = number | Date | bigint
214
+
213
215
  /**
214
216
  * Creates a min aggregate function that computes the minimum value in a group
215
- * @param valueExtractor Function to extract a numeric value from each data entry
217
+ * @param valueExtractor Function to extract a comparable value from each data entry
216
218
  */
217
- export function min<T>(
218
- valueExtractor: (value: T) => number = (v) => v as unknown as number
219
- ): AggregateFunction<T, number, number> {
219
+ export function min<T extends CanMinMax>(): AggregateFunction<
220
+ T,
221
+ T | undefined,
222
+ T | undefined
223
+ >
224
+ export function min<T, V extends CanMinMax>(
225
+ valueExtractor: (value: T) => V
226
+ ): AggregateFunction<T, V | undefined, V | undefined>
227
+ export function min<T, V extends CanMinMax>(
228
+ valueExtractor?: (value: T) => V
229
+ ): AggregateFunction<T, V | undefined, V | undefined> {
230
+ const extractor = valueExtractor ?? ((v: T) => v as unknown as V)
220
231
  return {
221
- preMap: (data: T) => valueExtractor(data),
222
- reduce: (values: Array<[number, number]>) => {
223
- let minValue = Number.POSITIVE_INFINITY
232
+ preMap: (data: T) => extractor(data),
233
+ reduce: (values) => {
234
+ let minValue: V | undefined
224
235
  for (const [value, _multiplicity] of values) {
225
- if (value < minValue) {
236
+ if (!minValue || (value && value < minValue)) {
226
237
  minValue = value
227
238
  }
228
239
  }
229
- return minValue === Number.POSITIVE_INFINITY ? 0 : minValue
240
+ return minValue
230
241
  },
231
242
  }
232
243
  }
233
244
 
234
245
  /**
235
246
  * Creates a max aggregate function that computes the maximum value in a group
236
- * @param valueExtractor Function to extract a numeric value from each data entry
247
+ * @param valueExtractor Function to extract a comparable value from each data entry
237
248
  */
238
- export function max<T>(
239
- valueExtractor: (value: T) => number = (v) => v as unknown as number
240
- ): AggregateFunction<T, number, number> {
249
+ export function max<T extends CanMinMax>(): AggregateFunction<
250
+ T,
251
+ T | undefined,
252
+ T | undefined
253
+ >
254
+ export function max<T, V extends CanMinMax>(
255
+ valueExtractor: (value: T) => V
256
+ ): AggregateFunction<T, V | undefined, V | undefined>
257
+ export function max<T, V extends CanMinMax>(
258
+ valueExtractor?: (value: T) => V
259
+ ): AggregateFunction<T, V | undefined, V | undefined> {
260
+ const extractor = valueExtractor ?? ((v: T) => v as unknown as V)
241
261
  return {
242
- preMap: (data: T) => valueExtractor(data),
243
- reduce: (values: Array<[number, number]>) => {
244
- let maxValue = Number.NEGATIVE_INFINITY
262
+ preMap: (data: T) => extractor(data),
263
+ reduce: (values) => {
264
+ let maxValue: V | undefined
245
265
  for (const [value, _multiplicity] of values) {
246
- if (value > maxValue) {
266
+ if (!maxValue || (value && value > maxValue)) {
247
267
  maxValue = value
248
268
  }
249
269
  }
250
- return maxValue === Number.NEGATIVE_INFINITY ? 0 : maxValue
270
+ return maxValue
251
271
  },
252
272
  }
253
273
  }