@tanstack/db-ivm 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/indexes.cjs +243 -85
- package/dist/cjs/indexes.cjs.map +1 -1
- package/dist/cjs/indexes.d.cts +58 -7
- package/dist/cjs/operators/distinct.cjs +2 -2
- package/dist/cjs/operators/distinct.cjs.map +1 -1
- package/dist/cjs/operators/distinct.d.cts +6 -4
- package/dist/cjs/operators/groupBy.cjs +12 -10
- package/dist/cjs/operators/groupBy.cjs.map +1 -1
- package/dist/cjs/operators/groupBy.d.cts +7 -4
- package/dist/cjs/utils.cjs +0 -12
- package/dist/cjs/utils.cjs.map +1 -1
- package/dist/esm/indexes.d.ts +58 -7
- package/dist/esm/indexes.js +243 -85
- package/dist/esm/indexes.js.map +1 -1
- package/dist/esm/operators/distinct.d.ts +6 -4
- package/dist/esm/operators/distinct.js +2 -2
- package/dist/esm/operators/distinct.js.map +1 -1
- package/dist/esm/operators/groupBy.d.ts +7 -4
- package/dist/esm/operators/groupBy.js +12 -10
- package/dist/esm/operators/groupBy.js.map +1 -1
- package/dist/esm/utils.js +1 -13
- package/dist/esm/utils.js.map +1 -1
- package/package.json +1 -1
- package/src/indexes.ts +365 -90
- package/src/operators/distinct.ts +13 -8
- package/src/operators/groupBy.ts +38 -18
- package/dist/cjs/hashIndex.cjs +0 -83
- package/dist/cjs/hashIndex.cjs.map +0 -1
- package/dist/cjs/hashIndex.d.cts +0 -25
- package/dist/cjs/valueIndex.cjs +0 -74
- package/dist/cjs/valueIndex.cjs.map +0 -1
- package/dist/cjs/valueIndex.d.cts +0 -17
- package/dist/esm/hashIndex.d.ts +0 -25
- package/dist/esm/hashIndex.js +0 -83
- package/dist/esm/hashIndex.js.map +0 -1
- package/dist/esm/valueIndex.d.ts +0 -17
- package/dist/esm/valueIndex.js +0 -74
- package/dist/esm/valueIndex.js.map +0 -1
- package/src/hashIndex.ts +0 -94
- package/src/valueIndex.ts +0 -78
package/src/indexes.ts
CHANGED
|
@@ -1,60 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* # Optimized Index Data Structure
|
|
3
|
+
*
|
|
4
|
+
* Multi-level index that adapts storage strategy based on data patterns to minimize memory
|
|
5
|
+
* usage, eliminate wasteful lookups, and avoid hashing whenever possible.
|
|
6
|
+
*
|
|
7
|
+
* ## Storage Strategy
|
|
8
|
+
*
|
|
9
|
+
* **Single value**: `IndexMap['key'] → [value, multiplicity]` (no hashing needed)
|
|
10
|
+
*
|
|
11
|
+
* **Multiple unprefixed values**: Direct ValueMap (avoids NO_PREFIX lookup)
|
|
12
|
+
* ```
|
|
13
|
+
* IndexMap['key'] → ValueMap { hash(value1) → [value1, mult1], ... }
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* **Values with prefixes**: PrefixMap uses prefix keys directly (no hashing)
|
|
17
|
+
* ```
|
|
18
|
+
* IndexMap['key'] → PrefixMap { 'prefix1' → [value1, mult1], NO_PREFIX → ValueMap{...} }
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* **Multiple values per prefix**: ValueMap within PrefixMap (hash only suffixes)
|
|
22
|
+
* ```
|
|
23
|
+
* PrefixMap['prefix'] → ValueMap { hash(suffix1) → [full_value1, mult1], ... }
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* ## Dynamic Evolution
|
|
27
|
+
*
|
|
28
|
+
* Structure automatically evolves as data is added:
|
|
29
|
+
* - Single → ValueMap (when both values unprefixed)
|
|
30
|
+
* - Single → PrefixMap (when at least one prefixed)
|
|
31
|
+
* - ValueMap → PrefixMap (adding prefixed value to unprefixed)
|
|
32
|
+
*
|
|
33
|
+
* Prefixes extracted from array values: `['prefix', 'suffix']` → prefix='prefix'
|
|
34
|
+
*/
|
|
35
|
+
|
|
1
36
|
import { MultiSet } from "./multiset.js"
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
37
|
+
import { hash } from "./hashing/index.js"
|
|
38
|
+
import type { Hash } from "./hashing/index.js"
|
|
39
|
+
|
|
40
|
+
// We use a symbol to represent the absence of a prefix, unprefixed values a stored
|
|
41
|
+
// against this key.
|
|
42
|
+
const NO_PREFIX = Symbol(`NO_PREFIX`)
|
|
43
|
+
type NO_PREFIX = typeof NO_PREFIX
|
|
44
|
+
|
|
45
|
+
// A single value is a tuple of the value and the multiplicity.
|
|
46
|
+
type SingleValue<TValue> = [TValue, number]
|
|
47
|
+
|
|
48
|
+
// Base map type for the index. Stores single values, prefix maps, or value maps against a key.
|
|
49
|
+
type IndexMap<TKey, TValue, TPrefix> = Map<
|
|
50
|
+
TKey,
|
|
51
|
+
SingleValue<TValue> | PrefixMap<TValue, TPrefix> | ValueMap<TValue>
|
|
52
|
+
>
|
|
53
|
+
|
|
54
|
+
// Second level map type for the index, stores single values or value maps against a prefix.
|
|
55
|
+
class PrefixMap<TValue, TPrefix> extends Map<
|
|
56
|
+
TPrefix | NO_PREFIX,
|
|
57
|
+
SingleValue<TValue> | ValueMap<TValue>
|
|
58
|
+
> {
|
|
59
|
+
/**
|
|
60
|
+
* Add a value to the PrefixMap. Returns true if the map becomes empty after the operation.
|
|
61
|
+
*/
|
|
62
|
+
addValue(value: TValue, multiplicity: number): boolean {
|
|
63
|
+
if (multiplicity === 0) return this.size === 0
|
|
64
|
+
|
|
65
|
+
const prefix = getPrefix<TValue, TPrefix>(value)
|
|
66
|
+
const valueMapOrSingleValue = this.get(prefix)
|
|
67
|
+
|
|
68
|
+
if (isSingleValue(valueMapOrSingleValue)) {
|
|
69
|
+
const [currentValue, currentMultiplicity] = valueMapOrSingleValue
|
|
70
|
+
const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
|
|
71
|
+
|
|
72
|
+
if (currentPrefix !== prefix) {
|
|
73
|
+
throw new Error(`Mismatching prefixes, this should never happen`)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (currentValue === value || hash(currentValue) === hash(value)) {
|
|
77
|
+
// Same value, update multiplicity
|
|
78
|
+
const newMultiplicity = currentMultiplicity + multiplicity
|
|
79
|
+
if (newMultiplicity === 0) {
|
|
80
|
+
this.delete(prefix)
|
|
81
|
+
} else {
|
|
82
|
+
this.set(prefix, [value, newMultiplicity])
|
|
83
|
+
}
|
|
84
|
+
} else {
|
|
85
|
+
// Different suffixes, need to create ValueMap
|
|
86
|
+
const valueMap = new ValueMap<TValue>()
|
|
87
|
+
valueMap.set(hash(currentValue), valueMapOrSingleValue)
|
|
88
|
+
valueMap.set(hash(value), [value, multiplicity])
|
|
89
|
+
this.set(prefix, valueMap)
|
|
90
|
+
}
|
|
91
|
+
} else if (valueMapOrSingleValue === undefined) {
|
|
92
|
+
// No existing value for this prefix
|
|
93
|
+
this.set(prefix, [value, multiplicity])
|
|
94
|
+
} else {
|
|
95
|
+
// Existing ValueMap
|
|
96
|
+
const isEmpty = valueMapOrSingleValue.addValue(value, multiplicity)
|
|
97
|
+
if (isEmpty) {
|
|
98
|
+
this.delete(prefix)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return this.size === 0
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Third level map type for the index, stores single values or value maps against a hash.
|
|
107
|
+
class ValueMap<TValue> extends Map<Hash, [TValue, number]> {
|
|
108
|
+
/**
|
|
109
|
+
* Add a value to the ValueMap. Returns true if the map becomes empty after the operation.
|
|
110
|
+
* @param value - The full value to store
|
|
111
|
+
* @param multiplicity - The multiplicity to add
|
|
112
|
+
* @param hashKey - Optional hash key to use instead of hashing the full value (used when in PrefixMap context)
|
|
113
|
+
*/
|
|
114
|
+
addValue(value: TValue, multiplicity: number): boolean {
|
|
115
|
+
if (multiplicity === 0) return this.size === 0
|
|
116
|
+
|
|
117
|
+
const key = hash(value)
|
|
118
|
+
const currentValue = this.get(key)
|
|
119
|
+
|
|
120
|
+
if (currentValue) {
|
|
121
|
+
const [, currentMultiplicity] = currentValue
|
|
122
|
+
const newMultiplicity = currentMultiplicity + multiplicity
|
|
123
|
+
if (newMultiplicity === 0) {
|
|
124
|
+
this.delete(key)
|
|
125
|
+
} else {
|
|
126
|
+
this.set(key, [value, newMultiplicity])
|
|
127
|
+
}
|
|
128
|
+
} else {
|
|
129
|
+
this.set(key, [value, multiplicity])
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return this.size === 0
|
|
133
|
+
}
|
|
134
|
+
}
|
|
5
135
|
|
|
6
136
|
/**
|
|
7
137
|
* A map from a difference collection trace's keys -> (value, multiplicities) that changed.
|
|
8
138
|
* Used in operations like join and reduce where the operation needs to
|
|
9
139
|
* exploit the key-value structure of the data to run efficiently.
|
|
10
140
|
*/
|
|
11
|
-
export class Index<
|
|
141
|
+
export class Index<TKey, TValue, TPrefix = any> {
|
|
12
142
|
/*
|
|
13
|
-
* This
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* it is added to the `hashIndex` and removed from the `valueIndex` and vice versa.
|
|
143
|
+
* This index maintains a nested map of keys -> (value, multiplicities), where:
|
|
144
|
+
* - initially the values are stored against the key as a single value tuple
|
|
145
|
+
* - when a key gets additional values, the values are stored against the key in a
|
|
146
|
+
* prefix map
|
|
147
|
+
* - the prefix is extract where possible from values that are structured as
|
|
148
|
+
* [rowPrimaryKey, rowValue], as they are in the Tanstack DB query pipeline.
|
|
149
|
+
* - only when there are multiple values for a given prefix do we fall back to a
|
|
150
|
+
* hash to identify identical values, storing them in a third level value map.
|
|
22
151
|
*/
|
|
23
|
-
#
|
|
24
|
-
#hashIndex: HashIndex<K, V>
|
|
152
|
+
#inner: IndexMap<TKey, TValue, TPrefix>
|
|
25
153
|
|
|
26
154
|
constructor() {
|
|
27
|
-
this.#
|
|
28
|
-
this.#hashIndex = new HashIndex<K, V>()
|
|
155
|
+
this.#inner = new Map()
|
|
29
156
|
}
|
|
30
157
|
|
|
158
|
+
/**
|
|
159
|
+
* This method returns a string representation of the index.
|
|
160
|
+
* @param indent - Whether to indent the string representation.
|
|
161
|
+
* @returns A string representation of the index.
|
|
162
|
+
*/
|
|
31
163
|
toString(indent = false): string {
|
|
32
|
-
return `Index(
|
|
164
|
+
return `Index(${JSON.stringify(
|
|
165
|
+
[...this.entries()],
|
|
166
|
+
undefined,
|
|
167
|
+
indent ? 2 : undefined
|
|
168
|
+
)})`
|
|
33
169
|
}
|
|
34
170
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return this.#
|
|
171
|
+
/**
|
|
172
|
+
* The size of the index.
|
|
173
|
+
*/
|
|
174
|
+
get size(): number {
|
|
175
|
+
return this.#inner.size
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* This method checks if the index has a given key.
|
|
180
|
+
* @param key - The key to check.
|
|
181
|
+
* @returns True if the index has the key, false otherwise.
|
|
182
|
+
*/
|
|
183
|
+
has(key: TKey): boolean {
|
|
184
|
+
return this.#inner.has(key)
|
|
40
185
|
}
|
|
41
186
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
187
|
+
/**
|
|
188
|
+
* This method returns all values for a given key.
|
|
189
|
+
* @param key - The key to get the values for.
|
|
190
|
+
* @returns An array of value tuples [value, multiplicity].
|
|
191
|
+
*/
|
|
192
|
+
get(key: TKey): Array<[TValue, number]> {
|
|
193
|
+
return [...this.getIterator(key)]
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* This method returns an iterator over all values for a given key.
|
|
198
|
+
* @param key - The key to get the values for.
|
|
199
|
+
* @returns An iterator of value tuples [value, multiplicity].
|
|
200
|
+
*/
|
|
201
|
+
*getIterator(key: TKey): Iterable<[TValue, number]> {
|
|
202
|
+
const mapOrSingleValue = this.#inner.get(key)
|
|
203
|
+
if (isSingleValue(mapOrSingleValue)) {
|
|
204
|
+
yield mapOrSingleValue
|
|
205
|
+
} else if (mapOrSingleValue === undefined) {
|
|
206
|
+
return
|
|
207
|
+
} else if (mapOrSingleValue instanceof ValueMap) {
|
|
208
|
+
// Direct ValueMap - all values have NO_PREFIX
|
|
209
|
+
for (const valueTuple of mapOrSingleValue.values()) {
|
|
210
|
+
yield valueTuple
|
|
211
|
+
}
|
|
212
|
+
} else {
|
|
213
|
+
// PrefixMap - iterate through all prefixes
|
|
214
|
+
for (const singleValueOrValueMap of mapOrSingleValue.values()) {
|
|
215
|
+
if (isSingleValue(singleValueOrValueMap)) {
|
|
216
|
+
yield singleValueOrValueMap
|
|
217
|
+
} else {
|
|
218
|
+
for (const valueTuple of singleValueOrValueMap.values()) {
|
|
219
|
+
yield valueTuple
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
45
223
|
}
|
|
46
|
-
return this.#hashIndex.getMultiplicity(key, value)
|
|
47
224
|
}
|
|
48
225
|
|
|
49
226
|
/**
|
|
50
227
|
* This returns an iterator that iterates over all key-value pairs.
|
|
51
228
|
* @returns An iterable of all key-value pairs (and their multiplicities) in the index.
|
|
52
229
|
*/
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
this
|
|
56
|
-
|
|
57
|
-
|
|
230
|
+
*entries(): Iterable<[TKey, [TValue, number]]> {
|
|
231
|
+
for (const key of this.#inner.keys()) {
|
|
232
|
+
for (const valueTuple of this.getIterator(key)) {
|
|
233
|
+
yield [key, valueTuple]
|
|
234
|
+
}
|
|
235
|
+
}
|
|
58
236
|
}
|
|
59
237
|
|
|
60
238
|
/**
|
|
@@ -63,90 +241,156 @@ export class Index<K, V> {
|
|
|
63
241
|
* It returns an iterator that you can use if you need to iterate over the values for a given key.
|
|
64
242
|
* @returns An iterator of all *keys* in the index and their corresponding value iterator.
|
|
65
243
|
*/
|
|
66
|
-
|
|
67
|
-
for (const
|
|
68
|
-
yield [key,
|
|
69
|
-
}
|
|
70
|
-
for (const [key, valueMap] of this.#hashIndex.entries()) {
|
|
71
|
-
yield [
|
|
72
|
-
key,
|
|
73
|
-
mapIterable(valueMap, ([_hash, [value, multiplicity]]) => [
|
|
74
|
-
value,
|
|
75
|
-
multiplicity,
|
|
76
|
-
]),
|
|
77
|
-
]
|
|
244
|
+
*entriesIterators(): Iterable<[TKey, Iterable<[TValue, number]>]> {
|
|
245
|
+
for (const key of this.#inner.keys()) {
|
|
246
|
+
yield [key, this.getIterator(key)]
|
|
78
247
|
}
|
|
79
248
|
}
|
|
80
249
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
250
|
+
/**
|
|
251
|
+
* This method adds a value to the index.
|
|
252
|
+
* @param key - The key to add the value to.
|
|
253
|
+
* @param valueTuple - The value tuple [value, multiplicity] to add to the index.
|
|
254
|
+
*/
|
|
255
|
+
addValue(key: TKey, valueTuple: SingleValue<TValue>) {
|
|
256
|
+
const [value, multiplicity] = valueTuple
|
|
257
|
+
// If the multiplicity is 0, do nothing
|
|
258
|
+
if (multiplicity === 0) return
|
|
84
259
|
|
|
85
|
-
|
|
86
|
-
return this.#valueIndex.size + this.#hashIndex.size
|
|
87
|
-
}
|
|
260
|
+
const mapOrSingleValue = this.#inner.get(key)
|
|
88
261
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
262
|
+
if (mapOrSingleValue === undefined) {
|
|
263
|
+
// First value for this key
|
|
264
|
+
this.#inner.set(key, valueTuple)
|
|
265
|
+
return
|
|
266
|
+
}
|
|
92
267
|
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
|
|
268
|
+
if (isSingleValue(mapOrSingleValue)) {
|
|
269
|
+
// Handle transition from single value to map
|
|
270
|
+
this.#handleSingleValueTransition(
|
|
271
|
+
key,
|
|
272
|
+
mapOrSingleValue,
|
|
273
|
+
value,
|
|
274
|
+
multiplicity
|
|
96
275
|
)
|
|
276
|
+
return
|
|
97
277
|
}
|
|
98
278
|
|
|
99
|
-
if (
|
|
100
|
-
//
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
279
|
+
if (mapOrSingleValue instanceof ValueMap) {
|
|
280
|
+
// Handle existing ValueMap
|
|
281
|
+
const prefix = getPrefix<TValue, TPrefix>(value)
|
|
282
|
+
if (prefix !== NO_PREFIX) {
|
|
283
|
+
// Convert ValueMap to PrefixMap since we have a prefixed value
|
|
284
|
+
const prefixMap = new PrefixMap<TValue, TPrefix>()
|
|
285
|
+
prefixMap.set(NO_PREFIX, mapOrSingleValue)
|
|
286
|
+
prefixMap.set(prefix, valueTuple)
|
|
287
|
+
this.#inner.set(key, prefixMap)
|
|
288
|
+
} else {
|
|
289
|
+
// Add to existing ValueMap
|
|
290
|
+
const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
|
|
291
|
+
if (isEmpty) {
|
|
292
|
+
this.#inner.delete(key)
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
} else {
|
|
296
|
+
// Handle existing PrefixMap
|
|
297
|
+
const isEmpty = mapOrSingleValue.addValue(value, multiplicity)
|
|
298
|
+
if (isEmpty) {
|
|
299
|
+
this.#inner.delete(key)
|
|
300
|
+
}
|
|
104
301
|
}
|
|
302
|
+
}
|
|
105
303
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
304
|
+
/**
|
|
305
|
+
* Handle the transition from a single value to either a ValueMap or PrefixMap
|
|
306
|
+
*/
|
|
307
|
+
#handleSingleValueTransition(
|
|
308
|
+
key: TKey,
|
|
309
|
+
currentSingleValue: SingleValue<TValue>,
|
|
310
|
+
newValue: TValue,
|
|
311
|
+
multiplicity: number
|
|
312
|
+
) {
|
|
313
|
+
const [currentValue, currentMultiplicity] = currentSingleValue
|
|
314
|
+
|
|
315
|
+
// Check for exact same value (reference equality)
|
|
316
|
+
if (currentValue === newValue) {
|
|
317
|
+
const newMultiplicity = currentMultiplicity + multiplicity
|
|
318
|
+
if (newMultiplicity === 0) {
|
|
319
|
+
this.#inner.delete(key)
|
|
320
|
+
} else {
|
|
321
|
+
this.#inner.set(key, [newValue, newMultiplicity])
|
|
119
322
|
}
|
|
120
323
|
return
|
|
121
324
|
}
|
|
122
325
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
326
|
+
// Get prefixes for both values
|
|
327
|
+
const newPrefix = getPrefix<TValue, TPrefix>(newValue)
|
|
328
|
+
const currentPrefix = getPrefix<TValue, TPrefix>(currentValue)
|
|
329
|
+
|
|
330
|
+
// Check if they're the same value by prefix/suffix comparison
|
|
331
|
+
if (
|
|
332
|
+
currentPrefix === newPrefix &&
|
|
333
|
+
(currentValue === newValue || hash(currentValue) === hash(newValue))
|
|
334
|
+
) {
|
|
335
|
+
const newMultiplicity = currentMultiplicity + multiplicity
|
|
336
|
+
if (newMultiplicity === 0) {
|
|
337
|
+
this.#inner.delete(key)
|
|
338
|
+
} else {
|
|
339
|
+
this.#inner.set(key, [newValue, newMultiplicity])
|
|
132
340
|
}
|
|
133
341
|
return
|
|
134
342
|
}
|
|
343
|
+
|
|
344
|
+
// Different values - choose appropriate map type
|
|
345
|
+
if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) {
|
|
346
|
+
// Both have NO_PREFIX, use ValueMap directly
|
|
347
|
+
const valueMap = new ValueMap<TValue>()
|
|
348
|
+
valueMap.set(hash(currentValue), currentSingleValue)
|
|
349
|
+
valueMap.set(hash(newValue), [newValue, multiplicity])
|
|
350
|
+
this.#inner.set(key, valueMap)
|
|
351
|
+
} else {
|
|
352
|
+
// At least one has a prefix, use PrefixMap
|
|
353
|
+
const prefixMap = new PrefixMap<TValue, TPrefix>()
|
|
354
|
+
|
|
355
|
+
if (currentPrefix === newPrefix) {
|
|
356
|
+
// Same prefix, different suffixes - need ValueMap within PrefixMap
|
|
357
|
+
const valueMap = new ValueMap<TValue>()
|
|
358
|
+
valueMap.set(hash(currentValue), currentSingleValue)
|
|
359
|
+
valueMap.set(hash(newValue), [newValue, multiplicity])
|
|
360
|
+
prefixMap.set(currentPrefix, valueMap)
|
|
361
|
+
} else {
|
|
362
|
+
// Different prefixes - store as separate single values
|
|
363
|
+
prefixMap.set(currentPrefix, currentSingleValue)
|
|
364
|
+
prefixMap.set(newPrefix, [newValue, multiplicity])
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
this.#inner.set(key, prefixMap)
|
|
368
|
+
}
|
|
135
369
|
}
|
|
136
370
|
|
|
137
|
-
|
|
138
|
-
|
|
371
|
+
/**
|
|
372
|
+
* This method appends another index to the current index.
|
|
373
|
+
* @param other - The index to append to the current index.
|
|
374
|
+
*/
|
|
375
|
+
append(other: Index<TKey, TValue>): void {
|
|
376
|
+
for (const [key, value] of other.entries()) {
|
|
139
377
|
this.addValue(key, value)
|
|
140
378
|
}
|
|
141
379
|
}
|
|
142
380
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
381
|
+
/**
|
|
382
|
+
* This method joins two indexes.
|
|
383
|
+
* @param other - The index to join with the current index.
|
|
384
|
+
* @returns A multiset of the joined values.
|
|
385
|
+
*/
|
|
386
|
+
join<TValue2>(
|
|
387
|
+
other: Index<TKey, TValue2>
|
|
388
|
+
): MultiSet<[TKey, [TValue, TValue2]]> {
|
|
389
|
+
const result: Array<[[TKey, [TValue, TValue2]], number]> = []
|
|
146
390
|
// We want to iterate over the smaller of the two indexes to reduce the
|
|
147
391
|
// number of operations we need to do.
|
|
148
392
|
if (this.size <= other.size) {
|
|
149
|
-
for (const [key, valueIt] of this
|
|
393
|
+
for (const [key, valueIt] of this.entriesIterators()) {
|
|
150
394
|
if (!other.has(key)) continue
|
|
151
395
|
const otherValues = other.get(key)
|
|
152
396
|
for (const [val1, mul1] of valueIt) {
|
|
@@ -158,7 +402,7 @@ export class Index<K, V> {
|
|
|
158
402
|
}
|
|
159
403
|
}
|
|
160
404
|
} else {
|
|
161
|
-
for (const [key, otherValueIt] of other
|
|
405
|
+
for (const [key, otherValueIt] of other.entriesIterators()) {
|
|
162
406
|
if (!this.has(key)) continue
|
|
163
407
|
const values = this.get(key)
|
|
164
408
|
for (const [val2, mul2] of otherValueIt) {
|
|
@@ -174,3 +418,34 @@ export class Index<K, V> {
|
|
|
174
418
|
return new MultiSet(result)
|
|
175
419
|
}
|
|
176
420
|
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* This function extracts the prefix from a value.
|
|
424
|
+
* @param value - The value to extract the prefix from.
|
|
425
|
+
* @returns The prefix and the suffix.
|
|
426
|
+
*/
|
|
427
|
+
function getPrefix<TValue, TPrefix>(value: TValue): TPrefix | NO_PREFIX {
|
|
428
|
+
// If the value is an array and the first element is a string or number, then the
|
|
429
|
+
// first element is the prefix. This is used to distinguish between values without
|
|
430
|
+
// the need for hashing unless there are multiple values for the same prefix.
|
|
431
|
+
if (
|
|
432
|
+
Array.isArray(value) &&
|
|
433
|
+
(typeof value[0] === `string` ||
|
|
434
|
+
typeof value[0] === `number` ||
|
|
435
|
+
typeof value[0] === `bigint`)
|
|
436
|
+
) {
|
|
437
|
+
return value[0] as TPrefix
|
|
438
|
+
}
|
|
439
|
+
return NO_PREFIX
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* This function checks if a value is a single value.
|
|
444
|
+
* @param value - The value to check.
|
|
445
|
+
* @returns True if the value is a single value, false otherwise.
|
|
446
|
+
*/
|
|
447
|
+
function isSingleValue<TValue>(
|
|
448
|
+
value: SingleValue<TValue> | unknown
|
|
449
|
+
): value is SingleValue<TValue> {
|
|
450
|
+
return Array.isArray(value)
|
|
451
|
+
}
|
|
@@ -4,21 +4,25 @@ import { hash } from "../hashing/index.js"
|
|
|
4
4
|
import { MultiSet } from "../multiset.js"
|
|
5
5
|
import type { Hash } from "../hashing/index.js"
|
|
6
6
|
import type { DifferenceStreamReader } from "../graph.js"
|
|
7
|
-
import type { IStreamBuilder } from "../types.js"
|
|
7
|
+
import type { IStreamBuilder, KeyValue } from "../types.js"
|
|
8
8
|
|
|
9
9
|
type Multiplicity = number
|
|
10
10
|
|
|
11
|
+
type GetValue<T> = T extends KeyValue<any, infer V> ? V : never
|
|
12
|
+
|
|
11
13
|
/**
|
|
12
14
|
* Operator that removes duplicates
|
|
13
15
|
*/
|
|
14
|
-
export class DistinctOperator<
|
|
16
|
+
export class DistinctOperator<
|
|
17
|
+
T extends KeyValue<any, any>,
|
|
18
|
+
> extends UnaryOperator<T, KeyValue<number, GetValue<T>>> {
|
|
15
19
|
#by: (value: T) => any
|
|
16
20
|
#values: Map<Hash, Multiplicity> // keeps track of the number of times each value has been seen
|
|
17
21
|
|
|
18
22
|
constructor(
|
|
19
23
|
id: number,
|
|
20
24
|
input: DifferenceStreamReader<T>,
|
|
21
|
-
output: DifferenceStreamWriter<T
|
|
25
|
+
output: DifferenceStreamWriter<KeyValue<number, GetValue<T>>>,
|
|
22
26
|
by: (value: T) => any = (value: T) => value
|
|
23
27
|
) {
|
|
24
28
|
super(id, input, output)
|
|
@@ -39,12 +43,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
|
|
|
39
43
|
this.#values.get(hashedValue) ??
|
|
40
44
|
0
|
|
41
45
|
const newMultiplicity = oldMultiplicity + diff
|
|
42
|
-
|
|
43
46
|
updatedValues.set(hashedValue, [newMultiplicity, value])
|
|
44
47
|
}
|
|
45
48
|
}
|
|
46
49
|
|
|
47
|
-
const result: Array<[
|
|
50
|
+
const result: Array<[KeyValue<number, GetValue<T>>, number]> = []
|
|
48
51
|
|
|
49
52
|
// Check which values became visible or disappeared
|
|
50
53
|
for (const [
|
|
@@ -62,11 +65,11 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
|
|
|
62
65
|
if (oldMultiplicity <= 0 && newMultiplicity > 0) {
|
|
63
66
|
// The value wasn't present in the stream
|
|
64
67
|
// but with this change it is now present in the stream
|
|
65
|
-
result.push([value, 1])
|
|
68
|
+
result.push([[hash(this.#by(value)), value[1]], 1])
|
|
66
69
|
} else if (oldMultiplicity > 0 && newMultiplicity <= 0) {
|
|
67
70
|
// The value was present in the stream
|
|
68
71
|
// but with this change it is no longer present in the stream
|
|
69
|
-
result.push([value, -1])
|
|
72
|
+
result.push([[hash(this.#by(value)), value[1]], -1])
|
|
70
73
|
}
|
|
71
74
|
}
|
|
72
75
|
|
|
@@ -79,7 +82,9 @@ export class DistinctOperator<T> extends UnaryOperator<T> {
|
|
|
79
82
|
/**
|
|
80
83
|
* Removes duplicate values
|
|
81
84
|
*/
|
|
82
|
-
export function distinct<T
|
|
85
|
+
export function distinct<T extends KeyValue<any, any>>(
|
|
86
|
+
by: (value: T) => any = (value: T) => value
|
|
87
|
+
) {
|
|
83
88
|
return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
|
|
84
89
|
const output = new StreamBuilder<T>(
|
|
85
90
|
stream.graph,
|
package/src/operators/groupBy.ts
CHANGED
|
@@ -210,44 +210,64 @@ export function avg<T>(
|
|
|
210
210
|
}
|
|
211
211
|
}
|
|
212
212
|
|
|
213
|
+
type CanMinMax = number | Date | bigint
|
|
214
|
+
|
|
213
215
|
/**
|
|
214
216
|
* Creates a min aggregate function that computes the minimum value in a group
|
|
215
|
-
* @param valueExtractor Function to extract a
|
|
217
|
+
* @param valueExtractor Function to extract a comparable value from each data entry
|
|
216
218
|
*/
|
|
217
|
-
export function min<T>(
|
|
218
|
-
|
|
219
|
-
|
|
219
|
+
export function min<T extends CanMinMax>(): AggregateFunction<
|
|
220
|
+
T,
|
|
221
|
+
T | undefined,
|
|
222
|
+
T | undefined
|
|
223
|
+
>
|
|
224
|
+
export function min<T, V extends CanMinMax>(
|
|
225
|
+
valueExtractor: (value: T) => V
|
|
226
|
+
): AggregateFunction<T, V | undefined, V | undefined>
|
|
227
|
+
export function min<T, V extends CanMinMax>(
|
|
228
|
+
valueExtractor?: (value: T) => V
|
|
229
|
+
): AggregateFunction<T, V | undefined, V | undefined> {
|
|
230
|
+
const extractor = valueExtractor ?? ((v: T) => v as unknown as V)
|
|
220
231
|
return {
|
|
221
|
-
preMap: (data: T) =>
|
|
222
|
-
reduce: (values
|
|
223
|
-
let minValue
|
|
232
|
+
preMap: (data: T) => extractor(data),
|
|
233
|
+
reduce: (values) => {
|
|
234
|
+
let minValue: V | undefined
|
|
224
235
|
for (const [value, _multiplicity] of values) {
|
|
225
|
-
if (value < minValue) {
|
|
236
|
+
if (!minValue || (value && value < minValue)) {
|
|
226
237
|
minValue = value
|
|
227
238
|
}
|
|
228
239
|
}
|
|
229
|
-
return minValue
|
|
240
|
+
return minValue
|
|
230
241
|
},
|
|
231
242
|
}
|
|
232
243
|
}
|
|
233
244
|
|
|
234
245
|
/**
|
|
235
246
|
* Creates a max aggregate function that computes the maximum value in a group
|
|
236
|
-
* @param valueExtractor Function to extract a
|
|
247
|
+
* @param valueExtractor Function to extract a comparable value from each data entry
|
|
237
248
|
*/
|
|
238
|
-
export function max<T>(
|
|
239
|
-
|
|
240
|
-
|
|
249
|
+
export function max<T extends CanMinMax>(): AggregateFunction<
|
|
250
|
+
T,
|
|
251
|
+
T | undefined,
|
|
252
|
+
T | undefined
|
|
253
|
+
>
|
|
254
|
+
export function max<T, V extends CanMinMax>(
|
|
255
|
+
valueExtractor: (value: T) => V
|
|
256
|
+
): AggregateFunction<T, V | undefined, V | undefined>
|
|
257
|
+
export function max<T, V extends CanMinMax>(
|
|
258
|
+
valueExtractor?: (value: T) => V
|
|
259
|
+
): AggregateFunction<T, V | undefined, V | undefined> {
|
|
260
|
+
const extractor = valueExtractor ?? ((v: T) => v as unknown as V)
|
|
241
261
|
return {
|
|
242
|
-
preMap: (data: T) =>
|
|
243
|
-
reduce: (values
|
|
244
|
-
let maxValue
|
|
262
|
+
preMap: (data: T) => extractor(data),
|
|
263
|
+
reduce: (values) => {
|
|
264
|
+
let maxValue: V | undefined
|
|
245
265
|
for (const [value, _multiplicity] of values) {
|
|
246
|
-
if (value > maxValue) {
|
|
266
|
+
if (!maxValue || (value && value > maxValue)) {
|
|
247
267
|
maxValue = value
|
|
248
268
|
}
|
|
249
269
|
}
|
|
250
|
-
return maxValue
|
|
270
|
+
return maxValue
|
|
251
271
|
},
|
|
252
272
|
}
|
|
253
273
|
}
|