@tanstack/db-ivm 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cjs/d2.cjs.map +1 -1
- package/dist/cjs/graph.cjs.map +1 -1
- package/dist/cjs/hashing/hash.cjs.map +1 -1
- package/dist/cjs/index.cjs +6 -3
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +1 -0
- package/dist/cjs/indexes.cjs +1 -1
- package/dist/cjs/indexes.cjs.map +1 -1
- package/dist/cjs/multiset.cjs.map +1 -1
- package/dist/cjs/operators/concat.cjs.map +1 -1
- package/dist/cjs/operators/consolidate.cjs.map +1 -1
- package/dist/cjs/operators/count.cjs.map +1 -1
- package/dist/cjs/operators/debug.cjs.map +1 -1
- package/dist/cjs/operators/distinct.cjs.map +1 -1
- package/dist/cjs/operators/filter.cjs.map +1 -1
- package/dist/cjs/operators/filterBy.cjs.map +1 -1
- package/dist/cjs/operators/groupBy.cjs +2 -1
- package/dist/cjs/operators/groupBy.cjs.map +1 -1
- package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs +116 -0
- package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs.map +1 -0
- package/dist/cjs/operators/groupedTopKWithFractionalIndex.d.cts +50 -0
- package/dist/cjs/operators/index.d.cts +1 -0
- package/dist/cjs/operators/join.cjs.map +1 -1
- package/dist/cjs/operators/keying.cjs.map +1 -1
- package/dist/cjs/operators/map.cjs.map +1 -1
- package/dist/cjs/operators/negate.cjs.map +1 -1
- package/dist/cjs/operators/orderBy.cjs.map +1 -1
- package/dist/cjs/operators/output.cjs.map +1 -1
- package/dist/cjs/operators/pipe.cjs.map +1 -1
- package/dist/cjs/operators/reduce.cjs.map +1 -1
- package/dist/cjs/operators/tap.cjs.map +1 -1
- package/dist/cjs/operators/topK.cjs.map +1 -1
- package/dist/cjs/operators/topKArray.cjs +137 -0
- package/dist/cjs/operators/topKArray.cjs.map +1 -0
- package/dist/cjs/operators/topKArray.d.cts +59 -0
- package/dist/cjs/operators/topKState.cjs +73 -0
- package/dist/cjs/operators/topKState.cjs.map +1 -0
- package/dist/cjs/operators/topKState.d.cts +35 -0
- package/dist/cjs/operators/topKWithFractionalIndex.cjs +17 -208
- package/dist/cjs/operators/topKWithFractionalIndex.cjs.map +1 -1
- package/dist/cjs/operators/topKWithFractionalIndex.d.cts +4 -43
- package/dist/cjs/operators/topKWithFractionalIndexBTree.d.cts +5 -4
- package/dist/cjs/utils.cjs +21 -0
- package/dist/cjs/utils.cjs.map +1 -1
- package/dist/cjs/utils.d.cts +11 -0
- package/dist/esm/d2.js.map +1 -1
- package/dist/esm/graph.js.map +1 -1
- package/dist/esm/hashing/hash.js.map +1 -1
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.js +7 -4
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/indexes.js +1 -1
- package/dist/esm/indexes.js.map +1 -1
- package/dist/esm/multiset.js.map +1 -1
- package/dist/esm/operators/concat.js.map +1 -1
- package/dist/esm/operators/consolidate.js.map +1 -1
- package/dist/esm/operators/count.js.map +1 -1
- package/dist/esm/operators/debug.js.map +1 -1
- package/dist/esm/operators/distinct.js.map +1 -1
- package/dist/esm/operators/filter.js.map +1 -1
- package/dist/esm/operators/filterBy.js.map +1 -1
- package/dist/esm/operators/groupBy.js +2 -1
- package/dist/esm/operators/groupBy.js.map +1 -1
- package/dist/esm/operators/groupedTopKWithFractionalIndex.d.ts +50 -0
- package/dist/esm/operators/groupedTopKWithFractionalIndex.js +116 -0
- package/dist/esm/operators/groupedTopKWithFractionalIndex.js.map +1 -0
- package/dist/esm/operators/index.d.ts +1 -0
- package/dist/esm/operators/join.js.map +1 -1
- package/dist/esm/operators/keying.js.map +1 -1
- package/dist/esm/operators/map.js.map +1 -1
- package/dist/esm/operators/negate.js.map +1 -1
- package/dist/esm/operators/orderBy.js.map +1 -1
- package/dist/esm/operators/output.js.map +1 -1
- package/dist/esm/operators/pipe.js.map +1 -1
- package/dist/esm/operators/reduce.js.map +1 -1
- package/dist/esm/operators/tap.js.map +1 -1
- package/dist/esm/operators/topK.js.map +1 -1
- package/dist/esm/operators/topKArray.d.ts +59 -0
- package/dist/esm/operators/topKArray.js +137 -0
- package/dist/esm/operators/topKArray.js.map +1 -0
- package/dist/esm/operators/topKState.d.ts +35 -0
- package/dist/esm/operators/topKState.js +73 -0
- package/dist/esm/operators/topKState.js.map +1 -0
- package/dist/esm/operators/topKWithFractionalIndex.d.ts +4 -43
- package/dist/esm/operators/topKWithFractionalIndex.js +16 -207
- package/dist/esm/operators/topKWithFractionalIndex.js.map +1 -1
- package/dist/esm/operators/topKWithFractionalIndexBTree.d.ts +5 -4
- package/dist/esm/utils.d.ts +11 -0
- package/dist/esm/utils.js +22 -1
- package/dist/esm/utils.js.map +1 -1
- package/package.json +29 -29
- package/src/d2.ts +4 -4
- package/src/graph.ts +6 -6
- package/src/hashing/hash.ts +3 -3
- package/src/hashing/index.ts +2 -2
- package/src/index.ts +5 -4
- package/src/indexes.ts +8 -8
- package/src/multiset.ts +5 -5
- package/src/operators/concat.ts +6 -6
- package/src/operators/consolidate.ts +6 -6
- package/src/operators/count.ts +9 -9
- package/src/operators/debug.ts +8 -8
- package/src/operators/distinct.ts +11 -11
- package/src/operators/filter.ts +8 -8
- package/src/operators/filterBy.ts +7 -7
- package/src/operators/groupBy.ts +24 -23
- package/src/operators/groupedTopKWithFractionalIndex.ts +192 -0
- package/src/operators/index.ts +20 -19
- package/src/operators/join.ts +23 -23
- package/src/operators/keying.ts +4 -4
- package/src/operators/map.ts +8 -8
- package/src/operators/negate.ts +6 -6
- package/src/operators/orderBy.ts +26 -26
- package/src/operators/orderByBTree.ts +7 -7
- package/src/operators/output.ts +9 -9
- package/src/operators/pipe.ts +1 -1
- package/src/operators/reduce.ts +9 -9
- package/src/operators/tap.ts +8 -8
- package/src/operators/topK.ts +8 -8
- package/src/operators/topKArray.ts +255 -0
- package/src/operators/topKState.ts +111 -0
- package/src/operators/topKWithFractionalIndex.ts +36 -381
- package/src/operators/topKWithFractionalIndexBTree.ts +21 -30
- package/src/types.ts +22 -22
- package/src/utils.ts +36 -4
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'
|
|
2
|
+
import { StreamBuilder } from '../d2.js'
|
|
3
|
+
import { MultiSet } from '../multiset.js'
|
|
4
|
+
import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js'
|
|
5
|
+
import { TopKArray, createKeyedComparator } from './topKArray.js'
|
|
6
|
+
import type { IndexedValue, TopK } from './topKArray.js'
|
|
7
|
+
import type { DifferenceStreamReader } from '../graph.js'
|
|
8
|
+
import type { IStreamBuilder, PipedOperator } from '../types.js'
|
|
9
|
+
|
|
10
|
+
export interface GroupedTopKWithFractionalIndexOptions<K, T> {
|
|
11
|
+
limit?: number
|
|
12
|
+
offset?: number
|
|
13
|
+
setSizeCallback?: (getSize: () => number) => void
|
|
14
|
+
setWindowFn?: (
|
|
15
|
+
windowFn: (options: { offset?: number; limit?: number }) => void,
|
|
16
|
+
) => void
|
|
17
|
+
/**
|
|
18
|
+
* Function to extract a group key from the element's key and value.
|
|
19
|
+
* Elements with the same group key will be sorted and limited together.
|
|
20
|
+
*/
|
|
21
|
+
groupKeyFn: (key: K, value: T) => unknown
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Operator for grouped fractional indexed topK operations.
|
|
26
|
+
* This operator maintains separate topK windows for each group,
|
|
27
|
+
* allowing per-group limits and ordering.
|
|
28
|
+
*
|
|
29
|
+
* The input is a keyed stream [K, T] and outputs [K, IndexedValue<T>].
|
|
30
|
+
* Elements are grouped by the groupKeyFn, and each group maintains
|
|
31
|
+
* its own sorted collection with independent limit/offset.
|
|
32
|
+
*/
|
|
33
|
+
export class GroupedTopKWithFractionalIndexOperator<
|
|
34
|
+
K extends string | number,
|
|
35
|
+
T,
|
|
36
|
+
> extends UnaryOperator<[K, T], [K, IndexedValue<T>]> {
|
|
37
|
+
#groupStates: Map<unknown, TopKState<K, T>> = new Map()
|
|
38
|
+
#groupKeyFn: (key: K, value: T) => unknown
|
|
39
|
+
#comparator: (a: [K, T], b: [K, T]) => number
|
|
40
|
+
#offset: number
|
|
41
|
+
#limit: number
|
|
42
|
+
|
|
43
|
+
constructor(
|
|
44
|
+
id: number,
|
|
45
|
+
inputA: DifferenceStreamReader<[K, T]>,
|
|
46
|
+
output: DifferenceStreamWriter<[K, IndexedValue<T>]>,
|
|
47
|
+
comparator: (a: T, b: T) => number,
|
|
48
|
+
options: GroupedTopKWithFractionalIndexOptions<K, T>,
|
|
49
|
+
) {
|
|
50
|
+
super(id, inputA, output)
|
|
51
|
+
this.#groupKeyFn = options.groupKeyFn
|
|
52
|
+
this.#limit = options.limit ?? Infinity
|
|
53
|
+
this.#offset = options.offset ?? 0
|
|
54
|
+
this.#comparator = createKeyedComparator(comparator)
|
|
55
|
+
options.setSizeCallback?.(() => this.#getTotalSize())
|
|
56
|
+
options.setWindowFn?.(this.#moveTopK.bind(this))
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Creates a new TopK data structure for a group.
|
|
61
|
+
* Can be overridden in subclasses to use different implementations (e.g., B+ tree).
|
|
62
|
+
*/
|
|
63
|
+
protected createTopK(
|
|
64
|
+
offset: number,
|
|
65
|
+
limit: number,
|
|
66
|
+
comparator: (a: [K, T], b: [K, T]) => number,
|
|
67
|
+
): TopK<[K, T]> {
|
|
68
|
+
return new TopKArray(offset, limit, comparator)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#getTotalSize(): number {
|
|
72
|
+
let size = 0
|
|
73
|
+
for (const state of this.#groupStates.values()) {
|
|
74
|
+
size += state.size
|
|
75
|
+
}
|
|
76
|
+
return size
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
#getOrCreateGroupState(groupKey: unknown): TopKState<K, T> {
|
|
80
|
+
let state = this.#groupStates.get(groupKey)
|
|
81
|
+
if (!state) {
|
|
82
|
+
const topK = this.createTopK(this.#offset, this.#limit, this.#comparator)
|
|
83
|
+
state = new TopKState(topK)
|
|
84
|
+
this.#groupStates.set(groupKey, state)
|
|
85
|
+
}
|
|
86
|
+
return state
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#cleanupGroupIfEmpty(groupKey: unknown, state: TopKState<K, T>): void {
|
|
90
|
+
if (state.isEmpty) {
|
|
91
|
+
this.#groupStates.delete(groupKey)
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Moves the topK window for all groups based on the provided offset and limit.
|
|
97
|
+
* Any changes to the topK are sent to the output.
|
|
98
|
+
*/
|
|
99
|
+
#moveTopK({ offset, limit }: { offset?: number; limit?: number }): void {
|
|
100
|
+
if (offset !== undefined) {
|
|
101
|
+
this.#offset = offset
|
|
102
|
+
}
|
|
103
|
+
if (limit !== undefined) {
|
|
104
|
+
this.#limit = limit
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const result: Array<[[K, IndexedValue<T>], number]> = []
|
|
108
|
+
let hasChanges = false
|
|
109
|
+
|
|
110
|
+
for (const state of this.#groupStates.values()) {
|
|
111
|
+
const diff = state.move({ offset: this.#offset, limit: this.#limit }) // TODO: think we should just pass offset and limit
|
|
112
|
+
|
|
113
|
+
diff.moveIns.forEach((moveIn) => handleMoveIn(moveIn, result))
|
|
114
|
+
diff.moveOuts.forEach((moveOut) => handleMoveOut(moveOut, result))
|
|
115
|
+
|
|
116
|
+
if (diff.changes) {
|
|
117
|
+
hasChanges = true
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (hasChanges) {
|
|
122
|
+
this.output.sendData(new MultiSet(result))
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
run(): void {
|
|
127
|
+
const result: Array<[[K, IndexedValue<T>], number]> = []
|
|
128
|
+
for (const message of this.inputMessages()) {
|
|
129
|
+
for (const [item, multiplicity] of message.getInner()) {
|
|
130
|
+
const [key, value] = item
|
|
131
|
+
this.#processElement(key, value, multiplicity, result)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (result.length > 0) {
|
|
136
|
+
this.output.sendData(new MultiSet(result))
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
#processElement(
|
|
141
|
+
key: K,
|
|
142
|
+
value: T,
|
|
143
|
+
multiplicity: number,
|
|
144
|
+
result: Array<[[K, IndexedValue<T>], number]>,
|
|
145
|
+
): void {
|
|
146
|
+
const groupKey = this.#groupKeyFn(key, value)
|
|
147
|
+
const state = this.#getOrCreateGroupState(groupKey)
|
|
148
|
+
|
|
149
|
+
const changes = state.processElement(key, value, multiplicity)
|
|
150
|
+
handleMoveIn(changes.moveIn, result)
|
|
151
|
+
handleMoveOut(changes.moveOut, result)
|
|
152
|
+
|
|
153
|
+
// Cleanup empty groups to prevent memory leaks
|
|
154
|
+
this.#cleanupGroupIfEmpty(groupKey, state)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Limits the number of results per group based on a comparator, with optional offset.
|
|
160
|
+
* Uses fractional indexing to minimize the number of changes when elements move positions.
|
|
161
|
+
* Each element is assigned a fractional index that is lexicographically sortable.
|
|
162
|
+
* When elements move, only the indices of the moved elements are updated, not all elements.
|
|
163
|
+
*
|
|
164
|
+
* This operator groups elements by the provided groupKeyFn and applies the limit/offset
|
|
165
|
+
* independently to each group.
|
|
166
|
+
*
|
|
167
|
+
* @param comparator - A function that compares two elements for ordering
|
|
168
|
+
* @param options - Configuration including groupKeyFn, limit, and offset
|
|
169
|
+
* @returns A piped operator that orders elements per group and limits results per group
|
|
170
|
+
*/
|
|
171
|
+
export function groupedTopKWithFractionalIndex<K extends string | number, T>(
|
|
172
|
+
comparator: (a: T, b: T) => number,
|
|
173
|
+
options: GroupedTopKWithFractionalIndexOptions<K, T>,
|
|
174
|
+
): PipedOperator<[K, T], [K, IndexedValue<T>]> {
|
|
175
|
+
return (
|
|
176
|
+
stream: IStreamBuilder<[K, T]>,
|
|
177
|
+
): IStreamBuilder<[K, IndexedValue<T>]> => {
|
|
178
|
+
const output = new StreamBuilder<[K, IndexedValue<T>]>(
|
|
179
|
+
stream.graph,
|
|
180
|
+
new DifferenceStreamWriter<[K, IndexedValue<T>]>(),
|
|
181
|
+
)
|
|
182
|
+
const operator = new GroupedTopKWithFractionalIndexOperator<K, T>(
|
|
183
|
+
stream.graph.getNextOperatorId(),
|
|
184
|
+
stream.connectReader(),
|
|
185
|
+
output.writer,
|
|
186
|
+
comparator,
|
|
187
|
+
options,
|
|
188
|
+
)
|
|
189
|
+
stream.graph.addOperator(operator)
|
|
190
|
+
return output
|
|
191
|
+
}
|
|
192
|
+
}
|
package/src/operators/index.ts
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
export * from
|
|
2
|
-
export * from
|
|
3
|
-
export * from
|
|
4
|
-
export * from
|
|
5
|
-
export * from
|
|
6
|
-
export * from
|
|
7
|
-
export * from
|
|
8
|
-
export * from
|
|
9
|
-
export * from
|
|
10
|
-
export * from
|
|
11
|
-
export * from
|
|
12
|
-
export * from
|
|
13
|
-
export * from
|
|
14
|
-
export * from
|
|
15
|
-
export * from
|
|
16
|
-
export * from
|
|
17
|
-
export * from
|
|
18
|
-
export * from
|
|
19
|
-
export
|
|
1
|
+
export * from './pipe.js'
|
|
2
|
+
export * from './map.js'
|
|
3
|
+
export * from './tap.js'
|
|
4
|
+
export * from './filter.js'
|
|
5
|
+
export * from './negate.js'
|
|
6
|
+
export * from './concat.js'
|
|
7
|
+
export * from './debug.js'
|
|
8
|
+
export * from './output.js'
|
|
9
|
+
export * from './consolidate.js'
|
|
10
|
+
export * from './join.js'
|
|
11
|
+
export * from './reduce.js'
|
|
12
|
+
export * from './count.js'
|
|
13
|
+
export * from './distinct.js'
|
|
14
|
+
export * from './keying.js'
|
|
15
|
+
export * from './topK.js'
|
|
16
|
+
export * from './topKWithFractionalIndex.js'
|
|
17
|
+
export * from './groupedTopKWithFractionalIndex.js'
|
|
18
|
+
export * from './orderBy.js'
|
|
19
|
+
export * from './filterBy.js'
|
|
20
|
+
export { groupBy, groupByOperators } from './groupBy.js'
|
package/src/operators/join.ts
CHANGED
|
@@ -48,12 +48,12 @@
|
|
|
48
48
|
* - **Bag semantics**: Proper multiplicity handling including negatives
|
|
49
49
|
*/
|
|
50
50
|
|
|
51
|
-
import { BinaryOperator, DifferenceStreamWriter } from
|
|
52
|
-
import { StreamBuilder } from
|
|
53
|
-
import { MultiSet } from
|
|
54
|
-
import { Index } from
|
|
55
|
-
import type { DifferenceStreamReader } from
|
|
56
|
-
import type { IStreamBuilder, KeyValue, PipedOperator } from
|
|
51
|
+
import { BinaryOperator, DifferenceStreamWriter } from '../graph.js'
|
|
52
|
+
import { StreamBuilder } from '../d2.js'
|
|
53
|
+
import { MultiSet } from '../multiset.js'
|
|
54
|
+
import { Index } from '../indexes.js'
|
|
55
|
+
import type { DifferenceStreamReader } from '../graph.js'
|
|
56
|
+
import type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'
|
|
57
57
|
|
|
58
58
|
/**
|
|
59
59
|
* Type of join to perform
|
|
@@ -75,7 +75,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
75
75
|
inputA: DifferenceStreamReader<[K, V1]>,
|
|
76
76
|
inputB: DifferenceStreamReader<[K, V2]>,
|
|
77
77
|
output: DifferenceStreamWriter<any>,
|
|
78
|
-
mode: JoinType = `inner
|
|
78
|
+
mode: JoinType = `inner`,
|
|
79
79
|
) {
|
|
80
80
|
super(id, inputA, inputB, output)
|
|
81
81
|
this.#mode = mode
|
|
@@ -84,10 +84,10 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
84
84
|
run(): void {
|
|
85
85
|
// Build deltas from input messages
|
|
86
86
|
const deltaA = Index.fromMultiSets<K, V1>(
|
|
87
|
-
this.inputAMessages() as Array<MultiSet<[K, V1]
|
|
87
|
+
this.inputAMessages() as Array<MultiSet<[K, V1]>>,
|
|
88
88
|
)
|
|
89
89
|
const deltaB = Index.fromMultiSets<K, V2>(
|
|
90
|
-
this.inputBMessages() as Array<MultiSet<[K, V2]
|
|
90
|
+
this.inputBMessages() as Array<MultiSet<[K, V2]>>,
|
|
91
91
|
)
|
|
92
92
|
|
|
93
93
|
// Early-out if nothing changed
|
|
@@ -130,7 +130,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
130
130
|
private emitInnerResults(
|
|
131
131
|
deltaA: Index<K, V1>,
|
|
132
132
|
deltaB: Index<K, V2>,
|
|
133
|
-
results: MultiSet<any
|
|
133
|
+
results: MultiSet<any>,
|
|
134
134
|
): void {
|
|
135
135
|
// Emit the three standard delta terms: ΔA⋈B_old, A_old⋈ΔB, ΔA⋈ΔB
|
|
136
136
|
if (deltaA.size > 0) results.extend(deltaA.join(this.#indexB))
|
|
@@ -141,7 +141,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
141
141
|
private emitLeftOuterResults(
|
|
142
142
|
deltaA: Index<K, V1>,
|
|
143
143
|
deltaB: Index<K, V2>,
|
|
144
|
-
results: MultiSet<any
|
|
144
|
+
results: MultiSet<any>,
|
|
145
145
|
): void {
|
|
146
146
|
// Emit unmatched left rows from deltaA
|
|
147
147
|
if (deltaA.size > 0) {
|
|
@@ -184,7 +184,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
184
184
|
if (multiplicity !== 0) {
|
|
185
185
|
results.add(
|
|
186
186
|
[key, [value, null]],
|
|
187
|
-
transitioningToMatched ? -multiplicity : +multiplicity
|
|
187
|
+
transitioningToMatched ? -multiplicity : +multiplicity,
|
|
188
188
|
)
|
|
189
189
|
}
|
|
190
190
|
}
|
|
@@ -195,7 +195,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
195
195
|
private emitRightOuterResults(
|
|
196
196
|
deltaA: Index<K, V1>,
|
|
197
197
|
deltaB: Index<K, V2>,
|
|
198
|
-
results: MultiSet<any
|
|
198
|
+
results: MultiSet<any>,
|
|
199
199
|
): void {
|
|
200
200
|
// Emit unmatched right rows from deltaB
|
|
201
201
|
if (deltaB.size > 0) {
|
|
@@ -238,7 +238,7 @@ export class JoinOperator<K, V1, V2> extends BinaryOperator<
|
|
|
238
238
|
if (multiplicity !== 0) {
|
|
239
239
|
results.add(
|
|
240
240
|
[key, [null, value]],
|
|
241
|
-
transitioningToMatched ? -multiplicity : +multiplicity
|
|
241
|
+
transitioningToMatched ? -multiplicity : +multiplicity,
|
|
242
242
|
)
|
|
243
243
|
}
|
|
244
244
|
}
|
|
@@ -259,24 +259,24 @@ export function join<
|
|
|
259
259
|
T,
|
|
260
260
|
>(
|
|
261
261
|
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
262
|
-
type: JoinType = `inner
|
|
262
|
+
type: JoinType = `inner`,
|
|
263
263
|
): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
|
|
264
264
|
return (
|
|
265
|
-
stream: IStreamBuilder<T
|
|
265
|
+
stream: IStreamBuilder<T>,
|
|
266
266
|
): IStreamBuilder<KeyValue<K, [V1 | null, V2 | null]>> => {
|
|
267
267
|
if (stream.graph !== other.graph) {
|
|
268
268
|
throw new Error(`Cannot join streams from different graphs`)
|
|
269
269
|
}
|
|
270
270
|
const output = new StreamBuilder<KeyValue<K, [V1 | null, V2 | null]>>(
|
|
271
271
|
stream.graph,
|
|
272
|
-
new DifferenceStreamWriter<KeyValue<K, [V1 | null, V2 | null]>>()
|
|
272
|
+
new DifferenceStreamWriter<KeyValue<K, [V1 | null, V2 | null]>>(),
|
|
273
273
|
)
|
|
274
274
|
const operator = new JoinOperator<K, V1, V2>(
|
|
275
275
|
stream.graph.getNextOperatorId(),
|
|
276
276
|
stream.connectReader() as DifferenceStreamReader<KeyValue<K, V1>>,
|
|
277
277
|
other.connectReader(),
|
|
278
278
|
output.writer,
|
|
279
|
-
type
|
|
279
|
+
type,
|
|
280
280
|
)
|
|
281
281
|
stream.graph.addOperator(operator)
|
|
282
282
|
return output
|
|
@@ -293,7 +293,7 @@ export function innerJoin<
|
|
|
293
293
|
V2,
|
|
294
294
|
T,
|
|
295
295
|
>(
|
|
296
|
-
other: IStreamBuilder<KeyValue<K, V2
|
|
296
|
+
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
297
297
|
): PipedOperator<T, KeyValue<K, [V1, V2]>> {
|
|
298
298
|
return join(other, `inner`) as unknown as PipedOperator<
|
|
299
299
|
T,
|
|
@@ -311,7 +311,7 @@ export function antiJoin<
|
|
|
311
311
|
V2,
|
|
312
312
|
T,
|
|
313
313
|
>(
|
|
314
|
-
other: IStreamBuilder<KeyValue<K, V2
|
|
314
|
+
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
315
315
|
): PipedOperator<T, KeyValue<K, [V1, null]>> {
|
|
316
316
|
return join(other, `anti`) as unknown as PipedOperator<
|
|
317
317
|
T,
|
|
@@ -329,7 +329,7 @@ export function leftJoin<
|
|
|
329
329
|
V2,
|
|
330
330
|
T,
|
|
331
331
|
>(
|
|
332
|
-
other: IStreamBuilder<KeyValue<K, V2
|
|
332
|
+
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
333
333
|
): PipedOperator<T, KeyValue<K, [V1, V2 | null]>> {
|
|
334
334
|
return join(other, `left`) as unknown as PipedOperator<
|
|
335
335
|
T,
|
|
@@ -347,7 +347,7 @@ export function rightJoin<
|
|
|
347
347
|
V2,
|
|
348
348
|
T,
|
|
349
349
|
>(
|
|
350
|
-
other: IStreamBuilder<KeyValue<K, V2
|
|
350
|
+
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
351
351
|
): PipedOperator<T, KeyValue<K, [V1 | null, V2]>> {
|
|
352
352
|
return join(other, `right`) as unknown as PipedOperator<
|
|
353
353
|
T,
|
|
@@ -365,7 +365,7 @@ export function fullJoin<
|
|
|
365
365
|
V2,
|
|
366
366
|
T,
|
|
367
367
|
>(
|
|
368
|
-
other: IStreamBuilder<KeyValue<K, V2
|
|
368
|
+
other: IStreamBuilder<KeyValue<K, V2>>,
|
|
369
369
|
): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
|
|
370
370
|
return join(other, `full`) as unknown as PipedOperator<
|
|
371
371
|
T,
|
package/src/operators/keying.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { map } from
|
|
2
|
-
import type { PipedOperator } from
|
|
1
|
+
import { map } from './map.js'
|
|
2
|
+
import type { PipedOperator } from '../types.js'
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Type for a keyed value
|
|
@@ -11,7 +11,7 @@ export type Keyed<K, V> = [K, V]
|
|
|
11
11
|
* @param keyFn - Function to generate the key for each value
|
|
12
12
|
*/
|
|
13
13
|
export function keyBy<T, K>(
|
|
14
|
-
keyFn: (value: T) => K
|
|
14
|
+
keyFn: (value: T) => K,
|
|
15
15
|
): PipedOperator<T, Keyed<K, T>> {
|
|
16
16
|
return map((value: T): Keyed<K, T> => [keyFn(value), value])
|
|
17
17
|
}
|
|
@@ -28,7 +28,7 @@ export function unkey<K, V>(): PipedOperator<Keyed<K, V>, V> {
|
|
|
28
28
|
* @param keyFn - Function to generate the new key for each value
|
|
29
29
|
*/
|
|
30
30
|
export function rekey<K1, K2, V>(
|
|
31
|
-
keyFn: (value: V) => K2
|
|
31
|
+
keyFn: (value: V) => K2,
|
|
32
32
|
): PipedOperator<Keyed<K1, V>, Keyed<K2, V>> {
|
|
33
33
|
return map(([_, value]: Keyed<K1, V>): Keyed<K2, V> => [keyFn(value), value])
|
|
34
34
|
}
|
package/src/operators/map.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { DifferenceStreamWriter, LinearUnaryOperator } from
|
|
2
|
-
import { StreamBuilder } from
|
|
3
|
-
import type { IStreamBuilder, PipedOperator } from
|
|
4
|
-
import type { DifferenceStreamReader } from
|
|
5
|
-
import type { MultiSet } from
|
|
1
|
+
import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
|
|
2
|
+
import { StreamBuilder } from '../d2.js'
|
|
3
|
+
import type { IStreamBuilder, PipedOperator } from '../types.js'
|
|
4
|
+
import type { DifferenceStreamReader } from '../graph.js'
|
|
5
|
+
import type { MultiSet } from '../multiset.js'
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Operator that applies a function to each element in the input stream
|
|
@@ -14,7 +14,7 @@ export class MapOperator<T, U> extends LinearUnaryOperator<T, U> {
|
|
|
14
14
|
id: number,
|
|
15
15
|
inputA: DifferenceStreamReader<T>,
|
|
16
16
|
output: DifferenceStreamWriter<U>,
|
|
17
|
-
f: (data: T) => U
|
|
17
|
+
f: (data: T) => U,
|
|
18
18
|
) {
|
|
19
19
|
super(id, inputA, output)
|
|
20
20
|
this.#f = f
|
|
@@ -33,13 +33,13 @@ export function map<T, O>(f: (data: T) => O): PipedOperator<T, O> {
|
|
|
33
33
|
return (stream: IStreamBuilder<T>): IStreamBuilder<O> => {
|
|
34
34
|
const output = new StreamBuilder<O>(
|
|
35
35
|
stream.graph,
|
|
36
|
-
new DifferenceStreamWriter<O>()
|
|
36
|
+
new DifferenceStreamWriter<O>(),
|
|
37
37
|
)
|
|
38
38
|
const operator = new MapOperator<T, O>(
|
|
39
39
|
stream.graph.getNextOperatorId(),
|
|
40
40
|
stream.connectReader(),
|
|
41
41
|
output.writer,
|
|
42
|
-
f
|
|
42
|
+
f,
|
|
43
43
|
)
|
|
44
44
|
stream.graph.addOperator(operator)
|
|
45
45
|
return output
|
package/src/operators/negate.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { DifferenceStreamWriter, LinearUnaryOperator } from
|
|
2
|
-
import { StreamBuilder } from
|
|
3
|
-
import type { IStreamBuilder, PipedOperator } from
|
|
4
|
-
import type { MultiSet } from
|
|
1
|
+
import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js'
|
|
2
|
+
import { StreamBuilder } from '../d2.js'
|
|
3
|
+
import type { IStreamBuilder, PipedOperator } from '../types.js'
|
|
4
|
+
import type { MultiSet } from '../multiset.js'
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Operator that negates the multiplicities in the input stream
|
|
@@ -19,12 +19,12 @@ export function negate<T>(): PipedOperator<T, T> {
|
|
|
19
19
|
return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
|
|
20
20
|
const output = new StreamBuilder<T>(
|
|
21
21
|
stream.graph,
|
|
22
|
-
new DifferenceStreamWriter<T>()
|
|
22
|
+
new DifferenceStreamWriter<T>(),
|
|
23
23
|
)
|
|
24
24
|
const operator = new NegateOperator<T>(
|
|
25
25
|
stream.graph.getNextOperatorId(),
|
|
26
26
|
stream.connectReader(),
|
|
27
|
-
output.writer
|
|
27
|
+
output.writer,
|
|
28
28
|
)
|
|
29
29
|
stream.graph.addOperator(operator)
|
|
30
30
|
return output
|
package/src/operators/orderBy.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { topK, topKWithIndex } from
|
|
2
|
-
import { topKWithFractionalIndex } from
|
|
3
|
-
import { map } from
|
|
4
|
-
import { innerJoin } from
|
|
5
|
-
import { consolidate } from
|
|
6
|
-
import type { IStreamBuilder, KeyValue } from
|
|
1
|
+
import { topK, topKWithIndex } from './topK.js'
|
|
2
|
+
import { topKWithFractionalIndex } from './topKWithFractionalIndex.js'
|
|
3
|
+
import { map } from './map.js'
|
|
4
|
+
import { innerJoin } from './join.js'
|
|
5
|
+
import { consolidate } from './consolidate.js'
|
|
6
|
+
import type { IStreamBuilder, KeyValue } from '../types.js'
|
|
7
7
|
|
|
8
8
|
export interface OrderByOptions<Ve> {
|
|
9
9
|
comparator?: (a: Ve, b: Ve) => number
|
|
@@ -14,7 +14,7 @@ export interface OrderByOptions<Ve> {
|
|
|
14
14
|
type OrderByWithFractionalIndexOptions<Ve> = OrderByOptions<Ve> & {
|
|
15
15
|
setSizeCallback?: (getSize: () => number) => void
|
|
16
16
|
setWindowFn?: (
|
|
17
|
-
windowFn: (options: { offset?: number; limit?: number }) => void
|
|
17
|
+
windowFn: (options: { offset?: number; limit?: number }) => void,
|
|
18
18
|
) => void
|
|
19
19
|
}
|
|
20
20
|
|
|
@@ -28,9 +28,9 @@ type OrderByWithFractionalIndexOptions<Ve> = OrderByOptions<Ve> & {
|
|
|
28
28
|
*/
|
|
29
29
|
export function orderBy<T extends KeyValue<unknown, unknown>, Ve = unknown>(
|
|
30
30
|
valueExtractor: (
|
|
31
|
-
value: T extends KeyValue<unknown, infer V> ? V : never
|
|
31
|
+
value: T extends KeyValue<unknown, infer V> ? V : never,
|
|
32
32
|
) => Ve,
|
|
33
|
-
options?: OrderByOptions<Ve
|
|
33
|
+
options?: OrderByOptions<Ve>,
|
|
34
34
|
) {
|
|
35
35
|
const limit = options?.limit ?? Infinity
|
|
36
36
|
const offset = options?.offset ?? 0
|
|
@@ -54,10 +54,10 @@ export function orderBy<T extends KeyValue<unknown, unknown>, Ve = unknown>(
|
|
|
54
54
|
[
|
|
55
55
|
key,
|
|
56
56
|
valueExtractor(
|
|
57
|
-
value as T extends KeyValue<unknown, infer V> ? V : never
|
|
57
|
+
value as T extends KeyValue<unknown, infer V> ? V : never,
|
|
58
58
|
),
|
|
59
59
|
],
|
|
60
|
-
] as KeyValue<null, [KeyType, Ve]
|
|
60
|
+
] as KeyValue<null, [KeyType, Ve]>,
|
|
61
61
|
),
|
|
62
62
|
topK((a, b) => comparator(a[1], b[1]), { limit, offset }),
|
|
63
63
|
map(([_, [key]]) => [key, null] as KeyValue<KeyType, null>),
|
|
@@ -65,7 +65,7 @@ export function orderBy<T extends KeyValue<unknown, unknown>, Ve = unknown>(
|
|
|
65
65
|
map(([key, value]) => {
|
|
66
66
|
return [key, value[1]] as T
|
|
67
67
|
}),
|
|
68
|
-
consolidate()
|
|
68
|
+
consolidate(),
|
|
69
69
|
)
|
|
70
70
|
}
|
|
71
71
|
}
|
|
@@ -84,9 +84,9 @@ export function orderByWithIndex<
|
|
|
84
84
|
Ve = unknown,
|
|
85
85
|
>(
|
|
86
86
|
valueExtractor: (
|
|
87
|
-
value: T extends KeyValue<unknown, infer V> ? V : never
|
|
87
|
+
value: T extends KeyValue<unknown, infer V> ? V : never,
|
|
88
88
|
) => Ve,
|
|
89
|
-
options?: OrderByOptions<Ve
|
|
89
|
+
options?: OrderByOptions<Ve>,
|
|
90
90
|
) {
|
|
91
91
|
const limit = options?.limit ?? Infinity
|
|
92
92
|
const offset = options?.offset ?? 0
|
|
@@ -100,7 +100,7 @@ export function orderByWithIndex<
|
|
|
100
100
|
})
|
|
101
101
|
|
|
102
102
|
return (
|
|
103
|
-
stream: IStreamBuilder<T
|
|
103
|
+
stream: IStreamBuilder<T>,
|
|
104
104
|
): IStreamBuilder<
|
|
105
105
|
KeyValue<
|
|
106
106
|
T extends KeyValue<infer K, unknown> ? K : never,
|
|
@@ -118,10 +118,10 @@ export function orderByWithIndex<
|
|
|
118
118
|
[
|
|
119
119
|
key,
|
|
120
120
|
valueExtractor(
|
|
121
|
-
value as T extends KeyValue<unknown, infer V> ? V : never
|
|
121
|
+
value as T extends KeyValue<unknown, infer V> ? V : never,
|
|
122
122
|
),
|
|
123
123
|
],
|
|
124
|
-
] as KeyValue<null, [KeyType, Ve]
|
|
124
|
+
] as KeyValue<null, [KeyType, Ve]>,
|
|
125
125
|
),
|
|
126
126
|
topKWithIndex((a, b) => comparator(a[1], b[1]), { limit, offset }),
|
|
127
127
|
map(([_, [[key], index]]) => [key, index] as KeyValue<KeyType, number>),
|
|
@@ -129,7 +129,7 @@ export function orderByWithIndex<
|
|
|
129
129
|
map(([key, [index, value]]) => {
|
|
130
130
|
return [key, [value, index]] as KeyValue<KeyType, [ValueType, number]>
|
|
131
131
|
}),
|
|
132
|
-
consolidate()
|
|
132
|
+
consolidate(),
|
|
133
133
|
)
|
|
134
134
|
}
|
|
135
135
|
}
|
|
@@ -140,9 +140,9 @@ export function orderByWithFractionalIndexBase<
|
|
|
140
140
|
>(
|
|
141
141
|
topKFunction: typeof topKWithFractionalIndex,
|
|
142
142
|
valueExtractor: (
|
|
143
|
-
value: T extends KeyValue<unknown, infer V> ? V : never
|
|
143
|
+
value: T extends KeyValue<unknown, infer V> ? V : never,
|
|
144
144
|
) => Ve,
|
|
145
|
-
options?: OrderByWithFractionalIndexOptions<Ve
|
|
145
|
+
options?: OrderByWithFractionalIndexOptions<Ve>,
|
|
146
146
|
) {
|
|
147
147
|
type KeyType = T extends KeyValue<infer K, unknown> ? K : never
|
|
148
148
|
type ValueType = T extends KeyValue<unknown, infer V> ? V : never
|
|
@@ -161,7 +161,7 @@ export function orderByWithFractionalIndexBase<
|
|
|
161
161
|
})
|
|
162
162
|
|
|
163
163
|
return (
|
|
164
|
-
stream: IStreamBuilder<T
|
|
164
|
+
stream: IStreamBuilder<T>,
|
|
165
165
|
): IStreamBuilder<[KeyType, [ValueType, string]]> => {
|
|
166
166
|
return stream.pipe(
|
|
167
167
|
topKFunction(
|
|
@@ -172,9 +172,9 @@ export function orderByWithFractionalIndexBase<
|
|
|
172
172
|
offset,
|
|
173
173
|
setSizeCallback,
|
|
174
174
|
setWindowFn,
|
|
175
|
-
}
|
|
175
|
+
},
|
|
176
176
|
),
|
|
177
|
-
consolidate()
|
|
177
|
+
consolidate(),
|
|
178
178
|
)
|
|
179
179
|
}
|
|
180
180
|
}
|
|
@@ -193,13 +193,13 @@ export function orderByWithFractionalIndex<
|
|
|
193
193
|
Ve = unknown,
|
|
194
194
|
>(
|
|
195
195
|
valueExtractor: (
|
|
196
|
-
value: T extends KeyValue<unknown, infer V> ? V : never
|
|
196
|
+
value: T extends KeyValue<unknown, infer V> ? V : never,
|
|
197
197
|
) => Ve,
|
|
198
|
-
options?: OrderByWithFractionalIndexOptions<Ve
|
|
198
|
+
options?: OrderByWithFractionalIndexOptions<Ve>,
|
|
199
199
|
) {
|
|
200
200
|
return orderByWithFractionalIndexBase(
|
|
201
201
|
topKWithFractionalIndex,
|
|
202
202
|
valueExtractor,
|
|
203
|
-
options
|
|
203
|
+
options,
|
|
204
204
|
)
|
|
205
205
|
}
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import { orderByWithFractionalIndexBase } from
|
|
2
|
-
import { topKWithFractionalIndexBTree } from
|
|
3
|
-
import type { KeyValue } from
|
|
4
|
-
import type { OrderByOptions } from
|
|
1
|
+
import { orderByWithFractionalIndexBase } from './orderBy.js'
|
|
2
|
+
import { topKWithFractionalIndexBTree } from './topKWithFractionalIndexBTree.js'
|
|
3
|
+
import type { KeyValue } from '../types.js'
|
|
4
|
+
import type { OrderByOptions } from './orderBy.js'
|
|
5
5
|
|
|
6
6
|
export function orderByWithFractionalIndexBTree<
|
|
7
7
|
T extends KeyValue<unknown, unknown>,
|
|
8
8
|
Ve = unknown,
|
|
9
9
|
>(
|
|
10
10
|
valueExtractor: (
|
|
11
|
-
value: T extends KeyValue<unknown, infer V> ? V : never
|
|
11
|
+
value: T extends KeyValue<unknown, infer V> ? V : never,
|
|
12
12
|
) => Ve,
|
|
13
|
-
options?: OrderByOptions<Ve
|
|
13
|
+
options?: OrderByOptions<Ve>,
|
|
14
14
|
) {
|
|
15
15
|
return orderByWithFractionalIndexBase(
|
|
16
16
|
topKWithFractionalIndexBTree,
|
|
17
17
|
valueExtractor,
|
|
18
|
-
options
|
|
18
|
+
options,
|
|
19
19
|
)
|
|
20
20
|
}
|