@tanstack/db-ivm 0.1.26 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +5 -6
  2. package/dist/cjs/hashing/hash.cjs +58 -3
  3. package/dist/cjs/hashing/hash.cjs.map +1 -1
  4. package/dist/cjs/hashing/hash.d.cts +9 -0
  5. package/dist/cjs/hashing/murmur.cjs +26 -15
  6. package/dist/cjs/hashing/murmur.cjs.map +1 -1
  7. package/dist/cjs/hashing/murmur.d.cts +2 -0
  8. package/dist/cjs/index.cjs +0 -26
  9. package/dist/cjs/index.cjs.map +1 -1
  10. package/dist/cjs/indexes.cjs +52 -16
  11. package/dist/cjs/indexes.cjs.map +1 -1
  12. package/dist/cjs/indexes.d.cts +4 -1
  13. package/dist/cjs/operators/groupBy.cjs +33 -36
  14. package/dist/cjs/operators/groupBy.cjs.map +1 -1
  15. package/dist/cjs/operators/groupBy.d.cts +1 -5
  16. package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs +3 -9
  17. package/dist/cjs/operators/groupedTopKWithFractionalIndex.cjs.map +1 -1
  18. package/dist/cjs/operators/groupedTopKWithFractionalIndex.d.cts +1 -15
  19. package/dist/cjs/operators/index.d.cts +0 -6
  20. package/dist/cjs/operators/join.cjs +0 -20
  21. package/dist/cjs/operators/join.cjs.map +1 -1
  22. package/dist/cjs/operators/join.d.cts +0 -40
  23. package/dist/cjs/operators/orderBy.cjs +2 -77
  24. package/dist/cjs/operators/orderBy.cjs.map +1 -1
  25. package/dist/cjs/operators/orderBy.d.cts +2 -31
  26. package/dist/cjs/operators/reduce.cjs +36 -5
  27. package/dist/cjs/operators/reduce.cjs.map +1 -1
  28. package/dist/cjs/operators/reduce.d.cts +11 -2
  29. package/dist/cjs/operators/topKArray.cjs +9 -0
  30. package/dist/cjs/operators/topKArray.cjs.map +1 -1
  31. package/dist/cjs/operators/topKArray.d.cts +6 -0
  32. package/dist/cjs/operators/topKState.cjs +3 -0
  33. package/dist/cjs/operators/topKState.cjs.map +1 -1
  34. package/dist/cjs/operators/topKState.d.cts +1 -0
  35. package/dist/cjs/operators/topKWithFractionalIndex.cjs +5 -9
  36. package/dist/cjs/operators/topKWithFractionalIndex.cjs.map +1 -1
  37. package/dist/cjs/operators/topKWithFractionalIndex.d.cts +7 -12
  38. package/dist/cjs/utils.cjs.map +1 -1
  39. package/dist/cjs/utils.d.cts +0 -7
  40. package/dist/esm/hashing/hash.d.ts +9 -0
  41. package/dist/esm/hashing/hash.js +60 -5
  42. package/dist/esm/hashing/hash.js.map +1 -1
  43. package/dist/esm/hashing/murmur.d.ts +2 -0
  44. package/dist/esm/hashing/murmur.js +26 -15
  45. package/dist/esm/hashing/murmur.js.map +1 -1
  46. package/dist/esm/index.js +3 -29
  47. package/dist/esm/index.js.map +1 -1
  48. package/dist/esm/indexes.d.ts +4 -1
  49. package/dist/esm/indexes.js +52 -16
  50. package/dist/esm/indexes.js.map +1 -1
  51. package/dist/esm/operators/groupBy.d.ts +1 -5
  52. package/dist/esm/operators/groupBy.js +33 -36
  53. package/dist/esm/operators/groupBy.js.map +1 -1
  54. package/dist/esm/operators/groupedTopKWithFractionalIndex.d.ts +1 -6
  55. package/dist/esm/operators/groupedTopKWithFractionalIndex.js +3 -9
  56. package/dist/esm/operators/groupedTopKWithFractionalIndex.js.map +1 -1
  57. package/dist/esm/operators/index.d.ts +0 -6
  58. package/dist/esm/operators/join.d.ts +0 -25
  59. package/dist/esm/operators/join.js +1 -21
  60. package/dist/esm/operators/join.js.map +1 -1
  61. package/dist/esm/operators/orderBy.d.ts +2 -22
  62. package/dist/esm/operators/orderBy.js +3 -78
  63. package/dist/esm/operators/orderBy.js.map +1 -1
  64. package/dist/esm/operators/reduce.d.ts +11 -2
  65. package/dist/esm/operators/reduce.js +36 -5
  66. package/dist/esm/operators/reduce.js.map +1 -1
  67. package/dist/esm/operators/topKArray.d.ts +6 -0
  68. package/dist/esm/operators/topKArray.js +9 -0
  69. package/dist/esm/operators/topKArray.js.map +1 -1
  70. package/dist/esm/operators/topKState.d.ts +1 -0
  71. package/dist/esm/operators/topKState.js +3 -0
  72. package/dist/esm/operators/topKState.js.map +1 -1
  73. package/dist/esm/operators/topKWithFractionalIndex.d.ts +7 -3
  74. package/dist/esm/operators/topKWithFractionalIndex.js +6 -10
  75. package/dist/esm/operators/topKWithFractionalIndex.js.map +1 -1
  76. package/dist/esm/utils.d.ts +0 -7
  77. package/dist/esm/utils.js.map +1 -1
  78. package/package.json +2 -4
  79. package/src/hashing/hash.ts +87 -5
  80. package/src/hashing/murmur.ts +31 -17
  81. package/src/indexes.ts +65 -16
  82. package/src/operators/groupBy.ts +48 -58
  83. package/src/operators/groupedTopKWithFractionalIndex.ts +4 -15
  84. package/src/operators/index.ts +0 -6
  85. package/src/operators/join.ts +0 -90
  86. package/src/operators/orderBy.ts +6 -141
  87. package/src/operators/reduce.ts +60 -3
  88. package/src/operators/topKArray.ts +15 -0
  89. package/src/operators/topKState.ts +7 -0
  90. package/src/operators/topKWithFractionalIndex.ts +15 -16
  91. package/src/utils.ts +0 -30
  92. package/dist/cjs/operators/count.cjs +0 -35
  93. package/dist/cjs/operators/count.cjs.map +0 -1
  94. package/dist/cjs/operators/count.d.cts +0 -19
  95. package/dist/cjs/operators/debug.cjs +0 -39
  96. package/dist/cjs/operators/debug.cjs.map +0 -1
  97. package/dist/cjs/operators/debug.d.cts +0 -16
  98. package/dist/cjs/operators/filterBy.cjs +0 -19
  99. package/dist/cjs/operators/filterBy.cjs.map +0 -1
  100. package/dist/cjs/operators/filterBy.d.cts +0 -9
  101. package/dist/cjs/operators/keying.cjs +0 -16
  102. package/dist/cjs/operators/keying.cjs.map +0 -1
  103. package/dist/cjs/operators/keying.d.cts +0 -22
  104. package/dist/cjs/operators/negate.cjs +0 -27
  105. package/dist/cjs/operators/negate.cjs.map +0 -1
  106. package/dist/cjs/operators/negate.d.cts +0 -13
  107. package/dist/cjs/operators/topK.cjs +0 -39
  108. package/dist/cjs/operators/topK.cjs.map +0 -1
  109. package/dist/cjs/operators/topK.d.cts +0 -34
  110. package/dist/cjs/operators/topKWithFractionalIndexBTree.d.cts +0 -43
  111. package/dist/esm/operators/count.d.ts +0 -13
  112. package/dist/esm/operators/count.js +0 -35
  113. package/dist/esm/operators/count.js.map +0 -1
  114. package/dist/esm/operators/debug.d.ts +0 -16
  115. package/dist/esm/operators/debug.js +0 -39
  116. package/dist/esm/operators/debug.js.map +0 -1
  117. package/dist/esm/operators/filterBy.d.ts +0 -9
  118. package/dist/esm/operators/filterBy.js +0 -19
  119. package/dist/esm/operators/filterBy.js.map +0 -1
  120. package/dist/esm/operators/keying.d.ts +0 -19
  121. package/dist/esm/operators/keying.js +0 -16
  122. package/dist/esm/operators/keying.js.map +0 -1
  123. package/dist/esm/operators/negate.d.ts +0 -13
  124. package/dist/esm/operators/negate.js +0 -27
  125. package/dist/esm/operators/negate.js.map +0 -1
  126. package/dist/esm/operators/topK.d.ts +0 -31
  127. package/dist/esm/operators/topK.js +0 -39
  128. package/dist/esm/operators/topK.js.map +0 -1
  129. package/dist/esm/operators/topKWithFractionalIndexBTree.d.ts +0 -28
  130. package/src/operators/count.ts +0 -51
  131. package/src/operators/debug.ts +0 -57
  132. package/src/operators/filterBy.ts +0 -30
  133. package/src/operators/keying.ts +0 -34
  134. package/src/operators/negate.ts +0 -32
  135. package/src/operators/topK.ts +0 -91
  136. package/src/operators/topKWithFractionalIndexBTree.ts +0 -297
@@ -282,93 +282,3 @@ export function join<
282
282
  return output
283
283
  }
284
284
  }
285
-
286
- /**
287
- * Joins two input streams (inner join)
288
- * @param other - The other stream to join with
289
- */
290
- export function innerJoin<
291
- K,
292
- V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
293
- V2,
294
- T,
295
- >(
296
- other: IStreamBuilder<KeyValue<K, V2>>,
297
- ): PipedOperator<T, KeyValue<K, [V1, V2]>> {
298
- return join(other, `inner`) as unknown as PipedOperator<
299
- T,
300
- KeyValue<K, [V1, V2]>
301
- >
302
- }
303
-
304
- /**
305
- * Joins two input streams (anti join)
306
- * @param other - The other stream to join with
307
- */
308
- export function antiJoin<
309
- K,
310
- V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
311
- V2,
312
- T,
313
- >(
314
- other: IStreamBuilder<KeyValue<K, V2>>,
315
- ): PipedOperator<T, KeyValue<K, [V1, null]>> {
316
- return join(other, `anti`) as unknown as PipedOperator<
317
- T,
318
- KeyValue<K, [V1, null]>
319
- >
320
- }
321
-
322
- /**
323
- * Joins two input streams (left join)
324
- * @param other - The other stream to join with
325
- */
326
- export function leftJoin<
327
- K,
328
- V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
329
- V2,
330
- T,
331
- >(
332
- other: IStreamBuilder<KeyValue<K, V2>>,
333
- ): PipedOperator<T, KeyValue<K, [V1, V2 | null]>> {
334
- return join(other, `left`) as unknown as PipedOperator<
335
- T,
336
- KeyValue<K, [V1, V2 | null]>
337
- >
338
- }
339
-
340
- /**
341
- * Joins two input streams (right join)
342
- * @param other - The other stream to join with
343
- */
344
- export function rightJoin<
345
- K,
346
- V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
347
- V2,
348
- T,
349
- >(
350
- other: IStreamBuilder<KeyValue<K, V2>>,
351
- ): PipedOperator<T, KeyValue<K, [V1 | null, V2]>> {
352
- return join(other, `right`) as unknown as PipedOperator<
353
- T,
354
- KeyValue<K, [V1 | null, V2]>
355
- >
356
- }
357
-
358
- /**
359
- * Joins two input streams (full join)
360
- * @param other - The other stream to join with
361
- */
362
- export function fullJoin<
363
- K,
364
- V1 extends (T extends KeyValue<infer _KT, infer VT> ? VT : never),
365
- V2,
366
- T,
367
- >(
368
- other: IStreamBuilder<KeyValue<K, V2>>,
369
- ): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
370
- return join(other, `full`) as unknown as PipedOperator<
371
- T,
372
- KeyValue<K, [V1 | null, V2 | null]>
373
- >
374
- }
@@ -1,9 +1,7 @@
1
- import { topK, topKWithIndex } from './topK.js'
2
1
  import { topKWithFractionalIndex } from './topKWithFractionalIndex.js'
3
- import { map } from './map.js'
4
- import { innerJoin } from './join.js'
5
2
  import { consolidate } from './consolidate.js'
6
3
  import type { IStreamBuilder, KeyValue } from '../types.js'
4
+ import type { SetSizeCallback } from './topKWithFractionalIndex.js'
7
5
 
8
6
  export interface OrderByOptions<Ve> {
9
7
  comparator?: (a: Ve, b: Ve) => number
@@ -12,133 +10,25 @@ export interface OrderByOptions<Ve> {
12
10
  }
13
11
 
14
12
  type OrderByWithFractionalIndexOptions<Ve> = OrderByOptions<Ve> & {
15
- setSizeCallback?: (getSize: () => number) => void
13
+ setSizeCallback?: SetSizeCallback
16
14
  setWindowFn?: (
17
15
  windowFn: (options: { offset?: number; limit?: number }) => void,
18
16
  ) => void
19
17
  }
20
18
 
21
- /**
22
- * Orders the elements and limits the number of results, with optional offset
23
- * This requires a keyed stream, and uses the `topK` operator to order all the elements.
24
- *
25
- * @param valueExtractor - A function that extracts the value to order by from the element
26
- * @param options - An optional object containing comparator, limit and offset properties
27
- * @returns A piped operator that orders the elements and limits the number of results
28
- */
29
- export function orderBy<T extends KeyValue<unknown, unknown>, Ve = unknown>(
30
- valueExtractor: (
31
- value: T extends KeyValue<unknown, infer V> ? V : never,
32
- ) => Ve,
33
- options?: OrderByOptions<Ve>,
34
- ) {
35
- const limit = options?.limit ?? Infinity
36
- const offset = options?.offset ?? 0
37
- const comparator =
38
- options?.comparator ??
39
- ((a, b) => {
40
- // Default to JS like ordering
41
- if (a === b) return 0
42
- if (a < b) return -1
43
- return 1
44
- })
45
-
46
- return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {
47
- type KeyType = T extends KeyValue<infer K, unknown> ? K : never
48
-
49
- return stream.pipe(
50
- map(
51
- ([key, value]) =>
52
- [
53
- null,
54
- [
55
- key,
56
- valueExtractor(
57
- value as T extends KeyValue<unknown, infer V> ? V : never,
58
- ),
59
- ],
60
- ] as KeyValue<null, [KeyType, Ve]>,
61
- ),
62
- topK((a, b) => comparator(a[1], b[1]), { limit, offset }),
63
- map(([_, [key]]) => [key, null] as KeyValue<KeyType, null>),
64
- innerJoin(stream),
65
- map(([key, value]) => {
66
- return [key, value[1]] as T
67
- }),
68
- consolidate(),
69
- )
70
- }
71
- }
72
-
73
19
  /**
74
20
  * Orders the elements and limits the number of results, with optional offset and
75
- * annotates the value with the index.
76
- * This requires a keyed stream, and uses the `topKWithIndex` operator to order all the elements.
21
+ * annotates the value with a fractional index.
22
+ * This requires a keyed stream, and uses the `topKWithFractionalIndex` operator to order all the elements.
77
23
  *
78
24
  * @param valueExtractor - A function that extracts the value to order by from the element
79
25
  * @param options - An optional object containing comparator, limit and offset properties
80
26
  * @returns A piped operator that orders the elements and limits the number of results
81
27
  */
82
- export function orderByWithIndex<
83
- T extends KeyValue<unknown, unknown>,
84
- Ve = unknown,
85
- >(
86
- valueExtractor: (
87
- value: T extends KeyValue<unknown, infer V> ? V : never,
88
- ) => Ve,
89
- options?: OrderByOptions<Ve>,
90
- ) {
91
- const limit = options?.limit ?? Infinity
92
- const offset = options?.offset ?? 0
93
- const comparator =
94
- options?.comparator ??
95
- ((a, b) => {
96
- // Default to JS like ordering
97
- if (a === b) return 0
98
- if (a < b) return -1
99
- return 1
100
- })
101
-
102
- return (
103
- stream: IStreamBuilder<T>,
104
- ): IStreamBuilder<
105
- KeyValue<
106
- T extends KeyValue<infer K, unknown> ? K : never,
107
- [T extends KeyValue<unknown, infer V> ? V : never, number]
108
- >
109
- > => {
110
- type KeyType = T extends KeyValue<infer K, unknown> ? K : never
111
- type ValueType = T extends KeyValue<unknown, infer V> ? V : never
112
-
113
- return stream.pipe(
114
- map(
115
- ([key, value]) =>
116
- [
117
- null,
118
- [
119
- key,
120
- valueExtractor(
121
- value as T extends KeyValue<unknown, infer V> ? V : never,
122
- ),
123
- ],
124
- ] as KeyValue<null, [KeyType, Ve]>,
125
- ),
126
- topKWithIndex((a, b) => comparator(a[1], b[1]), { limit, offset }),
127
- map(([_, [[key], index]]) => [key, index] as KeyValue<KeyType, number>),
128
- innerJoin(stream),
129
- map(([key, [index, value]]) => {
130
- return [key, [value, index]] as KeyValue<KeyType, [ValueType, number]>
131
- }),
132
- consolidate(),
133
- )
134
- }
135
- }
136
-
137
- export function orderByWithFractionalIndexBase<
28
+ export function orderByWithFractionalIndex<
138
29
  T extends KeyValue<unknown, unknown>,
139
30
  Ve = unknown,
140
31
  >(
141
- topKFunction: typeof topKWithFractionalIndex,
142
32
  valueExtractor: (
143
33
  value: T extends KeyValue<unknown, infer V> ? V : never,
144
34
  ) => Ve,
@@ -164,7 +54,7 @@ export function orderByWithFractionalIndexBase<
164
54
  stream: IStreamBuilder<T>,
165
55
  ): IStreamBuilder<[KeyType, [ValueType, string]]> => {
166
56
  return stream.pipe(
167
- topKFunction(
57
+ topKWithFractionalIndex(
168
58
  (a: ValueType, b: ValueType) =>
169
59
  comparator(valueExtractor(a), valueExtractor(b)),
170
60
  {
@@ -178,28 +68,3 @@ export function orderByWithFractionalIndexBase<
178
68
  )
179
69
  }
180
70
  }
181
-
182
- /**
183
- * Orders the elements and limits the number of results, with optional offset and
184
- * annotates the value with a fractional index.
185
- * This requires a keyed stream, and uses the `topKWithFractionalIndex` operator to order all the elements.
186
- *
187
- * @param valueExtractor - A function that extracts the value to order by from the element
188
- * @param options - An optional object containing comparator, limit and offset properties
189
- * @returns A piped operator that orders the elements and limits the number of results
190
- */
191
- export function orderByWithFractionalIndex<
192
- T extends KeyValue<unknown, unknown>,
193
- Ve = unknown,
194
- >(
195
- valueExtractor: (
196
- value: T extends KeyValue<unknown, infer V> ? V : never,
197
- ) => Ve,
198
- options?: OrderByWithFractionalIndexOptions<Ve>,
199
- ) {
200
- return orderByWithFractionalIndexBase(
201
- topKWithFractionalIndex,
202
- valueExtractor,
203
- options,
204
- )
205
- }
@@ -5,12 +5,23 @@ import { Index } from '../indexes.js'
5
5
  import type { DifferenceStreamReader } from '../graph.js'
6
6
  import type { IStreamBuilder, KeyValue } from '../types.js'
7
7
 
8
+ /** The key identity of a Map: SameValueZero, so NaN equals NaN. */
9
+ function sameOutput(left: unknown, right: unknown): boolean {
10
+ return left === right || (left !== left && right !== right)
11
+ }
12
+
13
+ /** The value a Map stores as a key: -0 becomes 0. */
14
+ function asMapKey<T>(value: T): T {
15
+ return (value === 0 ? 0 : value) as T
16
+ }
17
+
8
18
  /**
9
19
  * Base operator for reduction operations (version-free)
10
20
  */
11
21
  export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
12
- #index = new Index<K, V1>()
13
- #indexOut = new Index<K, V2>()
22
+ // Reduce never reads presence, so its indexes skip that bookkeeping.
23
+ #index: Index<K, V1>
24
+ #indexOut = new Index<K, V2>({ trackPresence: false })
14
25
  #f: (values: Array<[V1, number]>) => Array<[V2, number]>
15
26
 
16
27
  constructor(
@@ -18,8 +29,10 @@ export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
18
29
  inputA: DifferenceStreamReader<[K, V1]>,
19
30
  output: DifferenceStreamWriter<[K, V2]>,
20
31
  f: (values: Array<[V1, number]>) => Array<[V2, number]>,
32
+ options?: ReduceOptions,
21
33
  ) {
22
34
  super(id, inputA, output)
35
+ this.#index = new Index<K, V1>({ ...options, trackPresence: false })
23
36
  this.#f = f
24
37
  }
25
38
 
@@ -41,6 +54,36 @@ export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
41
54
  const currOut = this.#indexOut.get(key)
42
55
  const out = this.#f(curr)
43
56
 
57
+ // Most keys have at most one output before and after. Compare the two
58
+ // directly, by the SameValueZero identity the maps below use.
59
+ if (out.length <= 1 && currOut.length <= 1) {
60
+ // Earlier outputs were stored as Map keys, so only the new one needs it.
61
+ const before = currOut[0]
62
+ const after = out[0]
63
+ const value = after === undefined ? undefined : asMapKey(after[0])
64
+ if (
65
+ before !== undefined &&
66
+ after !== undefined &&
67
+ sameOutput(before[0], value)
68
+ ) {
69
+ const delta = after[1] - before[1]
70
+ if (delta !== 0) {
71
+ result.push([[key, before[0]], delta])
72
+ this.#indexOut.addValue(key, [before[0], delta])
73
+ }
74
+ continue
75
+ }
76
+ if (before !== undefined) {
77
+ result.push([[key, before[0]], -before[1]])
78
+ this.#indexOut.addValue(key, [before[0], -before[1]])
79
+ }
80
+ if (after !== undefined && after[1] !== 0) {
81
+ result.push([[key, value as V2], after[1]])
82
+ this.#indexOut.addValue(key, [value as V2, after[1]])
83
+ }
84
+ continue
85
+ }
86
+
44
87
  // Create maps for current and previous outputs using values directly as keys
45
88
  const newOutputMap = new Map<V2, number>()
46
89
  const oldOutputMap = new Map<V2, number>()
@@ -97,6 +140,16 @@ export class ReduceOperator<K, V1, V2> extends UnaryOperator<[K, V1], [K, V2]> {
97
140
  }
98
141
  }
99
142
 
143
+ /** @internal groupBy's option; its producer must keep the identity exact. */
144
+ export type ReduceOptions = {
145
+ /**
146
+ * Input values are `[identity, value]` with a string or number identity
147
+ * that determines the value: two inputs under one key with an equal
148
+ * identity are equal. They merge without a structural hash.
149
+ */
150
+ prefixIdentity?: boolean
151
+ }
152
+
100
153
  /**
101
154
  * Reduces the elements in the stream by key (version-free)
102
155
  */
@@ -105,7 +158,10 @@ export function reduce<
105
158
  V1Type extends (T extends KeyValue<KType, infer V> ? V : never),
106
159
  R,
107
160
  T,
108
- >(f: (values: Array<[V1Type, number]>) => Array<[R, number]>) {
161
+ >(
162
+ f: (values: Array<[V1Type, number]>) => Array<[R, number]>,
163
+ options?: ReduceOptions,
164
+ ) {
109
165
  return (stream: IStreamBuilder<T>): IStreamBuilder<KeyValue<KType, R>> => {
110
166
  const output = new StreamBuilder<KeyValue<KType, R>>(
111
167
  stream.graph,
@@ -116,6 +172,7 @@ export function reduce<
116
172
  stream.connectReader() as DifferenceStreamReader<KeyValue<KType, V1Type>>,
117
173
  output.writer,
118
174
  f,
175
+ options,
119
176
  )
120
177
  stream.graph.addOperator(operator)
121
178
  return output
@@ -62,6 +62,11 @@ export type TopKMoveChanges<V> = {
62
62
  */
63
63
  export interface TopK<V> {
64
64
  size: number
65
+ /**
66
+ * The number of leading window values that satisfy a predicate that holds
67
+ * for a prefix of the window order. Optional; a caller falls back to size.
68
+ */
69
+ countLeadingInWindow?: (predicate: (value: V) => boolean) => number
65
70
  insert: (value: V) => TopKChanges<V>
66
71
  delete: (value: V) => TopKChanges<V>
67
72
  }
@@ -95,6 +100,16 @@ export class TopKArray<V> implements TopK<V> {
95
100
  return Math.max(0, Math.min(limit, available))
96
101
  }
97
102
 
103
+ countLeadingInWindow(predicate: (value: V) => boolean): number {
104
+ let count = 0
105
+ const end = this.#topKStart + this.size
106
+ for (let index = this.#topKStart; index < end; index++) {
107
+ if (!predicate(getValue(this.#sortedValues[index]!))) break
108
+ count++
109
+ }
110
+ return count
111
+ }
112
+
98
113
  /**
99
114
  * Moves the topK window
100
115
  */
@@ -68,6 +68,13 @@ export class TopKState<K extends string | number, T> {
68
68
  return this.#topK.size
69
69
  }
70
70
 
71
+ countLeadingInWindow(predicate: (value: T) => boolean): number {
72
+ return (
73
+ this.#topK.countLeadingInWindow?.(([, value]) => predicate(value)) ??
74
+ this.#topK.size
75
+ )
76
+ }
77
+
71
78
  get isEmpty(): boolean {
72
79
  return this.#multiplicities.size === 0 && this.#topK.size === 0
73
80
  }
@@ -8,19 +8,27 @@ import {
8
8
  topKBatch,
9
9
  } from './topKState.js'
10
10
  import { TopKArray, createKeyedComparator } from './topKArray.js'
11
- import type { IndexedValue, TopK } from './topKArray.js'
11
+ import type { IndexedValue } from './topKArray.js'
12
12
  import type { DifferenceStreamReader } from '../graph.js'
13
13
  import type { IStreamBuilder, PipedOperator } from '../types.js'
14
14
 
15
15
  export interface TopKWithFractionalIndexOptions {
16
16
  limit?: number
17
17
  offset?: number
18
- setSizeCallback?: (getSize: () => number) => void
18
+ setSizeCallback?: SetSizeCallback
19
19
  setWindowFn?: (
20
20
  windowFn: (options: { offset?: number; limit?: number }) => void,
21
21
  ) => void
22
22
  }
23
23
 
24
+ /**
25
+ * Receives the window size. Given a predicate that holds for a prefix of the
26
+ * window order, the getter counts only the leading values that satisfy it.
27
+ */
28
+ export type SetSizeCallback = (
29
+ getSize: (counts?: (value: unknown) => boolean) => number,
30
+ ) => void
31
+
24
32
  /**
25
33
  * Operator for fractional indexed topK operations
26
34
  * This operator maintains fractional indices for sorted elements
@@ -42,24 +50,15 @@ export class TopKWithFractionalIndexOperator<
42
50
  super(id, inputA, output)
43
51
  const limit = options.limit ?? Infinity
44
52
  const offset = options.offset ?? 0
45
- const topK = this.createTopK(
46
- offset,
47
- limit,
48
- createKeyedComparator(comparator),
53
+ this.#state = new TopKState(
54
+ new TopKArray(offset, limit, createKeyedComparator(comparator)),
55
+ )
56
+ options.setSizeCallback?.((counts) =>
57
+ counts ? this.#state.countLeadingInWindow(counts) : this.#state.size,
49
58
  )
50
- this.#state = new TopKState(topK)
51
- options.setSizeCallback?.(() => this.#state.size)
52
59
  options.setWindowFn?.(this.moveTopK.bind(this))
53
60
  }
54
61
 
55
- protected createTopK(
56
- offset: number,
57
- limit: number,
58
- comparator: (a: [K, T], b: [K, T]) => number,
59
- ): TopK<[K, T]> {
60
- return new TopKArray(offset, limit, comparator)
61
- }
62
-
63
62
  /**
64
63
  * Moves the topK window based on the provided offset and limit.
65
64
  * Any changes to the topK are sent to the output.
package/src/utils.ts CHANGED
@@ -1,18 +1,5 @@
1
1
  import { isTemporal } from './hashing/temporal.js'
2
2
 
3
- /**
4
- * Simple assertion function for runtime checks.
5
- * Throws an error if the condition is false.
6
- */
7
- export function assert(
8
- condition: unknown,
9
- message?: string,
10
- ): asserts condition {
11
- if (!condition) {
12
- throw new Error(message || `Assertion failed`)
13
- }
14
- }
15
-
16
3
  // JS engines have various limits on how many args can be passed to a function
17
4
  // with a spread operator, so we need to split the operation into chunks
18
5
  // 32767 is the max for Chrome 14, all others are higher
@@ -70,23 +57,6 @@ export function getStringId(
70
57
  return `${type}_${String(value)}`
71
58
  }
72
59
 
73
- export function* concatIterable<T>(
74
- ...iterables: Array<Iterable<T>>
75
- ): Iterable<T> {
76
- for (const iterable of iterables) {
77
- yield* iterable
78
- }
79
- }
80
-
81
- export function* mapIterable<T, U>(
82
- it: Iterable<T>,
83
- fn: (t: T) => U,
84
- ): Iterable<U> {
85
- for (const t of it) {
86
- yield fn(t)
87
- }
88
- }
89
-
90
60
  export type HRange = [number, number] // half-open [start, end[ i.e. end is exclusive
91
61
 
92
62
  /**
@@ -1,35 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const graph = require("../graph.cjs");
4
- const d2 = require("../d2.cjs");
5
- const reduce = require("./reduce.cjs");
6
- class CountOperator extends reduce.ReduceOperator {
7
- constructor(id, inputA, output) {
8
- const countInner = (vals) => {
9
- let totalCount = 0;
10
- for (const [_, diff] of vals) {
11
- totalCount += diff;
12
- }
13
- return [[totalCount, 1]];
14
- };
15
- super(id, inputA, output, countInner);
16
- }
17
- }
18
- function count() {
19
- return (stream) => {
20
- const output = new d2.StreamBuilder(
21
- stream.graph,
22
- new graph.DifferenceStreamWriter()
23
- );
24
- const operator = new CountOperator(
25
- stream.graph.getNextOperatorId(),
26
- stream.connectReader(),
27
- output.writer
28
- );
29
- stream.graph.addOperator(operator);
30
- return output;
31
- };
32
- }
33
- exports.CountOperator = CountOperator;
34
- exports.count = count;
35
- //# sourceMappingURL=count.cjs.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"count.cjs","sources":["../../../src/operators/count.ts"],"sourcesContent":["import { DifferenceStreamWriter } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport { ReduceOperator } from './reduce.js'\nimport type { DifferenceStreamReader } from '../graph.js'\nimport type { IStreamBuilder, KeyValue } from '../types.js'\n\n/**\n * Operator that counts elements by key (version-free)\n */\nexport class CountOperator<K, V> extends ReduceOperator<K, V, number> {\n constructor(\n id: number,\n inputA: DifferenceStreamReader<[K, V]>,\n output: DifferenceStreamWriter<[K, number]>,\n ) {\n const countInner = (vals: Array<[V, number]>): Array<[number, number]> => {\n let totalCount = 0\n for (const [_, diff] of vals) {\n totalCount += diff\n }\n return [[totalCount, 1]]\n }\n\n super(id, inputA, output, countInner)\n }\n}\n\n/**\n * Counts the number of elements by key (version-free)\n */\nexport function count<\n KType extends (T extends KeyValue<infer K, infer _V> ? K : never),\n VType extends (T extends KeyValue<KType, infer V> ? V : never),\n T,\n>() {\n return (\n stream: IStreamBuilder<T>,\n ): IStreamBuilder<KeyValue<KType, number>> => {\n const output = new StreamBuilder<KeyValue<KType, number>>(\n stream.graph,\n new DifferenceStreamWriter<KeyValue<KType, number>>(),\n )\n const operator = new CountOperator<KType, VType>(\n stream.graph.getNextOperatorId(),\n stream.connectReader() as DifferenceStreamReader<KeyValue<KType, VType>>,\n output.writer,\n )\n stream.graph.addOperator(operator)\n return output\n }\n}\n"],"names":["ReduceOperator","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;;AASO,MAAM,sBAA4BA,OAAAA,eAA6B;AAAA,EACpE,YACE,IACA,QACA,QACA;AACA,UAAM,aAAa,CAAC,SAAsD;AACxE,UAAI,aAAa;AACjB,iBAAW,CAAC,GAAG,IAAI,KAAK,MAAM;AAC5B,sBAAc;AAAA,MAChB;AACA,aAAO,CAAC,CAAC,YAAY,CAAC,CAAC;AAAA,IACzB;AAEA,UAAM,IAAI,QAAQ,QAAQ,UAAU;AAAA,EACtC;AACF;AAKO,SAAS,QAIZ;AACF,SAAO,CACL,WAC4C;AAC5C,UAAM,SAAS,IAAIC,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAAgD;AAEtD,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,IAAA;AAET,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;;;"}
@@ -1,19 +0,0 @@
1
- import { DifferenceStreamWriter, DifferenceStreamReader } from "../graph.cjs";
2
- import { ReduceOperator } from "./reduce.cjs";
3
- import { IStreamBuilder, KeyValue } from "../types.cjs";
4
- /**
5
- * Operator that counts elements by key (version-free)
6
- */
7
- export declare class CountOperator<K, V> extends ReduceOperator<K, V, number> {
8
- constructor(id: number, inputA: DifferenceStreamReader<[
9
- K,
10
- V
11
- ]>, output: DifferenceStreamWriter<[
12
- K,
13
- number
14
- ]>);
15
- }
16
- /**
17
- * Counts the number of elements by key (version-free)
18
- */
19
- export declare function count<KType extends (T extends KeyValue<infer K, infer _V> ? K : never), VType extends (T extends KeyValue<KType, infer V> ? V : never), T>(): (stream: IStreamBuilder<T>) => IStreamBuilder<KeyValue<KType, number>>;
@@ -1,39 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const graph = require("../graph.cjs");
4
- const d2 = require("../d2.cjs");
5
- class DebugOperator extends graph.UnaryOperator {
6
- #name;
7
- #indent;
8
- constructor(id, inputA, output, name, indent = false) {
9
- super(id, inputA, output);
10
- this.#name = name;
11
- this.#indent = indent;
12
- }
13
- run() {
14
- for (const message of this.inputMessages()) {
15
- console.log(`debug ${this.#name} data: ${message.toString(this.#indent)}`);
16
- this.output.sendData(message);
17
- }
18
- }
19
- }
20
- function debug(name, indent = false) {
21
- return (stream) => {
22
- const output = new d2.StreamBuilder(
23
- stream.graph,
24
- new graph.DifferenceStreamWriter()
25
- );
26
- const operator = new DebugOperator(
27
- stream.graph.getNextOperatorId(),
28
- stream.connectReader(),
29
- output.writer,
30
- name,
31
- indent
32
- );
33
- stream.graph.addOperator(operator);
34
- return output;
35
- };
36
- }
37
- exports.DebugOperator = DebugOperator;
38
- exports.debug = debug;
39
- //# sourceMappingURL=debug.cjs.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"debug.cjs","sources":["../../../src/operators/debug.ts"],"sourcesContent":["import { DifferenceStreamWriter, UnaryOperator } from '../graph.js'\nimport { StreamBuilder } from '../d2.js'\nimport type { IStreamBuilder, PipedOperator } from '../types.js'\nimport type { DifferenceStreamReader } from '../graph.js'\n\n/**\n * Operator that logs debug information about the stream\n */\nexport class DebugOperator<T> extends UnaryOperator<T> {\n #name: string\n #indent: boolean\n\n constructor(\n id: number,\n inputA: DifferenceStreamReader<T>,\n output: DifferenceStreamWriter<T>,\n name: string,\n indent: boolean = false,\n ) {\n super(id, inputA, output)\n this.#name = name\n this.#indent = indent\n }\n\n run(): void {\n for (const message of this.inputMessages()) {\n console.log(`debug ${this.#name} data: ${message.toString(this.#indent)}`)\n this.output.sendData(message)\n }\n }\n}\n\n/**\n * Logs debug information about the stream using console.log\n * @param name - The name to prefix debug messages with\n * @param indent - Whether to indent the debug output\n */\nexport function debug<T>(\n name: string,\n indent: boolean = false,\n): PipedOperator<T, T> {\n return (stream: IStreamBuilder<T>): IStreamBuilder<T> => {\n const output = new StreamBuilder<T>(\n stream.graph,\n new DifferenceStreamWriter<T>(),\n )\n const operator = new DebugOperator<T>(\n stream.graph.getNextOperatorId(),\n stream.connectReader(),\n output.writer,\n name,\n indent,\n )\n stream.graph.addOperator(operator)\n return output\n }\n}\n"],"names":["UnaryOperator","StreamBuilder","DifferenceStreamWriter"],"mappings":";;;;AAQO,MAAM,sBAAyBA,MAAAA,cAAiB;AAAA,EACrD;AAAA,EACA;AAAA,EAEA,YACE,IACA,QACA,QACA,MACA,SAAkB,OAClB;AACA,UAAM,IAAI,QAAQ,MAAM;AACxB,SAAK,QAAQ;AACb,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,MAAY;AACV,eAAW,WAAW,KAAK,iBAAiB;AAC1C,cAAQ,IAAI,SAAS,KAAK,KAAK,UAAU,QAAQ,SAAS,KAAK,OAAO,CAAC,EAAE;AACzE,WAAK,OAAO,SAAS,OAAO;AAAA,IAC9B;AAAA,EACF;AACF;AAOO,SAAS,MACd,MACA,SAAkB,OACG;AACrB,SAAO,CAAC,WAAiD;AACvD,UAAM,SAAS,IAAIC,GAAAA;AAAAA,MACjB,OAAO;AAAA,MACP,IAAIC,MAAAA,uBAAA;AAAA,IAA0B;AAEhC,UAAM,WAAW,IAAI;AAAA,MACnB,OAAO,MAAM,kBAAA;AAAA,MACb,OAAO,cAAA;AAAA,MACP,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IAAA;AAEF,WAAO,MAAM,YAAY,QAAQ;AACjC,WAAO;AAAA,EACT;AACF;;;"}
@@ -1,16 +0,0 @@
1
- import { DifferenceStreamWriter, UnaryOperator, DifferenceStreamReader } from "../graph.cjs";
2
- import { PipedOperator } from "../types.cjs";
3
- /**
4
- * Operator that logs debug information about the stream
5
- */
6
- export declare class DebugOperator<T> extends UnaryOperator<T> {
7
- #private;
8
- constructor(id: number, inputA: DifferenceStreamReader<T>, output: DifferenceStreamWriter<T>, name: string, indent?: boolean);
9
- run(): void;
10
- }
11
- /**
12
- * Logs debug information about the stream using console.log
13
- * @param name - The name to prefix debug messages with
14
- * @param indent - Whether to indent the debug output
15
- */
16
- export declare function debug<T>(name: string, indent?: boolean): PipedOperator<T, T>;
@@ -1,19 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const map = require("./map.cjs");
4
- const join = require("./join.cjs");
5
- const consolidate = require("./consolidate.cjs");
6
- function filterBy(other) {
7
- return (stream) => {
8
- const otherKeys = other.pipe(
9
- map.map(([key, _]) => [key, null])
10
- );
11
- return stream.pipe(
12
- join.innerJoin(otherKeys),
13
- map.map(([key, [value, _]]) => [key, value]),
14
- consolidate.consolidate()
15
- );
16
- };
17
- }
18
- exports.filterBy = filterBy;
19
- //# sourceMappingURL=filterBy.cjs.map