@shutterstock/p-map-iterable 1.0.11 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -16
- package/dist/src/blocking-queue.js +61 -70
- package/dist/src/blocking-queue.js.map +1 -1
- package/dist/src/blocking-queue.test.js +70 -71
- package/dist/src/blocking-queue.test.js.map +1 -1
- package/dist/src/iterable-mapper.d.ts +173 -30
- package/dist/src/iterable-mapper.d.ts.map +1 -1
- package/dist/src/iterable-mapper.js +255 -141
- package/dist/src/iterable-mapper.js.map +1 -1
- package/dist/src/iterable-mapper.test.js +347 -720
- package/dist/src/iterable-mapper.test.js.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.d.ts +44 -29
- package/dist/src/iterable-queue-mapper-simple.d.ts.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.js +65 -57
- package/dist/src/iterable-queue-mapper-simple.js.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.test.js +34 -35
- package/dist/src/iterable-queue-mapper-simple.test.js.map +1 -1
- package/dist/src/iterable-queue-mapper.d.ts +38 -58
- package/dist/src/iterable-queue-mapper.d.ts.map +1 -1
- package/dist/src/iterable-queue-mapper.js +37 -42
- package/dist/src/iterable-queue-mapper.js.map +1 -1
- package/dist/src/iterable-queue-mapper.test.js +41 -81
- package/dist/src/iterable-queue-mapper.test.js.map +1 -1
- package/dist/src/iterable-queue.js +8 -11
- package/dist/src/iterable-queue.js.map +1 -1
- package/dist/src/iterable-queue.test.js +43 -122
- package/dist/src/iterable-queue.test.js.map +1 -1
- package/package.json +8 -3
|
@@ -1,27 +1,157 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.IterableMapper = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
4
|
//
|
|
6
5
|
// 2021-08-25 - Initially based on: https://raw.githubusercontent.com/sindresorhus/p-map/main/index.js
|
|
7
6
|
//
|
|
8
|
-
|
|
7
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
8
|
+
const AggregateError = require('aggregate-error');
|
|
9
9
|
const iterable_queue_1 = require("./iterable-queue");
|
|
10
10
|
/**
|
|
11
|
-
* Iterates over a source iterable with specified concurrency
|
|
11
|
+
* Iterates over a source iterable / generator with specified `concurrency`,
|
|
12
12
|
* calling the `mapper` on each iterated item, and storing the
|
|
13
|
-
* `mapper` result in a queue of
|
|
13
|
+
* `mapper` result in a queue of `maxUnread` size, before
|
|
14
14
|
* being iterated / read by the caller.
|
|
15
15
|
*
|
|
16
16
|
* @remarks
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* ### Typical Use Case
|
|
19
|
+
* - Prefetching items from an async I/O source
|
|
20
|
+
* - In the simple sequential (`concurrency: 1`) case, allows items to be prefetched async, preserving order, while caller processes an item
|
|
21
|
+
* - Can allow parallel prefetches for sources that allow for out of order reads (`concurrency: 2+`)
|
|
22
|
+
* - Prevents the producer from racing ahead of the consumer if `maxUnread` is reached
|
|
21
23
|
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
24
|
+
* ### Error Handling
|
|
25
|
+
* The mapper should ideally handle all errors internally to enable error handling
|
|
26
|
+
* closest to where they occur. However, if errors do escape the mapper:
|
|
27
|
+
*
|
|
28
|
+
* When `stopOnMapperError` is true (default):
|
|
29
|
+
* - First error immediately stops processing
|
|
30
|
+
* - Error is thrown from the `AsyncIterator`'s next() call
|
|
31
|
+
*
|
|
32
|
+
* When `stopOnMapperError` is false:
|
|
33
|
+
* - Processing continues despite errors
|
|
34
|
+
* - All errors are collected and thrown together
|
|
35
|
+
* - Errors are thrown as `AggregateError` after all items complete
|
|
36
|
+
*
|
|
37
|
+
* ### Usage
|
|
38
|
+
* - Items are exposed to the `mapper` via an iterator or async iterator (this includes generator and async generator functions)
|
|
39
|
+
* - IMPORTANT: `mapper` method not be invoked when `maxUnread` is reached, until items are consumed
|
|
40
|
+
* - The iterable will set `done` when the `input` has indicated `done` and all `mapper` promises have resolved
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
*
|
|
44
|
+
* ### Typical Processing Loop without `IterableMapper`
|
|
45
|
+
*
|
|
46
|
+
* ```typescript
|
|
47
|
+
* const source = new SomeSource();
|
|
48
|
+
* const sourceIds = [1, 2,... 1000];
|
|
49
|
+
* const sink = new SomeSink();
|
|
50
|
+
* for (const sourceId of sourceIds) {
|
|
51
|
+
* const item = await source.read(sourceId); // takes 300 ms of I/O wait, no CPU
|
|
52
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
53
|
+
* await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
54
|
+
* }
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* Each iteration takes 820ms total, but we waste time waiting for I/O.
|
|
58
|
+
* We could prefetch the next read (300ms) while processing (20ms) and writing (500ms),
|
|
59
|
+
* without changing the order of reads or writes.
|
|
60
|
+
*
|
|
61
|
+
* @example
|
|
62
|
+
*
|
|
63
|
+
* ### Using `IterableMapper` as Prefetcher with Blocking Sequential Writes
|
|
64
|
+
*
|
|
65
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads and and writes are sequential and blocking (unchanged).
|
|
66
|
+
*
|
|
67
|
+
* ```typescript
|
|
68
|
+
* const source = new SomeSource();
|
|
69
|
+
* const sourceIds = [1, 2,... 1000];
|
|
70
|
+
* // Pre-reads up to 8 items serially and releases in sequential order
|
|
71
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
72
|
+
* async (sourceId) => source.read(sourceId),
|
|
73
|
+
* { concurrency: 1, maxUnread: 10 }
|
|
74
|
+
* );
|
|
75
|
+
* const sink = new SomeSink();
|
|
76
|
+
* for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
77
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
78
|
+
* await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
79
|
+
* }
|
|
80
|
+
* ```
|
|
81
|
+
*
|
|
82
|
+
* This reduces iteration time to 520ms by overlapping reads with processing/writing.
|
|
83
|
+
*
|
|
84
|
+
* @example
|
|
85
|
+
*
|
|
86
|
+
* ### Using `IterableMapper` as Prefetcher with Background Sequential Writes with `IterableQueueMapperSimple`
|
|
87
|
+
*
|
|
88
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
89
|
+
* `concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
90
|
+
*
|
|
91
|
+
* ```typescript
|
|
92
|
+
* const source = new SomeSource();
|
|
93
|
+
* const sourceIds = [1, 2,... 1000];
|
|
94
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
95
|
+
* async (sourceId) => source.read(sourceId),
|
|
96
|
+
* { concurrency: 1, maxUnread: 10 }
|
|
97
|
+
* );
|
|
98
|
+
* const sink = new SomeSink();
|
|
99
|
+
* const flusher = new IterableQueueMapperSimple(
|
|
100
|
+
* async (outputItem) => sink.write(outputItem),
|
|
101
|
+
* { concurrency: 1 }
|
|
102
|
+
* );
|
|
103
|
+
* for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
104
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
105
|
+
* await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
106
|
+
* }
|
|
107
|
+
* // Wait for all writes to complete
|
|
108
|
+
* await flusher.onIdle();
|
|
109
|
+
* // Check for errors
|
|
110
|
+
* if (flusher.errors.length > 0) {
|
|
111
|
+
* // ...
|
|
112
|
+
* }
|
|
113
|
+
* ```
|
|
114
|
+
*
|
|
115
|
+
* This reduces iteration time to about `max((max(readTime, writeTime) - cpuOpTime, cpuOpTime))`
|
|
116
|
+
* by overlapping reads and writes with the CPU processing step.
|
|
117
|
+
* In this contrived example, the loop time is reduced to 500ms - 20ms = 480ms.
|
|
118
|
+
* In cases where the CPU usage time is higher, the impact can be greater.
|
|
119
|
+
*
|
|
120
|
+
* @example
|
|
121
|
+
*
|
|
122
|
+
* ### Using `IterableMapper` as Prefetcher with Out of Order Reads and Background Out of Order Writes with `IterableQueueMapperSimple`
|
|
123
|
+
*
|
|
124
|
+
* For maximum throughput, allow out of order reads and writes with
|
|
125
|
+
* `IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
126
|
+
* `IterableQueueMapperSimple` (to handle errors at end without custom iteration and applying backpressure to block further enqueues when `concurrency` items are in process):
|
|
127
|
+
*
|
|
128
|
+
* ```typescript
|
|
129
|
+
* const source = new SomeSource();
|
|
130
|
+
* const sourceIds = [1, 2,... 1000];
|
|
131
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
132
|
+
* async (sourceId) => source.read(sourceId),
|
|
133
|
+
* { concurrency: 10, maxUnread: 20 }
|
|
134
|
+
* );
|
|
135
|
+
* const sink = new SomeSink();
|
|
136
|
+
* const flusher = new IterableQueueMapperSimple(
|
|
137
|
+
* async (outputItem) => sink.write(outputItem),
|
|
138
|
+
* { concurrency: 10 }
|
|
139
|
+
* );
|
|
140
|
+
* for await (const item of sourcePrefetcher) { // typically will not block
|
|
141
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
142
|
+
* await flusher.enqueue(outputItem); // typically will not block
|
|
143
|
+
* }
|
|
144
|
+
* // Wait for all writes to complete
|
|
145
|
+
* await flusher.onIdle();
|
|
146
|
+
* // Check for errors
|
|
147
|
+
* if (flusher.errors.length > 0) {
|
|
148
|
+
* // ...
|
|
149
|
+
* }
|
|
150
|
+
* ```
|
|
151
|
+
*
|
|
152
|
+
* This reduces iteration time to about 20ms by overlapping reads and writes with the CPU processing step.
|
|
153
|
+
* In this contrived (but common) example we would get a 41x improvement in throughput, removing 97.5% of
|
|
154
|
+
* the time to process each item and fully utilizing the CPU time available in the JS event loop.
|
|
25
155
|
*
|
|
26
156
|
* @category Iterable Input
|
|
27
157
|
*/
|
|
@@ -29,23 +159,11 @@ class IterableMapper {
|
|
|
29
159
|
/**
|
|
30
160
|
* Create a new `IterableMapper`
|
|
31
161
|
*
|
|
32
|
-
* @param input Iterated over concurrently in the `mapper` function.
|
|
33
|
-
* @param mapper Function
|
|
34
|
-
* Expected to return a `Promise` or value.
|
|
35
|
-
*
|
|
36
|
-
* The `mapper` *should* handle all errors and not allow an error to be thrown
|
|
37
|
-
* out of the `mapper` function as this enables the best handling of errors
|
|
38
|
-
* closest to the time that they occur.
|
|
39
|
-
*
|
|
40
|
-
* If the `mapper` function does allow an error to be thrown then the
|
|
41
|
-
* `stopOnMapperError` option controls the behavior:
|
|
42
|
-
* - `stopOnMapperError`: `true` - will throw the error
|
|
43
|
-
* out of `next` or the `AsyncIterator` returned from `[Symbol.asyncIterator]`
|
|
44
|
-
* and stop processing.
|
|
45
|
-
* - `stopOnMapperError`: `false` - will continue processing
|
|
46
|
-
* and accumulate the errors to be thrown from `next` or the `AsyncIterator`
|
|
47
|
-
* returned from `[Symbol.asyncIterator]` when all items have been processed.
|
|
162
|
+
* @param input Iterated over concurrently, or serially, in the `mapper` function.
|
|
163
|
+
* @param mapper Function called for every item in `input`. Returns a `Promise` or value.
|
|
48
164
|
* @param options IterableMapper options
|
|
165
|
+
*
|
|
166
|
+
* @see {@link IterableQueueMapper} for full class documentation
|
|
49
167
|
*/
|
|
50
168
|
constructor(input, mapper, options = {}) {
|
|
51
169
|
this._errors = [];
|
|
@@ -98,21 +216,21 @@ class IterableMapper {
|
|
|
98
216
|
// promise. We need this so we can await the next() calls
|
|
99
217
|
// to stop creating runners before hitting the concurrency limit
|
|
100
218
|
// if the iterable has already been marked as done.
|
|
101
|
-
void (() =>
|
|
219
|
+
void (async () => {
|
|
102
220
|
for (let index = 0; index < concurrency; index++) {
|
|
103
221
|
// Setup the detached runner
|
|
104
222
|
this._activeRunners++;
|
|
105
223
|
// This only waits for the next source item to be iterated
|
|
106
224
|
// It does NOT wait for the mapper to be called for for a consumer to pickup
|
|
107
225
|
// the result out of the unread queue.
|
|
108
|
-
|
|
226
|
+
await this.sourceNext();
|
|
109
227
|
if (this._isIterableDone || this._isRejected) {
|
|
110
228
|
break;
|
|
111
229
|
}
|
|
112
230
|
}
|
|
113
231
|
// Signal that the next() function should now create runners if it sees too few of them
|
|
114
232
|
this._initialRunnersCreated = true;
|
|
115
|
-
})
|
|
233
|
+
})();
|
|
116
234
|
}
|
|
117
235
|
[Symbol.asyncIterator]() {
|
|
118
236
|
return this;
|
|
@@ -123,33 +241,31 @@ class IterableMapper {
|
|
|
123
241
|
*
|
|
124
242
|
* @returns Iterator result
|
|
125
243
|
*/
|
|
126
|
-
next() {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
if (
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
throw new aggregate_error_1.default(this._errors);
|
|
135
|
-
}
|
|
136
|
-
return { value: undefined, done };
|
|
137
|
-
}
|
|
138
|
-
// Check if queue has an item
|
|
139
|
-
const item = yield this._unreadQueue.dequeue();
|
|
140
|
-
if (item === undefined) {
|
|
141
|
-
// We finished - There were no more items
|
|
142
|
-
this.bubbleUpErrors();
|
|
143
|
-
return { value: undefined, done: true };
|
|
244
|
+
async next() {
|
|
245
|
+
// Bail out and release all waiters if there are no more items coming
|
|
246
|
+
const done = this.areWeDone();
|
|
247
|
+
if (done) {
|
|
248
|
+
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
249
|
+
// throw the errors as an aggregate exception
|
|
250
|
+
this._isRejected = true;
|
|
251
|
+
throw new AggregateError(this._errors);
|
|
144
252
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
253
|
+
return { value: undefined, done };
|
|
254
|
+
}
|
|
255
|
+
// Check if queue has an item
|
|
256
|
+
const item = await this._unreadQueue.dequeue();
|
|
257
|
+
if (item === undefined) {
|
|
258
|
+
// We finished - There were no more items
|
|
259
|
+
this.bubbleUpErrors();
|
|
260
|
+
return { value: undefined, done: true };
|
|
261
|
+
}
|
|
262
|
+
this.startARunnerIfNeeded();
|
|
263
|
+
return { value: this.throwIfError(item), done: false };
|
|
148
264
|
}
|
|
149
265
|
bubbleUpErrors() {
|
|
150
266
|
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
151
267
|
// throw the errors as an aggregate exception
|
|
152
|
-
throw new
|
|
268
|
+
throw new AggregateError(this._errors);
|
|
153
269
|
}
|
|
154
270
|
}
|
|
155
271
|
startARunnerIfNeeded() {
|
|
@@ -221,7 +337,7 @@ class IterableMapper {
|
|
|
221
337
|
return item.element;
|
|
222
338
|
}
|
|
223
339
|
/**
|
|
224
|
-
* Get the next item from the
|
|
340
|
+
* Get the next item from the `input` iterable.
|
|
225
341
|
*
|
|
226
342
|
* @remarks
|
|
227
343
|
*
|
|
@@ -235,104 +351,102 @@ class IterableMapper {
|
|
|
235
351
|
* If the read queue + runners = max read queue length then the runner
|
|
236
352
|
* will exit and will be restarted when an item is read from the queue.
|
|
237
353
|
*/
|
|
238
|
-
sourceNext() {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
354
|
+
async sourceNext() {
|
|
355
|
+
if (this._isRejected) {
|
|
356
|
+
this._activeRunners--;
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
// Note: do NOT await a non-async iterable as it will cause next() to be
|
|
360
|
+
// pushed into the event loop, slowing down iteration of non-async iterables.
|
|
361
|
+
let nextItem;
|
|
362
|
+
try {
|
|
363
|
+
if (this._asyncIterator) {
|
|
364
|
+
nextItem = await this._iterator.next();
|
|
365
|
+
}
|
|
366
|
+
else {
|
|
367
|
+
nextItem = this._iterator.next();
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
catch (error) {
|
|
371
|
+
// Iterator protocol / Iterables can throw exceptions - If this happens we have to just stop
|
|
372
|
+
// regardless of stopOnMapperError since we can't iterate any additional items
|
|
373
|
+
this._isRejected = true;
|
|
374
|
+
this._activeRunners--;
|
|
375
|
+
// Push the error onto the unread queue, to be rethrown by next()
|
|
376
|
+
await this._unreadQueue.enqueue({ error });
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const index = this._currentIndex;
|
|
380
|
+
this._currentIndex++;
|
|
381
|
+
if (nextItem.done) {
|
|
382
|
+
this._isIterableDone = true;
|
|
383
|
+
// If there are no active resolvers, then release all the waiters
|
|
384
|
+
if (this._resolvingCount === 0) {
|
|
385
|
+
// At this point the only waiters in the queue are not going to get an item
|
|
386
|
+
// as there are no source items left
|
|
387
|
+
this._unreadQueue.done();
|
|
243
388
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
389
|
+
this._activeRunners--;
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
this._resolvingCount++;
|
|
393
|
+
// This is created as a detached, non-awaited async
|
|
394
|
+
// to allow next() to return while the async mapper is awaited.
|
|
395
|
+
// More next() calls will be made up to the concurrency limit.
|
|
396
|
+
void (async () => {
|
|
397
|
+
//
|
|
398
|
+
// Push an item or error into the read queue
|
|
399
|
+
// Note: once we push an item we end this try/catch as any subsequent errors
|
|
400
|
+
// are errors in this class and not errors thrown by the mapper function.
|
|
401
|
+
// Once we've pushed an item we can't also push an error...
|
|
402
|
+
//
|
|
247
403
|
try {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
nextItem = this._iterator.next();
|
|
404
|
+
const element = nextItem.value;
|
|
405
|
+
if (this._isRejected) {
|
|
406
|
+
this._activeRunners--;
|
|
407
|
+
return;
|
|
253
408
|
}
|
|
409
|
+
const value = await this._mapper(element, index);
|
|
410
|
+
this._resolvingCount--;
|
|
411
|
+
// if (value === pMapSkip) {
|
|
412
|
+
// skippedIndexes.push(index);
|
|
413
|
+
// } else {
|
|
414
|
+
// result[index] = value;
|
|
415
|
+
// }
|
|
416
|
+
// Push item onto the ready queue
|
|
417
|
+
await this._unreadQueue.enqueue({ element: value });
|
|
418
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
254
419
|
}
|
|
255
420
|
catch (error) {
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
const index = this._currentIndex;
|
|
265
|
-
this._currentIndex++;
|
|
266
|
-
if (nextItem.done) {
|
|
267
|
-
this._isIterableDone = true;
|
|
268
|
-
// If there are no active resolvers, then release all the waiters
|
|
269
|
-
if (this._resolvingCount === 0) {
|
|
270
|
-
// At this point the only waiters in the queue are not going to get an item
|
|
271
|
-
// as there are no source items left
|
|
272
|
-
this._unreadQueue.done();
|
|
421
|
+
if (this._options.stopOnMapperError) {
|
|
422
|
+
this._isRejected = true;
|
|
423
|
+
this._activeRunners--;
|
|
424
|
+
// Push the error onto the unread queue, to be rethrown by next()
|
|
425
|
+
await this._unreadQueue.enqueue({ error });
|
|
426
|
+
// Fall through to release a reader
|
|
273
427
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
// This is created as a detached, non-awaited async
|
|
279
|
-
// to allow next() to return while the async mapper is awaited.
|
|
280
|
-
// More next() calls will be made up to the concurrency limit.
|
|
281
|
-
void (() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
282
|
-
//
|
|
283
|
-
// Push an item or error into the read queue
|
|
284
|
-
// Note: once we push an item we end this try/catch as any subsequent errors
|
|
285
|
-
// are errors in this class and not errors thrown by the mapper function.
|
|
286
|
-
// Once we've pushed an item we can't also push an error...
|
|
287
|
-
//
|
|
288
|
-
try {
|
|
289
|
-
const element = nextItem.value;
|
|
290
|
-
if (this._isRejected) {
|
|
291
|
-
this._activeRunners--;
|
|
292
|
-
return;
|
|
293
|
-
}
|
|
294
|
-
const value = yield this._mapper(element, index);
|
|
428
|
+
else {
|
|
429
|
+
// Collect the error but do not stop iterating
|
|
430
|
+
// These will be thrown in an AggregateError at the end
|
|
431
|
+
this._errors.push(error);
|
|
295
432
|
this._resolvingCount--;
|
|
296
|
-
|
|
297
|
-
//
|
|
298
|
-
// } else {
|
|
299
|
-
// result[index] = value;
|
|
300
|
-
// }
|
|
301
|
-
// Push item onto the ready queue
|
|
302
|
-
yield this._unreadQueue.enqueue({ element: value });
|
|
303
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
304
|
-
}
|
|
305
|
-
catch (error) {
|
|
306
|
-
if (this._options.stopOnMapperError) {
|
|
307
|
-
this._isRejected = true;
|
|
308
|
-
this._activeRunners--;
|
|
309
|
-
// Push the error onto the unread queue, to be rethrown by next()
|
|
310
|
-
yield this._unreadQueue.enqueue({ error });
|
|
311
|
-
// Fall through to release a reader
|
|
312
|
-
}
|
|
313
|
-
else {
|
|
314
|
-
// Collect the error but do not stop iterating
|
|
315
|
-
// These will be thrown in an AggregateError at the end
|
|
316
|
-
this._errors.push(error);
|
|
317
|
-
this._resolvingCount--;
|
|
318
|
-
yield this.sourceNext();
|
|
319
|
-
// Return so we don't release a reader since we didn't push an item
|
|
320
|
-
return;
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
//
|
|
324
|
-
// Tasks below are not related to the mapper
|
|
325
|
-
//
|
|
326
|
-
// Bail if read queue length + active runners will hit max unread
|
|
327
|
-
if (this._unreadQueue.length + this._activeRunners > this._options.maxUnread) {
|
|
328
|
-
this._activeRunners--;
|
|
433
|
+
await this.sourceNext();
|
|
434
|
+
// Return so we don't release a reader since we didn't push an item
|
|
329
435
|
return;
|
|
330
436
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
437
|
+
}
|
|
438
|
+
//
|
|
439
|
+
// Tasks below are not related to the mapper
|
|
440
|
+
//
|
|
441
|
+
// Bail if read queue length + active runners will hit max unread
|
|
442
|
+
if (this._unreadQueue.length + this._activeRunners > this._options.maxUnread) {
|
|
443
|
+
this._activeRunners--;
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
// Start myself again
|
|
447
|
+
// Note: this will bail out if it reaches the end of the source iterable
|
|
448
|
+
await this.sourceNext();
|
|
449
|
+
})();
|
|
336
450
|
}
|
|
337
451
|
}
|
|
338
452
|
exports.IterableMapper = IterableMapper;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterable-mapper.js","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"iterable-mapper.js","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":";;;AAAA,EAAE;AACF,sGAAsG;AACtG,EAAE;AACF,8DAA8D;AAC9D,MAAM,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAClD,qDAAiD;AA8EjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AACH,MAAa,cAAc;IAgBzB;;;;;;;;OAQG;IACH,YACE,KAAiD,EACjD,MAAmC,EACnC,UAAiC,EAAE;QArBpB,YAAO,GAAG,EAAa,CAAC;QACjC,mBAAc,GAAG,KAAK,CAAC;QACvB,gBAAW,GAAG,KAAK,CAAC;QACpB,oBAAe,GAAG,KAAK,CAAC;QACxB,mBAAc,GAAG,CAAC,CAAC;QACnB,oBAAe,GAAG,CAAC,CAAC;QACpB,kBAAa,GAAG,CAAC,CAAC;QAClB,2BAAsB,GAAG,KAAK,CAAC;QAgBrC,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,iBAAiB,GAAG,IAAI,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAE7E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,WAAW,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;QAE9D,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE;YAChC,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;SACpD;QAED,oCAAoC;QACpC,IACE,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS;YACvC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,KAAK,SAAS;YAC7C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EACrC;YACA,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;SACzD;QAED,8BAA8B;QAC9B,IACE,CAAC,CACC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACzD,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,CAC/B,EACD;YACA,MAAM,IAAI,SAAS,CACjB,kFAAkF,WAAW,OAAO,OAAO,WAAW,GAAG,CAC1H,CAAC;SACH;QAED,4BAA4B;QAC5B,IACE,CAAC,CACC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAC7B,EACD;YACA,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,OAAO,OAAO,SAAS,GAAG,CACpH,CAAC;SACH;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACvD,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,UAAU,WAAW,IAAI,CACnH,CAAC;SACH;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,8BAAa,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAErD,4BAA4B;QAC5B,IAAK,KAAgC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,SAAS,EAAE;YACzE,8BAA8B;YAC9B,IAAI,CAAC,SAAS,GAAI,KAAgC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC3E,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;SAC5B;aAAM;YACL,IAAI,CAAC,SAAS,GAAI,KAA2B,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;SAClE;QAED,oEAAoE;QACpE,0DAA0D;QAC1D,gEAAgE;QAChE,mDAAmD;QACnD,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,EAAE,EAAE;gBAChD,4BAA4B;gBAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,0DAA0D;gBAC1D,4EAA4E;gBAC5E,sCAAsC;gBACtC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAExB,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,EAAE;oBAC5C,MAAM;iBACP;aACF;YAED,uFAAuF;YACvF,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACrC,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,IAAI;QACf,qEAAqE;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,IAAI,EAAE;YACR,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/D,6CAA6C;gBAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aACxC;YACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;SACnC;QAED,6BAA6B;QAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAC/C,IAAI,IAAI,KAAK,SAAS,EAAE;YACtB,yCAAyC;YACzC,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACzC;QAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YAC/D,6CAA6C;YAC7C,MAAM,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACxC;IACH,CAAC;IAEO,oBAAoB;QAC1B,oEAAoE;QACpE,sFAAsF;QACtF,qCAAqC;QACrC,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC/B,uEAAuE;YACvE,uDAAuD;YACvD,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBAC9C,yEAAyE;gBACzE,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;oBACnD,8DAA8D;oBAC9D,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;wBAC7E,0EAA0E;wBAC1E,iCAAiC;wBAEjC,mDAAmD;wBACnD,IAAI,CAAC,kBAAkB,EAAE,CAAC;qBAC3B;iBACF;aACF;SACF;IACH,CAAC;IAEO,kBAAkB;QACxB,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACrD,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;SAC/E;QAED,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC5E,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;SAC3E;QAED,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;SACnF;QAED,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YAC3B,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;SACtD;QAED,0EAA0E;QAC1E,iCAAiC;QACjC,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,mDAAmD;QACnD,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;IACzB,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,wCAAwC;YACxC,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE;gBAC9B,sCAAsC;gBACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,iCAAiC;oBACjC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;oBAEzB,OAAO,IAAI,CAAC;iBACb;aACF;SACF;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,YAAY,CAAC,IAAmC;QACtD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC;SAClB;aAAM,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE;YACrC,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;SAChD;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO;SACR;QAED,wEAAwE;QACxE,6EAA6E;QAC7E,IAAI,QAAiC,CAAC;QACtC,IAAI;YACF,IAAI,IAAI,CAAC,cAAc,EAAE;gBACvB,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;aACxC;iBAAM;gBACL,QAAQ,GAAI,IAAI,CAAC,SAA+B,CAAC,IAAI,EAAE,CAAC;aACzD;SACF;QAAC,OAAO,KAAK,EAAE;YACd,4FAA4F;YAC5F,8EAA8E;YAC9E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,cAAc,EAAE,CAAC;YAEtB,iEAAiE;YACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAE3C,OAAO;SACR;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;QACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,IAAI,QAAQ,CAAC,IAAI,EAAE;YACjB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAE5B,iEAAiE;YACjE,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE;gBAC9B,2EAA2E;gBAC3E,oCAAoC;gBACpC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;aAC1B;YAED,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO;SACR;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,mDAAmD;QACnD,+DAA+D;QAC/D,8DAA8D;QAC9D,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,EAAE;YACF,4CAA4C;YAC5C,4EAA4E;YAC5E,yEAAyE;YACzE,2DAA2D;YAC3D,EAAE;YACF,IAAI;gBACF,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC;gBAE/B,IAAI,IAAI,CAAC,WAAW,EAAE;oBACpB,IAAI,CAAC,cAAc,EAAE,CAAC;oBACtB,OAAO;iBACR;gBAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjD,IAAI,CAAC,eAAe,EAAE,CAAC;gBAEvB,4BAA4B;gBAC5B,gCAAgC;gBAChC,WAAW;gBACX,2BAA2B;gBAC3B,IAAI;gBAEJ,iCAAiC;gBACjC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpD,8DAA8D;aAC/D;YAAC,OAAO,KAAU,EAAE;gBACnB,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE;oBACnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;oBACxB,IAAI,CAAC,cAAc,EAAE,CAAC;oBAEtB,iEAAiE;oBACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;oBAE3C,mCAAmC;iBACpC;qBAAM;oBACL,8CAA8C;oBAC9C,uDAAuD;oBACvD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACzB,IAAI,CAAC,eAAe,EAAE,CAAC;oBAEvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBAExB,mEAAmE;oBACnE,OAAO;iBACR;aACF;YAED,EAAE;YACF,4CAA4C;YAC5C,EAAE;YAEF,iEAAiE;YACjE,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;gBAC5E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,OAAO;aACR;YAED,qBAAqB;YACrB,wEAAwE;YACxE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;CACF;AA5WD,wCA4WC"}
|