@shutterstock/p-map-iterable 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +21 -0
- package/README.md +110 -0
- package/dist/blocking-queue.d.ts +57 -0
- package/dist/blocking-queue.d.ts.map +1 -0
- package/dist/blocking-queue.js +193 -0
- package/dist/blocking-queue.js.map +1 -0
- package/dist/blocking-queue.test.d.ts +2 -0
- package/dist/blocking-queue.test.d.ts.map +1 -0
- package/dist/blocking-queue.test.js +175 -0
- package/dist/blocking-queue.test.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/iterable-mapper.d.ts +104 -0
- package/dist/iterable-mapper.d.ts.map +1 -0
- package/dist/iterable-mapper.js +321 -0
- package/dist/iterable-mapper.js.map +1 -0
- package/dist/iterable-mapper.test.d.ts +2 -0
- package/dist/iterable-mapper.test.d.ts.map +1 -0
- package/dist/iterable-mapper.test.js +1421 -0
- package/dist/iterable-mapper.test.js.map +1 -0
- package/dist/iterable-queue-mapper-simple.d.ts +49 -0
- package/dist/iterable-queue-mapper-simple.d.ts.map +1 -0
- package/dist/iterable-queue-mapper-simple.js +92 -0
- package/dist/iterable-queue-mapper-simple.js.map +1 -0
- package/dist/iterable-queue-mapper-simple.test.d.ts +2 -0
- package/dist/iterable-queue-mapper-simple.test.d.ts.map +1 -0
- package/dist/iterable-queue-mapper-simple.test.js +103 -0
- package/dist/iterable-queue-mapper-simple.test.js.map +1 -0
- package/dist/iterable-queue-mapper.d.ts +73 -0
- package/dist/iterable-queue-mapper.d.ts.map +1 -0
- package/dist/iterable-queue-mapper.js +72 -0
- package/dist/iterable-queue-mapper.js.map +1 -0
- package/dist/iterable-queue-mapper.test.d.ts +2 -0
- package/dist/iterable-queue-mapper.test.d.ts.map +1 -0
- package/dist/iterable-queue-mapper.test.js +174 -0
- package/dist/iterable-queue-mapper.test.js.map +1 -0
- package/dist/iterable-queue.d.ts +26 -0
- package/dist/iterable-queue.d.ts.map +1 -0
- package/dist/iterable-queue.js +34 -0
- package/dist/iterable-queue.js.map +1 -0
- package/dist/iterable-queue.test.d.ts +2 -0
- package/dist/iterable-queue.test.d.ts.map +1 -0
- package/dist/iterable-queue.test.js +252 -0
- package/dist/iterable-queue.test.js.map +1 -0
- package/dist/queue.d.ts +9 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +33 -0
- package/dist/queue.js.map +1 -0
- package/dist/queue.test.d.ts +2 -0
- package/dist/queue.test.d.ts.map +1 -0
- package/dist/queue.test.js +75 -0
- package/dist/queue.test.js.map +1 -0
- package/package.json +45 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
interface IterableMapperOptions {
|
|
2
|
+
/**
|
|
3
|
+
Number of concurrently pending promises returned by `mapper`.
|
|
4
|
+
|
|
5
|
+
Must be an integer from 1 and up or `Infinity`, must be <= `maxUnread`.
|
|
6
|
+
|
|
7
|
+
@default 4
|
|
8
|
+
*/
|
|
9
|
+
readonly concurrency?: number;
|
|
10
|
+
/**
|
|
11
|
+
Number of pending unread iterable items.
|
|
12
|
+
|
|
13
|
+
Must be an integer from 1 and up or `Infinity`, must be >= `concurrency`.
|
|
14
|
+
|
|
15
|
+
@default 8
|
|
16
|
+
*/
|
|
17
|
+
readonly maxUnread?: number;
|
|
18
|
+
/**
|
|
19
|
+
When set to `false`, instead of stopping when a promise rejects, it will wait for all the promises to settle and then reject with an [aggregated error](https://github.com/sindresorhus/aggregate-error) containing all the errors from the rejected promises.
|
|
20
|
+
|
|
21
|
+
@default true
|
|
22
|
+
*/
|
|
23
|
+
readonly stopOnMapperError?: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
Function which is called for every item in `input`. Expected to return a `Promise` or value.
|
|
27
|
+
|
|
28
|
+
@param element - Iterated element.
|
|
29
|
+
@param index - Index of the element in the source array.
|
|
30
|
+
*/
|
|
31
|
+
export type Mapper<Element = unknown, NewElement = unknown> = (element: Element, index: number) => NewElement | Promise<NewElement>;
|
|
32
|
+
export declare class IterableMapper<Element, NewElement> implements AsyncIterable<NewElement> {
|
|
33
|
+
private _mapper;
|
|
34
|
+
private _options;
|
|
35
|
+
private _unreadQueue;
|
|
36
|
+
private _iterator;
|
|
37
|
+
private readonly _errors;
|
|
38
|
+
private _asyncIterator;
|
|
39
|
+
private _isRejected;
|
|
40
|
+
private _isIterableDone;
|
|
41
|
+
private _activeRunners;
|
|
42
|
+
private _resolvingCount;
|
|
43
|
+
private _currentIndex;
|
|
44
|
+
private _initialRunnersCreated;
|
|
45
|
+
/**
|
|
46
|
+
* Iterates over a source iterable with specified concurrency,
|
|
47
|
+
* calling the `mapper` on each iterated item, and storing the
|
|
48
|
+
* `mapper` result in a queue of specified max size, before
|
|
49
|
+
* being iterated / read by the caller.
|
|
50
|
+
*
|
|
51
|
+
* Essentially - This allows performing a concurrent mapping with
|
|
52
|
+
* back pressure (won't iterate all source items if the consumer is
|
|
53
|
+
* not reading).
|
|
54
|
+
*
|
|
55
|
+
* Typical use case is for a `prefetcher` that ensures that items
|
|
56
|
+
* are always ready for the consumer but that large numbers of items
|
|
57
|
+
* are not processed before the consumer is ready for them.
|
|
58
|
+
*
|
|
59
|
+
* @param input - Iterated over concurrently in the `mapper` function.
|
|
60
|
+
* @param mapper - Function which is called for every item in `input`. Expected to return a `Promise` or value.
|
|
61
|
+
* @param options - { concurrency: max mappers to run in parallel,
|
|
62
|
+
* maxUnread: max unread mapped results - this waits for results to be read,
|
|
63
|
+
* stopOnError: return if a mapper throws - note that any other running mappers will then not be able to return results
|
|
64
|
+
* }
|
|
65
|
+
* @returns A `Promise` that is fulfilled when all promises in `input`
|
|
66
|
+
* and ones returned from `mapper` are fulfilled, or rejects if any of the
|
|
67
|
+
* promises reject. The fulfilled value is an `Array` of the fulfilled
|
|
68
|
+
* values returned from `mapper` in `input` order.
|
|
69
|
+
*/
|
|
70
|
+
constructor(iterable: AsyncIterable<Element> | Iterable<Element>, mapper: Mapper<Element, NewElement>, { concurrency, stopOnMapperError, maxUnread }?: IterableMapperOptions);
|
|
71
|
+
[Symbol.asyncIterator](): AsyncIterator<NewElement>;
|
|
72
|
+
/**
|
|
73
|
+
* Used by the iterator returned from [Symbol.asyncIterator]
|
|
74
|
+
* Called every time an item is needed
|
|
75
|
+
* @returns Iterator result
|
|
76
|
+
*/
|
|
77
|
+
next(): Promise<IteratorResult<NewElement>>;
|
|
78
|
+
private bubbleUpErrors;
|
|
79
|
+
private startARunnerIfNeeded;
|
|
80
|
+
private startAnotherRunner;
|
|
81
|
+
private areWeDone;
|
|
82
|
+
/**
|
|
83
|
+
* Throw an exception if the wrapped NewElement is an Error
|
|
84
|
+
* @returns
|
|
85
|
+
*/
|
|
86
|
+
private throwIfError;
|
|
87
|
+
/**
|
|
88
|
+
* Get the next item from the source iterable.
|
|
89
|
+
*
|
|
90
|
+
* This is called up to `concurrency` times in parallel.
|
|
91
|
+
*
|
|
92
|
+
* If the read queue is not full, and there are source items to read,
|
|
93
|
+
* each instance of this will keep calling a new instance of itself
|
|
94
|
+
* that detaches and runs asynchronously (keeping the same number
|
|
95
|
+
* of instances running).
|
|
96
|
+
*
|
|
97
|
+
* If the read queue + runners = max read queue length then the runner
|
|
98
|
+
* will exit and will be restarted when an item is read from the queue.
|
|
99
|
+
* @returns
|
|
100
|
+
*/
|
|
101
|
+
private sourceNext;
|
|
102
|
+
}
|
|
103
|
+
export {};
|
|
104
|
+
//# sourceMappingURL=iterable-mapper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iterable-mapper.d.ts","sourceRoot":"","sources":["../src/iterable-mapper.ts"],"names":[],"mappings":"AAMA,UAAU,qBAAqB;IAC7B;;;;;;YAME;IACF,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;;YAME;IACF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;YAIE;IACF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;MAKG;AACH,MAAM,MAAM,MAAM,CAAC,OAAO,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO,IAAI,CAC5D,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,KACV,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAUtC,qBAAa,cAAc,CAAC,OAAO,EAAE,UAAU,CAAE,YAAW,aAAa,CAAC,UAAU,CAAC;IACnF,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,QAAQ,CAAkC;IAElD,OAAO,CAAC,YAAY,CAA+C;IAEnE,OAAO,CAAC,SAAS,CAA6C;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,sBAAsB,CAAS;IAEvC;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;gBAED,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,EACpD,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,EACnC,EAAE,WAAe,EAAE,iBAAwB,EAAE,SAAa,EAAE,GAAE,qBAA0B;IAsFnF,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,UAAU,CAAC;IAI1D;;;;OAIG;IACU,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;IAyBxD,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,oBAAoB;IAuB5B,OAAO,CAAC,kBAAkB;IAwB1B,OAAO,CAAC,SAAS;IAiBjB;;;OAGG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;;;;;;;;;OAaG;YACW,UAAU;CAiHzB"}
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.IterableMapper = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
//
|
|
6
|
+
// 2021-08-25 - Initially based on: https://raw.githubusercontent.com/sindresorhus/p-map/main/index.js
|
|
7
|
+
//
|
|
8
|
+
const aggregate_error_1 = tslib_1.__importDefault(require("aggregate-error"));
|
|
9
|
+
const iterable_queue_1 = require("./iterable-queue");
|
|
10
|
+
class IterableMapper {
|
|
11
|
+
/**
|
|
12
|
+
* Iterates over a source iterable with specified concurrency,
|
|
13
|
+
* calling the `mapper` on each iterated item, and storing the
|
|
14
|
+
* `mapper` result in a queue of specified max size, before
|
|
15
|
+
* being iterated / read by the caller.
|
|
16
|
+
*
|
|
17
|
+
* Essentially - This allows performing a concurrent mapping with
|
|
18
|
+
* back pressure (won't iterate all source items if the consumer is
|
|
19
|
+
* not reading).
|
|
20
|
+
*
|
|
21
|
+
* Typical use case is for a `prefetcher` that ensures that items
|
|
22
|
+
* are always ready for the consumer but that large numbers of items
|
|
23
|
+
* are not processed before the consumer is ready for them.
|
|
24
|
+
*
|
|
25
|
+
* @param input - Iterated over concurrently in the `mapper` function.
|
|
26
|
+
* @param mapper - Function which is called for every item in `input`. Expected to return a `Promise` or value.
|
|
27
|
+
* @param options - { concurrency: max mappers to run in parallel,
|
|
28
|
+
* maxUnread: max unread mapped results - this waits for results to be read,
|
|
29
|
+
* stopOnError: return if a mapper throws - note that any other running mappers will then not be able to return results
|
|
30
|
+
* }
|
|
31
|
+
* @returns A `Promise` that is fulfilled when all promises in `input`
|
|
32
|
+
* and ones returned from `mapper` are fulfilled, or rejects if any of the
|
|
33
|
+
* promises reject. The fulfilled value is an `Array` of the fulfilled
|
|
34
|
+
* values returned from `mapper` in `input` order.
|
|
35
|
+
*/
|
|
36
|
+
constructor(iterable, mapper, { concurrency = 4, stopOnMapperError = true, maxUnread = 8 } = {}) {
|
|
37
|
+
this._errors = [];
|
|
38
|
+
this._asyncIterator = false;
|
|
39
|
+
this._isRejected = false;
|
|
40
|
+
this._isIterableDone = false;
|
|
41
|
+
this._activeRunners = 0;
|
|
42
|
+
this._resolvingCount = 0;
|
|
43
|
+
this._currentIndex = 0;
|
|
44
|
+
this._initialRunnersCreated = false;
|
|
45
|
+
this._mapper = mapper;
|
|
46
|
+
this._options = { concurrency, stopOnMapperError, maxUnread };
|
|
47
|
+
if (typeof mapper !== 'function') {
|
|
48
|
+
throw new TypeError('Mapper function is required');
|
|
49
|
+
}
|
|
50
|
+
// Avoid undefined errors on options
|
|
51
|
+
if (this._options.concurrency === undefined ||
|
|
52
|
+
this._options.stopOnMapperError === undefined ||
|
|
53
|
+
this._options.maxUnread === undefined) {
|
|
54
|
+
throw new TypeError('Options are malformed after init');
|
|
55
|
+
}
|
|
56
|
+
// Validate concurrency option
|
|
57
|
+
if (!((Number.isSafeInteger(this._options.concurrency) ||
|
|
58
|
+
this._options.concurrency === Number.POSITIVE_INFINITY) &&
|
|
59
|
+
this._options.concurrency >= 1)) {
|
|
60
|
+
throw new TypeError(`Expected \`concurrency\` to be an integer from 1 and up or \`Infinity\`, got \`${concurrency}\` (${typeof concurrency})`);
|
|
61
|
+
}
|
|
62
|
+
// Validate maxUnread option
|
|
63
|
+
if (!((Number.isSafeInteger(this._options.maxUnread) ||
|
|
64
|
+
this._options.maxUnread === Number.POSITIVE_INFINITY) &&
|
|
65
|
+
this._options.maxUnread >= 1)) {
|
|
66
|
+
throw new TypeError(`Expected \`maxUnread\` to be an integer from 1 and up or \`Infinity\`, got \`${maxUnread}\` (${typeof maxUnread})`);
|
|
67
|
+
}
|
|
68
|
+
// Validate relationship between maxUnread and concurrency
|
|
69
|
+
if (this._options.maxUnread < this._options.concurrency) {
|
|
70
|
+
throw new TypeError(`Expected \`maxUnread\` to be greater than or equal to \`concurrency\`, got \`${maxUnread}\` < \`${concurrency}\``);
|
|
71
|
+
}
|
|
72
|
+
this._unreadQueue = new iterable_queue_1.IterableQueue({ maxUnread });
|
|
73
|
+
// Setup the source iterator
|
|
74
|
+
if (iterable[Symbol.asyncIterator] !== undefined) {
|
|
75
|
+
// We've got an async iterable
|
|
76
|
+
this._iterator = iterable[Symbol.asyncIterator]();
|
|
77
|
+
this._asyncIterator = true;
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
this._iterator = iterable[Symbol.iterator]();
|
|
81
|
+
}
|
|
82
|
+
// Create the initial concurrent runners in a detached (non-awaited)
|
|
83
|
+
// promise. We need this so we can await the next() calls
|
|
84
|
+
// to stop creating runners before hitting the concurrency limit
|
|
85
|
+
// if the iterable has already been marked as done.
|
|
86
|
+
void (() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
87
|
+
for (let index = 0; index < concurrency; index++) {
|
|
88
|
+
// Setup the detached runner
|
|
89
|
+
this._activeRunners++;
|
|
90
|
+
// This only waits for the next source item to be iterated
|
|
91
|
+
// It does NOT wait for the mapper to be called for for a consumer to pickup
|
|
92
|
+
// the result out of the unread queue.
|
|
93
|
+
yield this.sourceNext();
|
|
94
|
+
if (this._isIterableDone || this._isRejected) {
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
// Signal that the next() function should now create runners if it sees too few of them
|
|
99
|
+
this._initialRunnersCreated = true;
|
|
100
|
+
}))();
|
|
101
|
+
}
|
|
102
|
+
[Symbol.asyncIterator]() {
|
|
103
|
+
return this;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Used by the iterator returned from [Symbol.asyncIterator]
|
|
107
|
+
* Called every time an item is needed
|
|
108
|
+
* @returns Iterator result
|
|
109
|
+
*/
|
|
110
|
+
next() {
|
|
111
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
112
|
+
// Bail out and release all waiters if there are no more items coming
|
|
113
|
+
const done = this.areWeDone();
|
|
114
|
+
if (done) {
|
|
115
|
+
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
116
|
+
// throw the errors as an aggregate exception
|
|
117
|
+
this._isRejected = true;
|
|
118
|
+
throw new aggregate_error_1.default(this._errors);
|
|
119
|
+
}
|
|
120
|
+
return { value: undefined, done };
|
|
121
|
+
}
|
|
122
|
+
// Check if queue has an item
|
|
123
|
+
const item = yield this._unreadQueue.dequeue();
|
|
124
|
+
if (item === undefined) {
|
|
125
|
+
// We finished - There were no more items
|
|
126
|
+
this.bubbleUpErrors();
|
|
127
|
+
return { value: undefined, done: true };
|
|
128
|
+
}
|
|
129
|
+
this.startARunnerIfNeeded();
|
|
130
|
+
return { value: this.throwIfError(item), done: false };
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
bubbleUpErrors() {
|
|
134
|
+
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
135
|
+
// throw the errors as an aggregate exception
|
|
136
|
+
throw new aggregate_error_1.default(this._errors);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
startARunnerIfNeeded() {
|
|
140
|
+
// If there are items left AND there are not enough runners running,
|
|
141
|
+
// start one more runner - each subsequent read will check this and start more runners
|
|
142
|
+
// as items are pulled from the queue
|
|
143
|
+
if (this._initialRunnersCreated) {
|
|
144
|
+
// The init loop has finished - we don't create runners until that loop
|
|
145
|
+
// has finished else we'll end up with too many runners
|
|
146
|
+
if (!this._isIterableDone && !this._isRejected) {
|
|
147
|
+
// We only create more runners if the source iterable is not already done
|
|
148
|
+
if (this._activeRunners < this._options.concurrency) {
|
|
149
|
+
// We only create runners if we're under the concurrency limit
|
|
150
|
+
if (this._unreadQueue.length + this._activeRunners <= this._options.maxUnread) {
|
|
151
|
+
// We only create runners if the number of runners + unread items will not
|
|
152
|
+
// exceed the unread queue length
|
|
153
|
+
// Start another source runner, but do not await it
|
|
154
|
+
this.startAnotherRunner();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
startAnotherRunner() {
|
|
161
|
+
if (this._activeRunners === this._options.concurrency) {
|
|
162
|
+
throw new TypeError('active runners would be greater than concurrency limit');
|
|
163
|
+
}
|
|
164
|
+
if (this._activeRunners + this._unreadQueue.length > this._options.maxUnread) {
|
|
165
|
+
throw new TypeError('active runners would overflow the read queue limit');
|
|
166
|
+
}
|
|
167
|
+
if (this._isIterableDone) {
|
|
168
|
+
throw new TypeError('runner should not be started when iterable is already done');
|
|
169
|
+
}
|
|
170
|
+
if (this._activeRunners < 0) {
|
|
171
|
+
throw new TypeError('active runners is less than 0');
|
|
172
|
+
}
|
|
173
|
+
// We only create runners if the number of runners + unread items will not
|
|
174
|
+
// exceed the unread queue length
|
|
175
|
+
this._activeRunners++;
|
|
176
|
+
// Start another source runner, but do not await it
|
|
177
|
+
void this.sourceNext();
|
|
178
|
+
}
|
|
179
|
+
areWeDone() {
|
|
180
|
+
if (this._isIterableDone) {
|
|
181
|
+
// The source iterable has no more items
|
|
182
|
+
if (this._resolvingCount === 0) {
|
|
183
|
+
// There are no more resolvers running
|
|
184
|
+
if (this._unreadQueue.length === 0) {
|
|
185
|
+
// There are no unread items left
|
|
186
|
+
this._unreadQueue.done();
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Throw an exception if the wrapped NewElement is an Error
|
|
195
|
+
* @returns
|
|
196
|
+
*/
|
|
197
|
+
throwIfError(item) {
|
|
198
|
+
if (item.error !== undefined) {
|
|
199
|
+
throw item.error;
|
|
200
|
+
}
|
|
201
|
+
else if (item.element === undefined) {
|
|
202
|
+
throw new TypeError('no element was returned');
|
|
203
|
+
}
|
|
204
|
+
return item.element;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Get the next item from the source iterable.
|
|
208
|
+
*
|
|
209
|
+
* This is called up to `concurrency` times in parallel.
|
|
210
|
+
*
|
|
211
|
+
* If the read queue is not full, and there are source items to read,
|
|
212
|
+
* each instance of this will keep calling a new instance of itself
|
|
213
|
+
* that detaches and runs asynchronously (keeping the same number
|
|
214
|
+
* of instances running).
|
|
215
|
+
*
|
|
216
|
+
* If the read queue + runners = max read queue length then the runner
|
|
217
|
+
* will exit and will be restarted when an item is read from the queue.
|
|
218
|
+
* @returns
|
|
219
|
+
*/
|
|
220
|
+
sourceNext() {
|
|
221
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
222
|
+
if (this._isRejected) {
|
|
223
|
+
this._activeRunners--;
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
// Note: do NOT await a non-async iterable as it will cause next() to be
|
|
227
|
+
// pushed into the event loop, slowing down iteration of non-async iterables.
|
|
228
|
+
let nextItem;
|
|
229
|
+
try {
|
|
230
|
+
if (this._asyncIterator) {
|
|
231
|
+
nextItem = yield this._iterator.next();
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
nextItem = this._iterator.next();
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
// Iterator protocol / Iterables can throw exceptions - If this happens we have to just stop
|
|
239
|
+
// regardless of stopOnMapperError since we can't iterate any additional items
|
|
240
|
+
this._isRejected = true;
|
|
241
|
+
this._activeRunners--;
|
|
242
|
+
// Push the error onto the unread queue, to be rethrown by next()
|
|
243
|
+
yield this._unreadQueue.enqueue({ error });
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
const index = this._currentIndex;
|
|
247
|
+
this._currentIndex++;
|
|
248
|
+
if (nextItem.done) {
|
|
249
|
+
this._isIterableDone = true;
|
|
250
|
+
// If there are no active resolvers, then release all the waiters
|
|
251
|
+
if (this._resolvingCount === 0) {
|
|
252
|
+
// At this point the only waiters in the queue are not going to get an item
|
|
253
|
+
// as there are no source items left
|
|
254
|
+
this._unreadQueue.done();
|
|
255
|
+
}
|
|
256
|
+
this._activeRunners--;
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this._resolvingCount++;
|
|
260
|
+
// This is created as a detached, non-awaited async
|
|
261
|
+
// to allow next() to return while the async mapper is awaited.
|
|
262
|
+
// More next() calls will be made up to the concurrency limit.
|
|
263
|
+
void (() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
264
|
+
//
|
|
265
|
+
// Push an item or error into the read queue
|
|
266
|
+
// Note: once we push an item we end this try/catch as any subsequent errors
|
|
267
|
+
// are errors in this class and not errors thrown by the mapper function.
|
|
268
|
+
// Once we've pushed an item we can't also push an error...
|
|
269
|
+
//
|
|
270
|
+
try {
|
|
271
|
+
const element = nextItem.value;
|
|
272
|
+
if (this._isRejected) {
|
|
273
|
+
this._activeRunners--;
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const value = yield this._mapper(element, index);
|
|
277
|
+
this._resolvingCount--;
|
|
278
|
+
// if (value === pMapSkip) {
|
|
279
|
+
// skippedIndexes.push(index);
|
|
280
|
+
// } else {
|
|
281
|
+
// result[index] = value;
|
|
282
|
+
// }
|
|
283
|
+
// Push item onto the ready queue
|
|
284
|
+
yield this._unreadQueue.enqueue({ element: value });
|
|
285
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
286
|
+
}
|
|
287
|
+
catch (error) {
|
|
288
|
+
if (this._options.stopOnMapperError) {
|
|
289
|
+
this._isRejected = true;
|
|
290
|
+
this._activeRunners--;
|
|
291
|
+
// Push the error onto the unread queue, to be rethrown by next()
|
|
292
|
+
yield this._unreadQueue.enqueue({ error });
|
|
293
|
+
// Fall through to release a reader
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
// Collect the error but do not stop iterating
|
|
297
|
+
// These will be thrown in an AggregateError at the end
|
|
298
|
+
this._errors.push(error);
|
|
299
|
+
this._resolvingCount--;
|
|
300
|
+
yield this.sourceNext();
|
|
301
|
+
// Return so we don't release a reader since we didn't push an item
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
//
|
|
306
|
+
// Tasks below are not related to the mapper
|
|
307
|
+
//
|
|
308
|
+
// Bail if read queue length + active runners will hit max unread
|
|
309
|
+
if (this._unreadQueue.length + this._activeRunners > this._options.maxUnread) {
|
|
310
|
+
this._activeRunners--;
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
// Start myself again
|
|
314
|
+
// Note: this will bail out if it reaches the end of the source iterable
|
|
315
|
+
yield this.sourceNext();
|
|
316
|
+
}))();
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
exports.IterableMapper = IterableMapper;
|
|
321
|
+
//# sourceMappingURL=iterable-mapper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iterable-mapper.js","sourceRoot":"","sources":["../src/iterable-mapper.ts"],"names":[],"mappings":";;;;AAAA,EAAE;AACF,sGAAsG;AACtG,EAAE;AACF,8EAA6C;AAC7C,qDAAiD;AAgDjD,MAAa,cAAc;IAgBzB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YACE,QAAoD,EACpD,MAAmC,EACnC,EAAE,WAAW,GAAG,CAAC,EAAE,iBAAiB,GAAG,IAAI,EAAE,SAAS,GAAG,CAAC,KAA4B,EAAE;QArCzE,YAAO,GAAG,EAAa,CAAC;QACjC,mBAAc,GAAG,KAAK,CAAC;QACvB,gBAAW,GAAG,KAAK,CAAC;QACpB,oBAAe,GAAG,KAAK,CAAC;QACxB,mBAAc,GAAG,CAAC,CAAC;QACnB,oBAAe,GAAG,CAAC,CAAC;QACpB,kBAAa,GAAG,CAAC,CAAC;QAClB,2BAAsB,GAAG,KAAK,CAAC;QAgCrC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,WAAW,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;QAE9D,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE;YAChC,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;SACpD;QAED,oCAAoC;QACpC,IACE,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS;YACvC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,KAAK,SAAS;YAC7C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EACrC;YACA,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;SACzD;QAED,8BAA8B;QAC9B,IACE,CAAC,CACC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACzD,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,CAC/B,EACD;YACA,MAAM,IAAI,SAAS,CACjB,kFAAkF,WAAW,OAAO,OAAO,WAAW,GAAG,CAC1H,CAAC;SACH;QAED,4BAA4B;QAC5B,IACE,CAAC,CACC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAC7B,EACD;YACA,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,OAAO,OAAO,SAAS,GAAG,CACpH,CAAC;SACH;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACvD,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,UAAU,WAAW,IAAI,CACnH,CAAC;SACH;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,8BAAa,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAErD,4BAA4B;QAC5B,IAAK,QAAmC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,SAAS,EAAE;YAC5E,8BAA8B;YAC9B,IAAI,CAAC,SAAS,GAAI,QAAmC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC9E,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;SAC5B;aAAM;YACL,IAAI,CAAC,SAAS,GAAI,QAA8B,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;SACrE;QAED,oEAAoE;QACpE,0DAA0D;QAC1D,gEAAgE;QAChE,mDAAmD;QACnD,KAAK,CAAC,GAAS,EAAE;YACf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,EAAE,EAAE;gBAChD,4BAA4B;gBAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,0DAA0D;gBAC1D,4EAA4E;gBAC5E,sCAAsC;gBACtC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAExB,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,EAAE;oBAC5C,MAAM;iBACP;aACF;YAED,uFAAuF;YACvF,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACrC,CAAC,CAAA,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACU,IAAI;;YACf,qEAAqE;YACrE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,IAAI,IAAI,EAAE;gBACR,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC/D,6CAA6C;oBAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;oBACxB,MAAM,IAAI,yBAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACxC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aACnC;YAED,6BAA6B;YAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAC/C,IAAI,IAAI,KAAK,SAAS,EAAE;gBACtB,yCAAyC;gBACzC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACzC;YAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACzD,CAAC;KAAA;IAEO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YAC/D,6CAA6C;YAC7C,MAAM,IAAI,yBAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACxC;IACH,CAAC;IAEO,oBAAoB;QAC1B,oEAAoE;QACpE,sFAAsF;QACtF,qCAAqC;QACrC,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC/B,uEAAuE;YACvE,uDAAuD;YACvD,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBAC9C,yEAAyE;gBACzE,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;oBACnD,8DAA8D;oBAC9D,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;wBAC7E,0EAA0E;wBAC1E,iCAAiC;wBAEjC,mDAAmD;wBACnD,IAAI,CAAC,kBAAkB,EAAE,CAAC;qBAC3B;iBACF;aACF;SACF;IACH,CAAC;IAEO,kBAAkB;QACxB,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACrD,MAAM,IAAI,SAAS,CAAC,wDAAwD,CAAC,CAAC;SAC/E;QAED,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC5E,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;SAC3E;QAED,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;SACnF;QAED,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YAC3B,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;SACtD;QAED,0EAA0E;QAC1E,iCAAiC;QACjC,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,mDAAmD;QACnD,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;IACzB,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,wCAAwC;YACxC,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE;gBAC9B,sCAAsC;gBACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,iCAAiC;oBACjC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;oBAEzB,OAAO,IAAI,CAAC;iBACb;aACF;SACF;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,IAAmC;QACtD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC;SAClB;aAAM,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE;YACrC,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;SAChD;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACW,UAAU;;YACtB,IAAI,IAAI,CAAC,WAAW,EAAE;gBACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,OAAO;aACR;YAED,wEAAwE;YACxE,6EAA6E;YAC7E,IAAI,QAAiC,CAAC;YACtC,IAAI;gBACF,IAAI,IAAI,CAAC,cAAc,EAAE;oBACvB,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;iBACxC;qBAAM;oBACL,QAAQ,GAAI,IAAI,CAAC,SAA+B,CAAC,IAAI,EAAE,CAAC;iBACzD;aACF;YAAC,OAAO,KAAK,EAAE;gBACd,4FAA4F;gBAC5F,8EAA8E;gBAC9E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,iEAAiE;gBACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE3C,OAAO;aACR;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;YACjC,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACjB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAE5B,iEAAiE;gBACjE,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE;oBAC9B,2EAA2E;oBAC3E,oCAAoC;oBACpC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;iBAC1B;gBAED,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,OAAO;aACR;YAED,IAAI,CAAC,eAAe,EAAE,CAAC;YAEvB,mDAAmD;YACnD,+DAA+D;YAC/D,8DAA8D;YAC9D,KAAK,CAAC,GAAS,EAAE;gBACf,EAAE;gBACF,4CAA4C;gBAC5C,4EAA4E;gBAC5E,yEAAyE;gBACzE,2DAA2D;gBAC3D,EAAE;gBACF,IAAI;oBACF,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC;oBAE/B,IAAI,IAAI,CAAC,WAAW,EAAE;wBACpB,IAAI,CAAC,cAAc,EAAE,CAAC;wBACtB,OAAO;qBACR;oBAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACjD,IAAI,CAAC,eAAe,EAAE,CAAC;oBAEvB,4BAA4B;oBAC5B,gCAAgC;oBAChC,WAAW;oBACX,2BAA2B;oBAC3B,IAAI;oBAEJ,iCAAiC;oBACjC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;oBACpD,8DAA8D;iBAC/D;gBAAC,OAAO,KAAU,EAAE;oBACnB,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE;wBACnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,cAAc,EAAE,CAAC;wBAEtB,iEAAiE;wBACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;wBAE3C,mCAAmC;qBACpC;yBAAM;wBACL,8CAA8C;wBAC9C,uDAAuD;wBACvD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACzB,IAAI,CAAC,eAAe,EAAE,CAAC;wBAEvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;wBAExB,mEAAmE;wBACnE,OAAO;qBACR;iBACF;gBAED,EAAE;gBACF,4CAA4C;gBAC5C,EAAE;gBAEF,iEAAiE;gBACjE,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE;oBAC5E,IAAI,CAAC,cAAc,EAAE,CAAC;oBACtB,OAAO;iBACR;gBAED,qBAAqB;gBACrB,wEAAwE;gBACxE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YAC1B,CAAC,CAAA,CAAC,EAAE,CAAC;QACP,CAAC;KAAA;CACF;AAvXD,wCAuXC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iterable-mapper.test.d.ts","sourceRoot":"","sources":["../src/iterable-mapper.test.ts"],"names":[],"mappings":""}
|