@shutterstock/p-map-iterable 1.1.1 → 1.2.0-prerelease.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -17
- package/dist/src/blocking-queue.js +6 -20
- package/dist/src/blocking-queue.js.map +1 -1
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.js +4 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/iterable-mapper.d.ts +18 -13
- package/dist/src/iterable-mapper.js +28 -45
- package/dist/src/iterable-mapper.js.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.d.ts +28 -21
- package/dist/src/iterable-queue-mapper-simple.js +29 -23
- package/dist/src/iterable-queue-mapper-simple.js.map +1 -1
- package/dist/src/iterable-queue-mapper.d.ts +18 -9
- package/dist/src/iterable-queue-mapper.js +17 -8
- package/dist/src/iterable-queue-mapper.js.map +1 -1
- package/dist/src/iterable-queue.js.map +1 -1
- package/dist/src/queue.js.map +1 -1
- package/package.json +25 -19
- package/dist/src/blocking-queue.d.ts.map +0 -1
- package/dist/src/blocking-queue.test.d.ts +0 -2
- package/dist/src/blocking-queue.test.d.ts.map +0 -1
- package/dist/src/blocking-queue.test.js +0 -174
- package/dist/src/blocking-queue.test.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/iterable-mapper.d.ts.map +0 -1
- package/dist/src/iterable-mapper.test.d.ts +0 -2
- package/dist/src/iterable-mapper.test.d.ts.map +0 -1
- package/dist/src/iterable-mapper.test.js +0 -898
- package/dist/src/iterable-mapper.test.js.map +0 -1
- package/dist/src/iterable-queue-mapper-simple.d.ts.map +0 -1
- package/dist/src/iterable-queue-mapper-simple.test.d.ts +0 -2
- package/dist/src/iterable-queue-mapper-simple.test.d.ts.map +0 -1
- package/dist/src/iterable-queue-mapper-simple.test.js +0 -102
- package/dist/src/iterable-queue-mapper-simple.test.js.map +0 -1
- package/dist/src/iterable-queue-mapper.d.ts.map +0 -1
- package/dist/src/iterable-queue-mapper.test.d.ts +0 -2
- package/dist/src/iterable-queue-mapper.test.d.ts.map +0 -1
- package/dist/src/iterable-queue-mapper.test.js +0 -119
- package/dist/src/iterable-queue-mapper.test.js.map +0 -1
- package/dist/src/iterable-queue.d.ts.map +0 -1
- package/dist/src/iterable-queue.test.d.ts +0 -2
- package/dist/src/iterable-queue.test.d.ts.map +0 -1
- package/dist/src/iterable-queue.test.js +0 -143
- package/dist/src/iterable-queue.test.js.map +0 -1
- package/dist/src/queue.d.ts.map +0 -1
- package/dist/src/queue.test.d.ts +0 -2
- package/dist/src/queue.test.d.ts.map +0 -1
- package/dist/src/queue.test.js +0 -75
- package/dist/src/queue.test.js.map +0 -1
package/README.md
CHANGED
|
@@ -4,9 +4,11 @@
|
|
|
4
4
|
|
|
5
5
|
`@shutterstock/p-map-iterable` provides several classes that allow processing results of `p-map`-style mapper functions by iterating the results as they are completed, with backpressure to limit the number of items that are processed ahead of the consumer.
|
|
6
6
|
|
|
7
|
+
Use these classes for concurrent metadata lookups, capability probes, background status checks, or read/write pipelines. The [optional aliases](#optional-names-for-concurrent-work) `ConcurrentMapper`, `MappingQueue`, and `WorkerQueue` describe the input and result contracts without tying the work to a particular I/O operation.
|
|
8
|
+
|
|
7
9
|
A common use case for `@shutterstock/p-map-iterable` is as a "prefetcher" that will fetch, for example, AWS S3 files in an AWS Lambda function. By prefetching large files the consumer is able to use 100% of the paid-for Lambda CPU time for the JS thread, rather than waiting idle while the next file is fetched. The backpressure (set by `maxUnread`) prevents the prefetcher from consuming unlimited memory or disk space by racing ahead of the consumer.
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
The caller supplies the operation to run for each input. Concurrency overlaps asynchronous work in the current JavaScript process; synchronous CPU work still uses the JavaScript event loop.
|
|
10
12
|
|
|
11
13
|
# Example Usage Scenarios
|
|
12
14
|
|
|
@@ -25,14 +27,14 @@ for (const sourceId of sourceIds) {
|
|
|
25
27
|
|
|
26
28
|
Each iteration takes 820ms total, but we waste time waiting for I/O. We could prefetch the next read (300ms) while processing (20ms) and writing (500ms), without changing the order of reads or writes.
|
|
27
29
|
|
|
28
|
-
##
|
|
30
|
+
## Prefetching with `IterableMapper` and Blocking Sequential Writes
|
|
29
31
|
|
|
30
|
-
`concurrency: 1` on the prefetcher preserves the order of the reads and
|
|
32
|
+
`concurrency: 1` on the prefetcher preserves the order of the reads, and writes remain sequential and blocking.
|
|
31
33
|
|
|
32
34
|
```typescript
|
|
33
35
|
const source = new SomeSource();
|
|
34
36
|
const sourceIds = [1, 2,... 1000];
|
|
35
|
-
//
|
|
37
|
+
// Prefetches serially within maxUnread and releases results in sequential order
|
|
36
38
|
const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
37
39
|
async (sourceId) => source.read(sourceId),
|
|
38
40
|
{ concurrency: 1, maxUnread: 10 }
|
|
@@ -46,7 +48,7 @@ for await (const item of sourcePrefetcher) { // may not block for fast source
|
|
|
46
48
|
|
|
47
49
|
This reduces iteration time to 520ms by overlapping reads with processing/writing.
|
|
48
50
|
|
|
49
|
-
##
|
|
51
|
+
## Prefetching with `IterableMapper` and Background Sequential Writes with `IterableQueueMapperSimple`
|
|
50
52
|
|
|
51
53
|
`concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
52
54
|
`concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
@@ -67,7 +69,7 @@ for await (const item of sourcePrefetcher) { // may not block for fast source
|
|
|
67
69
|
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
68
70
|
await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
69
71
|
}
|
|
70
|
-
//
|
|
72
|
+
// Close input permanently after the last enqueue and wait for all writes
|
|
71
73
|
await flusher.onIdle();
|
|
72
74
|
// Check for errors
|
|
73
75
|
if (flusher.errors.length > 0) {
|
|
@@ -80,7 +82,7 @@ by overlapping reads and writes with the CPU processing step.
|
|
|
80
82
|
In this contrived example, the loop time is reduced to 500ms - 20ms = 480ms.
|
|
81
83
|
In cases where the CPU usage time is higher, the impact can be greater.
|
|
82
84
|
|
|
83
|
-
##
|
|
85
|
+
## Prefetching with `IterableMapper` and Out of Order Background Writes with `IterableQueueMapperSimple`
|
|
84
86
|
|
|
85
87
|
For maximum throughput, allow out of order reads and writes with
|
|
86
88
|
`IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
@@ -102,7 +104,7 @@ for await (const item of sourcePrefetcher) { // typically will not block
|
|
|
102
104
|
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
103
105
|
await flusher.enqueue(outputItem); // typically will not block
|
|
104
106
|
}
|
|
105
|
-
//
|
|
107
|
+
// Close input permanently after the last enqueue and wait for all writes
|
|
106
108
|
await flusher.onIdle();
|
|
107
109
|
// Check for errors
|
|
108
110
|
if (flusher.errors.length > 0) {
|
|
@@ -152,6 +154,7 @@ These diagrams illustrate the differences in operation betweeen `p-map`, `p-queu
|
|
|
152
154
|
# Features
|
|
153
155
|
|
|
154
156
|
- [IterableMapper](https://tech.shutterstock.com/p-map-iterable/classes/IterableMapper.html)
|
|
157
|
+
- Also exported as `ConcurrentMapper`
|
|
155
158
|
- Interface and concept based on: [p-map](https://github.com/sindresorhus/p-map)
|
|
156
159
|
- Allows a sync or async iterable input
|
|
157
160
|
- User supplied sync or async mapper function
|
|
@@ -159,13 +162,15 @@ These diagrams illustrate the differences in operation betweeen `p-map`, `p-queu
|
|
|
159
162
|
- Allows a maximum queue depth of mapped items - if the consumer stops consuming, the queue will fill up, at which point the mapper will stop being invoked until an item is consumed from the queue
|
|
160
163
|
- This allows mapping with backpressure so that the mapper does not consume unlimited resources (e.g. memory, disk, network, event loop time) by racing ahead of the consumer
|
|
161
164
|
- [IterableQueueMapper](https://tech.shutterstock.com/p-map-iterable/classes/IterableQueueMapper.html)
|
|
165
|
+
- Also exported as `MappingQueue`
|
|
162
166
|
- Wraps `IterableMapper`
|
|
163
167
|
- Adds items to the queue via the `enqueue` method
|
|
164
168
|
- [IterableQueueMapperSimple](https://tech.shutterstock.com/p-map-iterable/classes/IterableQueueMapperSimple.html)
|
|
169
|
+
- Also exported as `WorkerQueue`
|
|
165
170
|
- Wraps `IterableQueueMapper`
|
|
166
171
|
- Discards results as they become available
|
|
167
172
|
- Exposes any accumulated errors through the `errors` property instead of throwing an `AggregateError`
|
|
168
|
-
-
|
|
173
|
+
- Consumes results internally and exposes no result iterator
|
|
169
174
|
|
|
170
175
|
## Lower Level Utilities
|
|
171
176
|
- [IterableQueue](https://tech.shutterstock.com/p-map-iterable/classes/IterableQueue.html)
|
|
@@ -184,30 +189,87 @@ See [p-map](https://github.com/sindresorhus/p-map) docs for a good start in unde
|
|
|
184
189
|
|
|
185
190
|
The key difference between `IterableMapper` and `pMap` are that `IterableMapper` does not return when the entire mapping is done, rather it exposes an iterable that the caller loops through. This enables results to be processed while the mapping is still happening, while optionally allowing for backpressure to slow or stop the mapping if the caller is not consuming items fast enough. Common use cases include `prefetching` items from a remote service - the next set of requests are dispatched asyncronously while the current responses are processed and the prefetch requests will pause when the unread queue fills up.
|
|
186
191
|
|
|
192
|
+
`ConcurrentMapper` is an alias for this class. It also fits general mapping work such as enriching project metadata or probing a list of services. Consume results with `for await`; with `concurrency` greater than one, results can arrive out of input order.
|
|
193
|
+
|
|
187
194
|
See [examples/iterable-mapper.ts](./examples/iterable-mapper.ts) for an example.
|
|
188
195
|
|
|
189
|
-
Run the example with `
|
|
196
|
+
Run the example with `pnpm run example:iterable-mapper`
|
|
190
197
|
|
|
191
198
|
# `IterableQueueMapper`
|
|
192
199
|
|
|
193
200
|
`IterableQueueMapper` is similar to `IterableMapper` but instead of taking an iterable input it instead adds data via the `enqueue` method which will block if `maxUnread` will be reached by the current number of `mapper`'s running in parallel.
|
|
194
201
|
|
|
202
|
+
`MappingQueue` is an alias for this class. Produce inputs and consume mapped results concurrently so the result buffer can drain. After the last awaited enqueue, call `done()` and finish consuming the iterator; `done()` closes input and does not wait for work to complete.
|
|
203
|
+
|
|
195
204
|
See [examples/iterable-queue-mapper.ts](./examples/iterable-queue-mapper.ts) for an example.
|
|
196
205
|
|
|
197
|
-
Run the example with `
|
|
206
|
+
Run the example with `pnpm run example:iterable-queue-mapper`
|
|
198
207
|
|
|
199
208
|
# `IterableQueueMapperSimple`
|
|
200
209
|
|
|
201
210
|
`IterableQueueMapperSimple` is similar to `IterableQueueMapper` but instead exposing the results as an iterable it discards the results as soon as they are ready and exposes any errors through the `errors` property.
|
|
202
211
|
|
|
212
|
+
`WorkerQueue` is an alias for this class. Supply one worker callback to process every enqueued input, such as a background Git status check. Await each enqueue for producer backpressure. After the last enqueue, await `onIdle()` to permanently close input and finish the accepted work, then inspect `errors`. An enqueue resolves when the input is accepted, rather than when its work completes.
|
|
213
|
+
|
|
203
214
|
See [examples/iterable-queue-mapper-simple.ts](./examples/iterable-queue-mapper-simple.ts) for an example.
|
|
204
215
|
|
|
205
|
-
Run the example with `
|
|
216
|
+
Run the example with `pnpm run example:iterable-queue-mapper-simple`
|
|
217
|
+
|
|
218
|
+
## Optional names for concurrent work
|
|
219
|
+
|
|
220
|
+
The package exports three optional class aliases. Choose the name that makes the input and result contracts clearest in your application:
|
|
221
|
+
|
|
222
|
+
| Alias | Original class | Input | Results and completion |
|
|
223
|
+
| --- | --- | --- | --- |
|
|
224
|
+
| `ConcurrentMapper` | `IterableMapper` | Sync or async iterable | Consume mapped results with `for await`. Mapping pauses when the unread result buffer fills. |
|
|
225
|
+
| `MappingQueue` | `IterableQueueMapper` | Awaited `enqueue(item)` calls | Consume results concurrently with production. Call `done()` after the last enqueue, then finish consuming the iterator. |
|
|
226
|
+
| `WorkerQueue` | `IterableQueueMapperSimple` | Awaited `enqueue(item)` calls | Results are consumed internally. After the last enqueue, await `onIdle()` and check `errors`. |
|
|
227
|
+
|
|
228
|
+
```typescript
|
|
229
|
+
import {
|
|
230
|
+
ConcurrentMapper,
|
|
231
|
+
MappingQueue,
|
|
232
|
+
WorkerQueue,
|
|
233
|
+
} from '@shutterstock/p-map-iterable';
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
The aliases are the original classes, with the same constructor identity, generic instance types, and subclassing behavior. Existing imports continue to work. Defaults, result ordering, backpressure, and error handling are identical through either name.
|
|
237
|
+
|
|
238
|
+
Matching option types are available from the package root using `import type`:
|
|
239
|
+
|
|
240
|
+
| Option alias | Original option type | Configuration |
|
|
241
|
+
| --- | --- | --- |
|
|
242
|
+
| `ConcurrentMapperOptions` | `IterableMapperOptions` | `concurrency`, `maxUnread`, `stopOnMapperError` |
|
|
243
|
+
| `MappingQueueOptions` | `IterableQueueMapperOptions` | `concurrency`, `maxUnread`, `stopOnMapperError` |
|
|
244
|
+
| `WorkerQueueOptions` | `IterableQueueMapperSimpleOptions` | `concurrency` |
|
|
245
|
+
|
|
246
|
+
`MappingQueue` must have a result consumer even when you do not need the results. Awaiting every enqueue before starting iteration can block once the result buffer fills. Use `WorkerQueue` when results can be discarded. Its `onIdle()` permanently ends input; subsequent enqueues reject. Worker failures are collected in `errors` while other inputs continue to run, so check that property after shutdown.
|
|
247
|
+
|
|
248
|
+
Each queue uses one fixed callback supplied at construction. Awaiting `enqueue()` provides producer backpressure and confirms acceptance of an input. Calling it without awaiting can accumulate pending inputs. For ongoing event-driven work, the application supplies any admission limits, cancellation, deduplication, or per-item completion handles. `WorkerQueue.onIdle()` is a final shutdown operation, so an application that needs reusable idle waits must manage that separately.
|
|
249
|
+
|
|
250
|
+
See [examples/semantic-aliases.ts](./examples/semantic-aliases.ts) for metadata enrichment, queued capability probes with a concurrent result consumer, and background status checks with collected errors. Run it with `pnpm run example:semantic-aliases`.
|
|
206
251
|
|
|
207
252
|
# Contributing - Setting up Build Environment
|
|
208
253
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
254
|
+
Use Node.js 24 and the pnpm version pinned in `package.json`:
|
|
255
|
+
|
|
256
|
+
```sh
|
|
257
|
+
nvm use
|
|
258
|
+
corepack enable pnpm
|
|
259
|
+
corepack prepare pnpm@12.7.0 --activate
|
|
260
|
+
pnpm install --frozen-lockfile
|
|
261
|
+
pnpm run build
|
|
262
|
+
pnpm run build:docs
|
|
263
|
+
pnpm run lint
|
|
264
|
+
pnpm run test
|
|
265
|
+
pnpm run example:semantic-aliases
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Corepack enforces the pnpm version in `package.json`. pnpm's additional package
|
|
269
|
+
manager switching is disabled to keep the single-document lockfile readable by
|
|
270
|
+
GitHub's dependency graph and Dependabot. The pnpm configuration requires package
|
|
271
|
+
releases to be at least seven days old.
|
|
272
|
+
On macOS, `packageImportMethod: auto` prefers APFS copy-on-write clones from the
|
|
273
|
+
shared pnpm store, so worktrees share package data until a file changes. Keep the
|
|
274
|
+
store on the same APFS volume as the checkout. Other supported filesystems use
|
|
275
|
+
pnpm's available import method.
|
|
@@ -36,10 +36,6 @@ class BlockingQueue {
|
|
|
36
36
|
this._doneAdding = false;
|
|
37
37
|
const { maxUnread = 8 } = options;
|
|
38
38
|
this._options = { maxUnread };
|
|
39
|
-
// Avoid undefined errors on options
|
|
40
|
-
if (this._options.maxUnread === undefined) {
|
|
41
|
-
throw new TypeError('`maxUnread` must be set');
|
|
42
|
-
}
|
|
43
39
|
// Validate maxUnread option
|
|
44
40
|
if (!((Number.isSafeInteger(this._options.maxUnread) ||
|
|
45
41
|
this._options.maxUnread === Number.POSITIVE_INFINITY) &&
|
|
@@ -77,14 +73,9 @@ class BlockingQueue {
|
|
|
77
73
|
}
|
|
78
74
|
// We always add to the internal queue, we just wait to return if the queue is full
|
|
79
75
|
this._unreadQueue.enqueue(item);
|
|
80
|
-
if
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const reader = this._readersWaiting.dequeue();
|
|
84
|
-
if (reader === undefined) {
|
|
85
|
-
throw new TypeError('reader queue returned undefined');
|
|
86
|
-
}
|
|
87
|
-
// Resolve the Promise for a waiting reader
|
|
76
|
+
// Take the oldest reader if one is waiting; an empty queue returns undefined.
|
|
77
|
+
const reader = this._readersWaiting.dequeue();
|
|
78
|
+
if (reader !== undefined) {
|
|
88
79
|
reader(undefined);
|
|
89
80
|
}
|
|
90
81
|
if (this._unreadQueue.length > this._options.maxUnread) {
|
|
@@ -115,14 +106,9 @@ class BlockingQueue {
|
|
|
115
106
|
await this.waitForWrite();
|
|
116
107
|
item = this._unreadQueue.dequeue();
|
|
117
108
|
}
|
|
118
|
-
if
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
const writer = this._writersWaiting.dequeue();
|
|
122
|
-
if (writer === undefined) {
|
|
123
|
-
throw new TypeError('writer queue returned undefined');
|
|
124
|
-
}
|
|
125
|
-
// Resolve the Promise for a waiting writer
|
|
109
|
+
// Take the oldest writer if one is waiting; an empty queue returns undefined.
|
|
110
|
+
const writer = this._writersWaiting.dequeue();
|
|
111
|
+
if (writer !== undefined) {
|
|
126
112
|
writer(undefined);
|
|
127
113
|
}
|
|
128
114
|
return item;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blocking-queue.js","sourceRoot":"","sources":["../../src/blocking-queue.ts"],"names":[],"mappings":";;;AAAA,mCAAgC;AAkBhC;;;;;;;GAOG;AACH,MAAa,aAAa;IAwBxB;;;;OAIG;IACH,YAAY,UAAgC,EAAE;QA1B9C,mFAAmF;QACnF,8EAA8E;QAC9E,yEAAyE;QACxD,iBAAY,GAAmB,IAAI,aAAK,EAAE,CAAC;QAE5D,8EAA8E;QAC9E,+EAA+E;QAC/E,gDAAgD;QAChD,2EAA2E;QAC3E,YAAY;QACK,oBAAe,GAA8B,IAAI,aAAK,EAAE,CAAC;QAE1E,6EAA6E;QAC7E,+EAA+E;QAC/E,gDAAgD;QAChD,2EAA2E;QAC3E,YAAY;QACK,oBAAe,GAA8B,IAAI,aAAK,EAAE,CAAC;QAElE,gBAAW,GAAG,KAAK,CAAC;QAQ1B,MAAM,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,EAAE,SAAS,EAAE,CAAC;QAE9B,
|
|
1
|
+
{"version":3,"file":"blocking-queue.js","sourceRoot":"","sources":["../../src/blocking-queue.ts"],"names":[],"mappings":";;;AAAA,mCAAgC;AAkBhC;;;;;;;GAOG;AACH,MAAa,aAAa;IAwBxB;;;;OAIG;IACH,YAAY,UAAgC,EAAE;QA1B9C,mFAAmF;QACnF,8EAA8E;QAC9E,yEAAyE;QACxD,iBAAY,GAAmB,IAAI,aAAK,EAAE,CAAC;QAE5D,8EAA8E;QAC9E,+EAA+E;QAC/E,gDAAgD;QAChD,2EAA2E;QAC3E,YAAY;QACK,oBAAe,GAA8B,IAAI,aAAK,EAAE,CAAC;QAE1E,6EAA6E;QAC7E,+EAA+E;QAC/E,gDAAgD;QAChD,2EAA2E;QAC3E,YAAY;QACK,oBAAe,GAA8B,IAAI,aAAK,EAAE,CAAC;QAElE,gBAAW,GAAG,KAAK,CAAC;QAQ1B,MAAM,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,EAAE,SAAS,EAAE,CAAC;QAE9B,4BAA4B;QAC5B,IAAI,CAAC,CACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAC7B,EAAE,CAAC;YACF,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,OAAO,OAAO,SAAS,GAAG,CACpH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAW,MAAM;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACI,IAAI;QACT,mCAAmC;QACnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,+EAA+E;QAC/E,+DAA+D;QAC/D,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO;QACT,CAAC;QAED,kEAAkE;QAClE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,IAAa;QAChC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,mFAAmF;QACnF,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAEhC,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,SAAS,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvD,wCAAwC;YACxC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO;QAClB,qEAAqE;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,6BAA6B;QAC7B,IAAI,IAAyB,CAAC;QAE9B,+BAA+B;QAC/B,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxE,2DAA2D;YAC3D,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,8FAA8F;YAC9F,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC;QAED,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,SAAS,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB;QACvB,mDAAmD;QACnD,IAAI,MAAsC,CAAC;QAC3C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YAC/D,MAAM,CAAC,SAAS,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,0CAA0C;YAC1C,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtC,gDAAgD;gBAChD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnC,iCAAiC;oBACjC,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAEzB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM;IACN,gDAAgD;IAChD,wCAAwC;IACxC,MAAM;IACN,gEAAgE;IAChE,4EAA4E;IAC5E,sFAAsF;IACtF,iDAAiD;IACjD,6CAA6C;IAC7C,QAAQ;IAER,qCAAqC;IACrC,qBAAqB;IAErB,8CAA8C;IAE9C,oCAAoC;IACpC,iBAAiB;IACjB,IAAI;IAEJ;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,uEAAuE;QACvE,iFAAiF;QACjF,MAAM,SAAS,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACxC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,gCAAgC;QAChC,MAAM,SAAS,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW;QACvB,uEAAuE;QACvE,iFAAiF;QACjF,MAAM,UAAU,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACzC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,UAAU,CAAC;IACnB,CAAC;CACF;AAxMD,sCAwMC"}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -2,7 +2,8 @@ import { Mapper, IterableMapper, IterableMapperOptions } from './iterable-mapper
|
|
|
2
2
|
import { BlockingQueue, BlockingQueueOptions } from './blocking-queue';
|
|
3
3
|
import { IterableQueue, IterableQueueOptions } from './iterable-queue';
|
|
4
4
|
import { IterableQueueMapper, IterableQueueMapperOptions } from './iterable-queue-mapper';
|
|
5
|
-
import { IterableQueueMapperSimple } from './iterable-queue-mapper-simple';
|
|
5
|
+
import { IterableQueueMapperSimple, IterableQueueMapperSimpleOptions } from './iterable-queue-mapper-simple';
|
|
6
6
|
import { Queue } from './queue';
|
|
7
|
-
export {
|
|
7
|
+
export { IterableMapper, IterableMapper as ConcurrentMapper, IterableQueueMapper, IterableQueueMapper as MappingQueue, IterableQueueMapperSimple, IterableQueueMapperSimple as WorkerQueue, BlockingQueue, IterableQueue, Queue, };
|
|
8
|
+
export type { Mapper, IterableMapperOptions, IterableMapperOptions as ConcurrentMapperOptions, IterableQueueMapperOptions, IterableQueueMapperOptions as MappingQueueOptions, IterableQueueMapperSimpleOptions, IterableQueueMapperSimpleOptions as WorkerQueueOptions, BlockingQueueOptions, IterableQueueOptions, };
|
|
8
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/src/index.js
CHANGED
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Queue = exports.IterableQueue = exports.BlockingQueue = exports.IterableQueueMapperSimple = exports.IterableQueueMapper = exports.IterableMapper = void 0;
|
|
3
|
+
exports.Queue = exports.IterableQueue = exports.BlockingQueue = exports.WorkerQueue = exports.IterableQueueMapperSimple = exports.MappingQueue = exports.IterableQueueMapper = exports.ConcurrentMapper = exports.IterableMapper = void 0;
|
|
4
4
|
const iterable_mapper_1 = require("./iterable-mapper");
|
|
5
5
|
Object.defineProperty(exports, "IterableMapper", { enumerable: true, get: function () { return iterable_mapper_1.IterableMapper; } });
|
|
6
|
+
Object.defineProperty(exports, "ConcurrentMapper", { enumerable: true, get: function () { return iterable_mapper_1.IterableMapper; } });
|
|
6
7
|
const blocking_queue_1 = require("./blocking-queue");
|
|
7
8
|
Object.defineProperty(exports, "BlockingQueue", { enumerable: true, get: function () { return blocking_queue_1.BlockingQueue; } });
|
|
8
9
|
const iterable_queue_1 = require("./iterable-queue");
|
|
9
10
|
Object.defineProperty(exports, "IterableQueue", { enumerable: true, get: function () { return iterable_queue_1.IterableQueue; } });
|
|
10
11
|
const iterable_queue_mapper_1 = require("./iterable-queue-mapper");
|
|
11
12
|
Object.defineProperty(exports, "IterableQueueMapper", { enumerable: true, get: function () { return iterable_queue_mapper_1.IterableQueueMapper; } });
|
|
13
|
+
Object.defineProperty(exports, "MappingQueue", { enumerable: true, get: function () { return iterable_queue_mapper_1.IterableQueueMapper; } });
|
|
12
14
|
const iterable_queue_mapper_simple_1 = require("./iterable-queue-mapper-simple");
|
|
13
15
|
Object.defineProperty(exports, "IterableQueueMapperSimple", { enumerable: true, get: function () { return iterable_queue_mapper_simple_1.IterableQueueMapperSimple; } });
|
|
16
|
+
Object.defineProperty(exports, "WorkerQueue", { enumerable: true, get: function () { return iterable_queue_mapper_simple_1.IterableQueueMapperSimple; } });
|
|
14
17
|
const queue_1 = require("./queue");
|
|
15
18
|
Object.defineProperty(exports, "Queue", { enumerable: true, get: function () { return queue_1.Queue; } });
|
|
16
19
|
//# sourceMappingURL=index.js.map
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,uDAAkF;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,uDAAkF;AAWhF,+FAXe,gCAAc,OAWf;AACI,iGAZH,gCAAc,OAYK;AAXpC,qDAAuE;AAgBrE,8FAhBO,8BAAa,OAgBP;AAff,qDAAuE;AAgBrE,8FAhBO,8BAAa,OAgBP;AAff,mEAA0F;AAUxF,oGAVO,2CAAmB,OAUP;AACI,6FAXhB,2CAAmB,OAWS;AAVrC,iFAGwC;AAQtC,0GAVA,wDAAyB,OAUA;AACI,4FAX7B,wDAAyB,OAWe;AAR1C,mCAAgC;AAW9B,sFAXO,aAAK,OAWP"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Options for IterableMapper
|
|
2
|
+
* Options for `IterableMapper`, also exported as `ConcurrentMapperOptions`.
|
|
3
3
|
*/
|
|
4
4
|
export interface IterableMapperOptions {
|
|
5
5
|
/**
|
|
@@ -56,7 +56,7 @@ export interface IterableMapperOptions {
|
|
|
56
56
|
* @template Element - Source element type
|
|
57
57
|
* @template NewElement - Element type returned by the mapper
|
|
58
58
|
* @param element - Iterated element
|
|
59
|
-
* @param index -
|
|
59
|
+
* @param index - Zero-based index of the input element
|
|
60
60
|
*/
|
|
61
61
|
export type Mapper<Element = unknown, NewElement = unknown> = (element: Element, index: number) => NewElement | Promise<NewElement>;
|
|
62
62
|
/**
|
|
@@ -65,9 +65,14 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
65
65
|
* `mapper` result in a queue of `maxUnread` size, before
|
|
66
66
|
* being iterated / read by the caller.
|
|
67
67
|
*
|
|
68
|
+
* Also exported as `ConcurrentMapper`, with the same constructor and instance types.
|
|
69
|
+
* Concurrency overlaps asynchronous callback work in the current JavaScript process.
|
|
70
|
+
* Results can arrive out of input order when `concurrency` is greater than one.
|
|
71
|
+
*
|
|
68
72
|
* @remarks
|
|
69
73
|
*
|
|
70
|
-
* ### Typical Use
|
|
74
|
+
* ### Typical Use Cases
|
|
75
|
+
* - Enriching metadata or probing capabilities for a sync or async iterable of inputs
|
|
71
76
|
* - Prefetching items from an async I/O source
|
|
72
77
|
* - In the simple sequential (`concurrency: 1`) case, allows items to be prefetched async, preserving order, while caller processes an item
|
|
73
78
|
* - Can allow parallel prefetches for sources that allow for out of order reads (`concurrency: 2+`)
|
|
@@ -88,7 +93,8 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
88
93
|
*
|
|
89
94
|
* ### Usage
|
|
90
95
|
* - Items are exposed to the `mapper` via an iterator or async iterator (this includes generator and async generator functions)
|
|
91
|
-
* -
|
|
96
|
+
* - Consume results with `for await` while mapping proceeds
|
|
97
|
+
* - The `mapper` will not be invoked when `maxUnread` is reached, until items are consumed
|
|
92
98
|
* - The iterable will set `done` when the `input` has indicated `done` and all `mapper` promises have resolved
|
|
93
99
|
*
|
|
94
100
|
* @example
|
|
@@ -112,14 +118,14 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
112
118
|
*
|
|
113
119
|
* @example
|
|
114
120
|
*
|
|
115
|
-
* ###
|
|
121
|
+
* ### Prefetching with `IterableMapper` and Blocking Sequential Writes
|
|
116
122
|
*
|
|
117
|
-
* `concurrency: 1` on the prefetcher preserves the order of the reads and
|
|
123
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads, and writes remain sequential and blocking.
|
|
118
124
|
*
|
|
119
125
|
* ```typescript
|
|
120
126
|
* const source = new SomeSource();
|
|
121
127
|
* const sourceIds = [1, 2,... 1000];
|
|
122
|
-
* //
|
|
128
|
+
* // Prefetches serially within maxUnread and releases results in sequential order
|
|
123
129
|
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
124
130
|
* async (sourceId) => source.read(sourceId),
|
|
125
131
|
* { concurrency: 1, maxUnread: 10 }
|
|
@@ -135,7 +141,7 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
135
141
|
*
|
|
136
142
|
* @example
|
|
137
143
|
*
|
|
138
|
-
* ###
|
|
144
|
+
* ### Prefetching with `IterableMapper` and Background Sequential Writes with `IterableQueueMapperSimple`
|
|
139
145
|
*
|
|
140
146
|
* `concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
141
147
|
* `concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
@@ -156,7 +162,7 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
156
162
|
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
157
163
|
* await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
158
164
|
* }
|
|
159
|
-
* //
|
|
165
|
+
* // Close input permanently after the last enqueue and wait for all writes
|
|
160
166
|
* await flusher.onIdle();
|
|
161
167
|
* // Check for errors
|
|
162
168
|
* if (flusher.errors.length > 0) {
|
|
@@ -171,7 +177,7 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
171
177
|
*
|
|
172
178
|
* @example
|
|
173
179
|
*
|
|
174
|
-
* ###
|
|
180
|
+
* ### Prefetching with `IterableMapper` and Out of Order Background Writes with `IterableQueueMapperSimple`
|
|
175
181
|
*
|
|
176
182
|
* For maximum throughput, allow out of order reads and writes with
|
|
177
183
|
* `IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
@@ -193,7 +199,7 @@ export type Mapper<Element = unknown, NewElement = unknown> = (element: Element,
|
|
|
193
199
|
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
194
200
|
* await flusher.enqueue(outputItem); // typically will not block
|
|
195
201
|
* }
|
|
196
|
-
* //
|
|
202
|
+
* // Close input permanently after the last enqueue and wait for all writes
|
|
197
203
|
* await flusher.onIdle();
|
|
198
204
|
* // Check for errors
|
|
199
205
|
* if (flusher.errors.length > 0) {
|
|
@@ -227,7 +233,7 @@ export declare class IterableMapper<Element, NewElement> implements AsyncIterabl
|
|
|
227
233
|
* @param mapper Function called for every item in `input`. Returns a `Promise` or value.
|
|
228
234
|
* @param options IterableMapper options
|
|
229
235
|
*
|
|
230
|
-
* @see {@link
|
|
236
|
+
* @see {@link IterableMapper} for full class documentation
|
|
231
237
|
*/
|
|
232
238
|
constructor(input: AsyncIterable<Element> | Iterable<Element>, mapper: Mapper<Element, NewElement>, options?: IterableMapperOptions);
|
|
233
239
|
[Symbol.asyncIterator](): AsyncIterator<NewElement>;
|
|
@@ -240,7 +246,6 @@ export declare class IterableMapper<Element, NewElement> implements AsyncIterabl
|
|
|
240
246
|
next(): Promise<IteratorResult<NewElement>>;
|
|
241
247
|
private bubbleUpErrors;
|
|
242
248
|
private startARunnerIfNeeded;
|
|
243
|
-
private startAnotherRunner;
|
|
244
249
|
private areWeDone;
|
|
245
250
|
/**
|
|
246
251
|
* Throw an exception if the wrapped NewElement is an Error
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.IterableMapper = void 0;
|
|
4
2
|
//
|
|
5
3
|
// 2021-08-25 - Initially based on: https://raw.githubusercontent.com/sindresorhus/p-map/main/index.js
|
|
6
4
|
//
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
6
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.IterableMapper = void 0;
|
|
10
|
+
const aggregate_error_1 = __importDefault(require("aggregate-error"));
|
|
9
11
|
const iterable_queue_1 = require("./iterable-queue");
|
|
10
12
|
/**
|
|
11
13
|
* Iterates over a source iterable / generator with specified `concurrency`,
|
|
@@ -13,9 +15,14 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
13
15
|
* `mapper` result in a queue of `maxUnread` size, before
|
|
14
16
|
* being iterated / read by the caller.
|
|
15
17
|
*
|
|
18
|
+
* Also exported as `ConcurrentMapper`, with the same constructor and instance types.
|
|
19
|
+
* Concurrency overlaps asynchronous callback work in the current JavaScript process.
|
|
20
|
+
* Results can arrive out of input order when `concurrency` is greater than one.
|
|
21
|
+
*
|
|
16
22
|
* @remarks
|
|
17
23
|
*
|
|
18
|
-
* ### Typical Use
|
|
24
|
+
* ### Typical Use Cases
|
|
25
|
+
* - Enriching metadata or probing capabilities for a sync or async iterable of inputs
|
|
19
26
|
* - Prefetching items from an async I/O source
|
|
20
27
|
* - In the simple sequential (`concurrency: 1`) case, allows items to be prefetched async, preserving order, while caller processes an item
|
|
21
28
|
* - Can allow parallel prefetches for sources that allow for out of order reads (`concurrency: 2+`)
|
|
@@ -36,7 +43,8 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
36
43
|
*
|
|
37
44
|
* ### Usage
|
|
38
45
|
* - Items are exposed to the `mapper` via an iterator or async iterator (this includes generator and async generator functions)
|
|
39
|
-
* -
|
|
46
|
+
* - Consume results with `for await` while mapping proceeds
|
|
47
|
+
* - The `mapper` will not be invoked when `maxUnread` is reached, until items are consumed
|
|
40
48
|
* - The iterable will set `done` when the `input` has indicated `done` and all `mapper` promises have resolved
|
|
41
49
|
*
|
|
42
50
|
* @example
|
|
@@ -60,14 +68,14 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
60
68
|
*
|
|
61
69
|
* @example
|
|
62
70
|
*
|
|
63
|
-
* ###
|
|
71
|
+
* ### Prefetching with `IterableMapper` and Blocking Sequential Writes
|
|
64
72
|
*
|
|
65
|
-
* `concurrency: 1` on the prefetcher preserves the order of the reads and
|
|
73
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads, and writes remain sequential and blocking.
|
|
66
74
|
*
|
|
67
75
|
* ```typescript
|
|
68
76
|
* const source = new SomeSource();
|
|
69
77
|
* const sourceIds = [1, 2,... 1000];
|
|
70
|
-
* //
|
|
78
|
+
* // Prefetches serially within maxUnread and releases results in sequential order
|
|
71
79
|
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
72
80
|
* async (sourceId) => source.read(sourceId),
|
|
73
81
|
* { concurrency: 1, maxUnread: 10 }
|
|
@@ -83,7 +91,7 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
83
91
|
*
|
|
84
92
|
* @example
|
|
85
93
|
*
|
|
86
|
-
* ###
|
|
94
|
+
* ### Prefetching with `IterableMapper` and Background Sequential Writes with `IterableQueueMapperSimple`
|
|
87
95
|
*
|
|
88
96
|
* `concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
89
97
|
* `concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
@@ -104,7 +112,7 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
104
112
|
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
105
113
|
* await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
106
114
|
* }
|
|
107
|
-
* //
|
|
115
|
+
* // Close input permanently after the last enqueue and wait for all writes
|
|
108
116
|
* await flusher.onIdle();
|
|
109
117
|
* // Check for errors
|
|
110
118
|
* if (flusher.errors.length > 0) {
|
|
@@ -119,7 +127,7 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
119
127
|
*
|
|
120
128
|
* @example
|
|
121
129
|
*
|
|
122
|
-
* ###
|
|
130
|
+
* ### Prefetching with `IterableMapper` and Out of Order Background Writes with `IterableQueueMapperSimple`
|
|
123
131
|
*
|
|
124
132
|
* For maximum throughput, allow out of order reads and writes with
|
|
125
133
|
* `IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
@@ -141,7 +149,7 @@ const iterable_queue_1 = require("./iterable-queue");
|
|
|
141
149
|
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
142
150
|
* await flusher.enqueue(outputItem); // typically will not block
|
|
143
151
|
* }
|
|
144
|
-
* //
|
|
152
|
+
* // Close input permanently after the last enqueue and wait for all writes
|
|
145
153
|
* await flusher.onIdle();
|
|
146
154
|
* // Check for errors
|
|
147
155
|
* if (flusher.errors.length > 0) {
|
|
@@ -163,7 +171,7 @@ class IterableMapper {
|
|
|
163
171
|
* @param mapper Function called for every item in `input`. Returns a `Promise` or value.
|
|
164
172
|
* @param options IterableMapper options
|
|
165
173
|
*
|
|
166
|
-
* @see {@link
|
|
174
|
+
* @see {@link IterableMapper} for full class documentation
|
|
167
175
|
*/
|
|
168
176
|
constructor(input, mapper, options = {}) {
|
|
169
177
|
this._errors = [];
|
|
@@ -180,12 +188,6 @@ class IterableMapper {
|
|
|
180
188
|
if (typeof mapper !== 'function') {
|
|
181
189
|
throw new TypeError('Mapper function is required');
|
|
182
190
|
}
|
|
183
|
-
// Avoid undefined errors on options
|
|
184
|
-
if (this._options.concurrency === undefined ||
|
|
185
|
-
this._options.stopOnMapperError === undefined ||
|
|
186
|
-
this._options.maxUnread === undefined) {
|
|
187
|
-
throw new TypeError('Options are malformed after init');
|
|
188
|
-
}
|
|
189
191
|
// Validate concurrency option
|
|
190
192
|
if (!((Number.isSafeInteger(this._options.concurrency) ||
|
|
191
193
|
this._options.concurrency === Number.POSITIVE_INFINITY) &&
|
|
@@ -248,7 +250,7 @@ class IterableMapper {
|
|
|
248
250
|
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
249
251
|
// throw the errors as an aggregate exception
|
|
250
252
|
this._isRejected = true;
|
|
251
|
-
throw new
|
|
253
|
+
throw new aggregate_error_1.default(this._errors);
|
|
252
254
|
}
|
|
253
255
|
return { value: undefined, done };
|
|
254
256
|
}
|
|
@@ -265,7 +267,7 @@ class IterableMapper {
|
|
|
265
267
|
bubbleUpErrors() {
|
|
266
268
|
if (!this._options.stopOnMapperError && this._errors.length > 0) {
|
|
267
269
|
// throw the errors as an aggregate exception
|
|
268
|
-
throw new
|
|
270
|
+
throw new aggregate_error_1.default(this._errors);
|
|
269
271
|
}
|
|
270
272
|
}
|
|
271
273
|
startARunnerIfNeeded() {
|
|
@@ -280,34 +282,15 @@ class IterableMapper {
|
|
|
280
282
|
if (this._activeRunners < this._options.concurrency) {
|
|
281
283
|
// We only create runners if we're under the concurrency limit
|
|
282
284
|
if (this._unreadQueue.length + this._activeRunners <= this._options.maxUnread) {
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
|
|
286
|
-
this.
|
|
285
|
+
// The eligibility checks above and reservation below run synchronously.
|
|
286
|
+
// Reserve the slot before starting asynchronous source work.
|
|
287
|
+
this._activeRunners++;
|
|
288
|
+
void this.sourceNext();
|
|
287
289
|
}
|
|
288
290
|
}
|
|
289
291
|
}
|
|
290
292
|
}
|
|
291
293
|
}
|
|
292
|
-
startAnotherRunner() {
|
|
293
|
-
if (this._activeRunners === this._options.concurrency) {
|
|
294
|
-
throw new TypeError('active runners would be greater than concurrency limit');
|
|
295
|
-
}
|
|
296
|
-
if (this._activeRunners + this._unreadQueue.length > this._options.maxUnread) {
|
|
297
|
-
throw new TypeError('active runners would overflow the read queue limit');
|
|
298
|
-
}
|
|
299
|
-
if (this._isIterableDone) {
|
|
300
|
-
throw new TypeError('runner should not be started when iterable is already done');
|
|
301
|
-
}
|
|
302
|
-
if (this._activeRunners < 0) {
|
|
303
|
-
throw new TypeError('active runners is less than 0');
|
|
304
|
-
}
|
|
305
|
-
// We only create runners if the number of runners + unread items will not
|
|
306
|
-
// exceed the unread queue length
|
|
307
|
-
this._activeRunners++;
|
|
308
|
-
// Start another source runner, but do not await it
|
|
309
|
-
void this.sourceNext();
|
|
310
|
-
}
|
|
311
294
|
areWeDone() {
|
|
312
295
|
if (this._isIterableDone) {
|
|
313
296
|
// The source iterable has no more items
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterable-mapper.js","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"iterable-mapper.js","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":";AAAA,EAAE;AACF,sGAAsG;AACtG,EAAE;;;;;;AAEF,sEAA6C;AAC7C,qDAAiD;AA8EjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyJG;AACH,MAAa,cAAc;IAgBzB;;;;;;;;OAQG;IACH,YACE,KAAiD,EACjD,MAAmC,EACnC,UAAiC,EAAE;QArBpB,YAAO,GAAG,EAAa,CAAC;QACjC,mBAAc,GAAG,KAAK,CAAC;QACvB,gBAAW,GAAG,KAAK,CAAC;QACpB,oBAAe,GAAG,KAAK,CAAC;QACxB,mBAAc,GAAG,CAAC,CAAC;QACnB,oBAAe,GAAG,CAAC,CAAC;QACpB,kBAAa,GAAG,CAAC,CAAC;QAClB,2BAAsB,GAAG,KAAK,CAAC;QAgBrC,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,iBAAiB,GAAG,IAAI,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAE7E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,WAAW,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;QAE9D,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,CAAC;YACjC,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;QACrD,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,CACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,WAAW,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACzD,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,CAC/B,EAAE,CAAC;YACF,MAAM,IAAI,SAAS,CACjB,kFAAkF,WAAW,OAAO,OAAO,WAAW,GAAG,CAC1H,CAAC;QACJ,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,CACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,MAAM,CAAC,iBAAiB,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAC7B,EAAE,CAAC;YACF,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,OAAO,OAAO,SAAS,GAAG,CACpH,CAAC;QACJ,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxD,MAAM,IAAI,SAAS,CACjB,gFAAgF,SAAS,UAAU,WAAW,IAAI,CACnH,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,8BAAa,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QAErD,4BAA4B;QAC5B,IAAK,KAAgC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,SAAS,EAAE,CAAC;YAC1E,8BAA8B;YAC9B,IAAI,CAAC,SAAS,GAAI,KAAgC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC3E,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,GAAI,KAA2B,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnE,CAAC;QAED,oEAAoE;QACpE,0DAA0D;QAC1D,gEAAgE;QAChE,mDAAmD;QACnD,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC;gBACjD,4BAA4B;gBAC5B,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,0DAA0D;gBAC1D,4EAA4E;gBAC5E,sCAAsC;gBACtC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBAExB,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC7C,MAAM;gBACR,CAAC;YACH,CAAC;YAED,uFAAuF;YACvF,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACrC,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,IAAI;QACf,qEAAqE;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,6CAA6C;gBAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,MAAM,IAAI,yBAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAC/C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,yCAAyC;YACzC,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,6CAA6C;YAC7C,MAAM,IAAI,yBAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,oEAAoE;QACpE,sFAAsF;QACtF,qCAAqC;QACrC,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAChC,uEAAuE;YACvE,uDAAuD;YACvD,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/C,yEAAyE;gBACzE,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;oBACpD,8DAA8D;oBAC9D,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;wBAC9E,wEAAwE;wBACxE,6DAA6D;wBAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;wBACtB,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;oBACzB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,wCAAwC;YACxC,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;gBAC/B,sCAAsC;gBACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnC,iCAAiC;oBACjC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;oBAEzB,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,YAAY,CAAC,IAAmC;QACtD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;aAAM,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,6EAA6E;QAC7E,IAAI,QAAiC,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAI,IAAI,CAAC,SAA+B,CAAC,IAAI,EAAE,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4FAA4F;YAC5F,8EAA8E;YAC9E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,cAAc,EAAE,CAAC;YAEtB,iEAAiE;YACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAE3C,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;QACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAE5B,iEAAiE;YACjE,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;gBAC/B,2EAA2E;gBAC3E,oCAAoC;gBACpC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;YAC3B,CAAC;YAED,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,mDAAmD;QACnD,+DAA+D;QAC/D,8DAA8D;QAC9D,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,EAAE;YACF,4CAA4C;YAC5C,4EAA4E;YAC5E,yEAAyE;YACzE,2DAA2D;YAC3D,EAAE;YACF,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC;gBAE/B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,IAAI,CAAC,cAAc,EAAE,CAAC;oBACtB,OAAO;gBACT,CAAC;gBAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjD,IAAI,CAAC,eAAe,EAAE,CAAC;gBAEvB,4BAA4B;gBAC5B,gCAAgC;gBAChC,WAAW;gBACX,2BAA2B;gBAC3B,IAAI;gBAEJ,iCAAiC;gBACjC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpD,8DAA8D;YAChE,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;oBACpC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;oBACxB,IAAI,CAAC,cAAc,EAAE,CAAC;oBAEtB,iEAAiE;oBACjE,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;oBAE3C,mCAAmC;gBACrC,CAAC;qBAAM,CAAC;oBACN,8CAA8C;oBAC9C,uDAAuD;oBACvD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACzB,IAAI,CAAC,eAAe,EAAE,CAAC;oBAEvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBAExB,mEAAmE;oBACnE,OAAO;gBACT,CAAC;YACH,CAAC;YAED,EAAE;YACF,4CAA4C;YAC5C,EAAE;YAEF,iEAAiE;YACjE,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC7E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,OAAO;YACT,CAAC;YAED,qBAAqB;YACrB,wEAAwE;YACxE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;CACF;AAtUD,wCAsUC"}
|