@shutterstock/p-map-iterable 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -14
- package/dist/src/blocking-queue.test.js +2 -2
- package/dist/src/blocking-queue.test.js.map +1 -1
- package/dist/src/iterable-mapper.d.ts +173 -30
- package/dist/src/iterable-mapper.d.ts.map +1 -1
- package/dist/src/iterable-mapper.js +143 -25
- package/dist/src/iterable-mapper.js.map +1 -1
- package/dist/src/iterable-mapper.test.js +28 -0
- package/dist/src/iterable-mapper.test.js.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.d.ts +44 -29
- package/dist/src/iterable-queue-mapper-simple.d.ts.map +1 -1
- package/dist/src/iterable-queue-mapper-simple.js +38 -21
- package/dist/src/iterable-queue-mapper-simple.js.map +1 -1
- package/dist/src/iterable-queue-mapper.d.ts +38 -58
- package/dist/src/iterable-queue-mapper.d.ts.map +1 -1
- package/dist/src/iterable-queue-mapper.js +33 -33
- package/dist/src/iterable-queue-mapper.js.map +1 -1
- package/dist/src/iterable-queue-mapper.test.js +6 -4
- package/dist/src/iterable-queue-mapper.test.js.map +1 -1
- package/dist/src/iterable-queue.test.js +3 -3
- package/dist/src/iterable-queue.test.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -2,24 +2,115 @@
|
|
|
2
2
|
|
|
3
3
|
# Overview
|
|
4
4
|
|
|
5
|
-
`@shutterstock/p-map-iterable` provides several classes that allow processing results of `p-map`-style mapper functions by iterating the results as they are completed, with
|
|
5
|
+
`@shutterstock/p-map-iterable` provides several classes that allow processing results of `p-map`-style mapper functions by iterating the results as they are completed, with backpressure to limit the number of items that are processed ahead of the consumer.
|
|
6
6
|
|
|
7
7
|
A common use case for `@shutterstock/p-map-iterable` is as a "prefetcher" that will fetch, for example, AWS S3 files in an AWS Lambda function. By prefetching large files the consumer is able to use 100% of the paid-for Lambda CPU time for the JS thread, rather than waiting idle while the next file is fetched. The backpressure (set by `maxUnread`) prevents the prefetcher from consuming unlimited memory or disk space by racing ahead of the consumer.
|
|
8
8
|
|
|
9
9
|
These classes will typically be helpful in batch or queue consumers, not as much in request/response services.
|
|
10
10
|
|
|
11
|
-
# Example Usage
|
|
11
|
+
# Example Usage Scenarios
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
13
|
+
## Typical Processing Loop without `IterableMapper`
|
|
14
|
+
|
|
15
|
+
```typescript
|
|
16
|
+
const source = new SomeSource();
|
|
17
|
+
const sourceIds = [1, 2,... 1000];
|
|
18
|
+
const sink = new SomeSink();
|
|
19
|
+
for (const sourceId of sourceIds) {
|
|
20
|
+
const item = await source.read(sourceId); // takes 300 ms of I/O wait, no CPU
|
|
21
|
+
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
22
|
+
await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Each iteration takes 820ms total, but we waste time waiting for I/O. We could prefetch the next read (300ms) while processing (20ms) and writing (500ms), without changing the order of reads or writes.
|
|
27
|
+
|
|
28
|
+
## Using `IterableMapper` as Prefetcher with Blocking Sequential Writes
|
|
29
|
+
|
|
30
|
+
`concurrency: 1` on the prefetcher preserves the order of the reads and and writes are sequential and blocking (unchanged).
|
|
31
|
+
|
|
32
|
+
```typescript
|
|
33
|
+
const source = new SomeSource();
|
|
34
|
+
const sourceIds = [1, 2,... 1000];
|
|
35
|
+
// Pre-reads up to 8 items serially and releases in sequential order
|
|
36
|
+
const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
37
|
+
async (sourceId) => source.read(sourceId),
|
|
38
|
+
{ concurrency: 1, maxUnread: 10 }
|
|
39
|
+
);
|
|
40
|
+
const sink = new SomeSink();
|
|
41
|
+
for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
42
|
+
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
43
|
+
await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
This reduces iteration time to 520ms by overlapping reads with processing/writing.
|
|
48
|
+
|
|
49
|
+
## Using `IterableMapper` as Prefetcher with Background Sequential Writes with `IterableQueueMapperSimple`
|
|
50
|
+
|
|
51
|
+
`concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
52
|
+
`concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
53
|
+
|
|
54
|
+
```typescript
|
|
55
|
+
const source = new SomeSource();
|
|
56
|
+
const sourceIds = [1, 2,... 1000];
|
|
57
|
+
const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
58
|
+
async (sourceId) => source.read(sourceId),
|
|
59
|
+
{ concurrency: 1, maxUnread: 10 }
|
|
60
|
+
);
|
|
61
|
+
const sink = new SomeSink();
|
|
62
|
+
const flusher = new IterableQueueMapperSimple(
|
|
63
|
+
async (outputItem) => sink.write(outputItem),
|
|
64
|
+
{ concurrency: 1 }
|
|
65
|
+
);
|
|
66
|
+
for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
67
|
+
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
68
|
+
await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
69
|
+
}
|
|
70
|
+
// Wait for all writes to complete
|
|
71
|
+
await flusher.onIdle();
|
|
72
|
+
// Check for errors
|
|
73
|
+
if (flusher.errors.length > 0) {
|
|
74
|
+
// ...
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
This reduces iteration time to about `max((max(readTime, writeTime) - cpuOpTime, cpuOpTime))`
|
|
79
|
+
by overlapping reads and writes with the CPU processing step.
|
|
80
|
+
In this contrived example, the loop time is reduced to 500ms - 20ms = 480ms.
|
|
81
|
+
In cases where the CPU usage time is higher, the impact can be greater.
|
|
82
|
+
|
|
83
|
+
## Using `IterableMapper` as Prefetcher with Out of Order Reads and Background Out of Order Writes with `IterableQueueMapperSimple`
|
|
84
|
+
|
|
85
|
+
For maximum throughput, allow out of order reads and writes with
|
|
86
|
+
`IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
87
|
+
`IterableQueueMapperSimple` (to handle errors at end without custom iteration and applying backpressure to block further enqueues when `concurrency` items are in process):
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
const source = new SomeSource();
|
|
91
|
+
const sourceIds = [1, 2,... 1000];
|
|
92
|
+
const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
93
|
+
async (sourceId) => source.read(sourceId),
|
|
94
|
+
{ concurrency: 10, maxUnread: 20 }
|
|
95
|
+
);
|
|
96
|
+
const sink = new SomeSink();
|
|
97
|
+
const flusher = new IterableQueueMapperSimple(
|
|
98
|
+
async (outputItem) => sink.write(outputItem),
|
|
99
|
+
{ concurrency: 10 }
|
|
100
|
+
);
|
|
101
|
+
for await (const item of sourcePrefetcher) { // typically will not block
|
|
102
|
+
const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
103
|
+
await flusher.enqueue(outputItem); // typically will not block
|
|
104
|
+
}
|
|
105
|
+
// Wait for all writes to complete
|
|
106
|
+
await flusher.onIdle();
|
|
107
|
+
// Check for errors
|
|
108
|
+
if (flusher.errors.length > 0) {
|
|
109
|
+
// ...
|
|
110
|
+
}
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
This reduces iteration time to about 20ms by overlapping reads and writes with the CPU processing step. In this contrived (but common) example we would get a 41x improvement in throughput, removing 97.5% of the time to process each item and fully utilizing the CPU time available in the JS event loop.
|
|
23
114
|
|
|
24
115
|
# Getting Started
|
|
25
116
|
|
|
@@ -66,7 +157,7 @@ These diagrams illustrate the differences in operation betweeen `p-map`, `p-queu
|
|
|
66
157
|
- User supplied sync or async mapper function
|
|
67
158
|
- Exposes an async iterable interface for consuming mapped items
|
|
68
159
|
- Allows a maximum queue depth of mapped items - if the consumer stops consuming, the queue will fill up, at which point the mapper will stop being invoked until an item is consumed from the queue
|
|
69
|
-
- This allows mapping with
|
|
160
|
+
- This allows mapping with backpressure so that the mapper does not consume unlimited resources (e.g. memory, disk, network, event loop time) by racing ahead of the consumer
|
|
70
161
|
- [IterableQueueMapper](https://tech.shutterstock.com/p-map-iterable/classes/IterableQueueMapper.html)
|
|
71
162
|
- Wraps `IterableMapper`
|
|
72
163
|
- Adds items to the queue via the `enqueue` method
|
|
@@ -91,7 +182,7 @@ These diagrams illustrate the differences in operation betweeen `p-map`, `p-queu
|
|
|
91
182
|
|
|
92
183
|
See [p-map](https://github.com/sindresorhus/p-map) docs for a good start in understanding what this does.
|
|
93
184
|
|
|
94
|
-
The key difference between `IterableMapper` and `pMap` are that `IterableMapper` does not return when the entire mapping is done, rather it exposes an iterable that the caller loops through. This enables results to be processed while the mapping is still happening, while optionally allowing for
|
|
185
|
+
The key difference between `IterableMapper` and `pMap` are that `IterableMapper` does not return when the entire mapping is done, rather it exposes an iterable that the caller loops through. This enables results to be processed while the mapping is still happening, while optionally allowing for backpressure to slow or stop the mapping if the caller is not consuming items fast enough. Common use cases include `prefetching` items from a remote service - the next set of requests are dispatched asyncronously while the current responses are processed and the prefetch requests will pause when the unread queue fills up.
|
|
95
186
|
|
|
96
187
|
See [examples/iterable-mapper.ts](./examples/iterable-mapper.ts) for an example.
|
|
97
188
|
|
|
@@ -65,7 +65,7 @@ describe('BlockingQueue', () => {
|
|
|
65
65
|
expect(await queue.dequeue()).toBe(1);
|
|
66
66
|
expect(Date.now() - startTime).toBeLessThan(2000);
|
|
67
67
|
expect(await queue.dequeue()).toBe(2);
|
|
68
|
-
expect(Date.now() - startTime).toBeGreaterThanOrEqual(2000);
|
|
68
|
+
expect(Math.ceil(Date.now() - startTime)).toBeGreaterThanOrEqual(2000);
|
|
69
69
|
queue.done();
|
|
70
70
|
});
|
|
71
71
|
});
|
|
@@ -145,7 +145,7 @@ describe('BlockingQueue', () => {
|
|
|
145
145
|
expect(await queue.dequeue()).toBe(1);
|
|
146
146
|
const startTime = Date.now();
|
|
147
147
|
expect(await queue.dequeue()).toBe(2);
|
|
148
|
-
const duration = Date.now() - startTime;
|
|
148
|
+
const duration = Math.ceil(Date.now() - startTime);
|
|
149
149
|
expect(duration).toBeGreaterThanOrEqual(2000);
|
|
150
150
|
queue.done();
|
|
151
151
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blocking-queue.test.js","sourceRoot":"","sources":["../../src/blocking-queue.test.ts"],"names":[],"mappings":";;AAAA,8BAA8B;AAC9B,qDAAiD;AAEjD,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,SAAS,CAAC,GAAG,EAAE;QACb,UAAU;IACZ,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAC7D,sCAAsC,CACvC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAkC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,KAAK,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAE1D,iBAAiB;YACjB,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,+EAA+E;YAC/E,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"blocking-queue.test.js","sourceRoot":"","sources":["../../src/blocking-queue.test.ts"],"names":[],"mappings":";;AAAA,8BAA8B;AAC9B,qDAAiD;AAEjD,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,SAAS,CAAC,GAAG,EAAE;QACb,UAAU;IACZ,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAC7D,sCAAsC,CACvC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAkC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,KAAK,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAE1D,iBAAiB;YACjB,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,+EAA+E;YAC/E,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAEvE,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAErB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAC7D,sCAAsC,CACvC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAkC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,KAAK,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAkC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,KAAK,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAE1D,6BAA6B;YAC7B,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,+EAA+E;YAC/E,UAAU,CAAC,GAAG,EAAE;gBACd,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;YACnD,MAAM,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAE9C,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,GAAG,IAAI,8BAAa,CAAS,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAE/B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,OAAO,GAAkC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -3,23 +3,48 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export interface IterableMapperOptions {
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* Maximum number of concurrent invocations of `mapper` to run at once.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
8
|
+
* The number of concurrent invocations is dynamically adjusted based on the `maxUnread` limit:
|
|
9
|
+
* - If there are no unread items and `maxUnread` is 10 with `concurrency` of 4, all 4 mappers can run.
|
|
10
|
+
* - If there are already 8 unread items in the queue, only 2 mappers will run to avoid exceeding
|
|
11
|
+
* the `maxUnread` limit of 10.
|
|
12
|
+
* - If there are 10 unread items, no mappers will run until an item is consumed from the queue.
|
|
13
|
+
*
|
|
14
|
+
* This ensures efficient processing while maintaining backpressure through the `maxUnread` limit.
|
|
15
|
+
*
|
|
16
|
+
* Setting `concurrency` to 1 enables serial processing, preserving the order of items
|
|
17
|
+
* while still benefiting from the backpressure mechanism.
|
|
18
|
+
*
|
|
19
|
+
* Must be an integer from 1 and up or `Infinity`, and must be <= `maxUnread`.
|
|
9
20
|
*
|
|
10
21
|
* @default 4
|
|
11
22
|
*/
|
|
12
23
|
readonly concurrency?: number;
|
|
13
24
|
/**
|
|
14
|
-
*
|
|
25
|
+
* Maximum number of unread items allowed to accumulate before applying backpressure.
|
|
26
|
+
*
|
|
27
|
+
* This parameter is crucial for controlling memory usage and system load by:
|
|
28
|
+
* 1. Limiting the number of processed but unread items in the queue
|
|
29
|
+
* 2. Automatically pausing mapper execution when the limit is reached
|
|
30
|
+
* 3. Resuming processing when items are consumed, maintaining optimal throughput
|
|
15
31
|
*
|
|
16
|
-
*
|
|
32
|
+
* For example, when reading from a slow database:
|
|
33
|
+
* - With maxUnread=10, only 10 items will be fetched before the consumer reads them
|
|
34
|
+
* - Additional items won't be fetched until the consumer reads existing items
|
|
35
|
+
* - This prevents runaway memory usage for items that cannot be processed quickly enough
|
|
36
|
+
*
|
|
37
|
+
* Must be an integer from 1 and up or `Infinity`, and must be >= `concurrency`.
|
|
38
|
+
* It is not typical to set this value to `Infinity`, but rather to a value such as 1 to 10.
|
|
17
39
|
*
|
|
18
40
|
* @default 8
|
|
19
41
|
*/
|
|
20
42
|
readonly maxUnread?: number;
|
|
21
43
|
/**
|
|
22
|
-
* When set to `false`, instead of stopping when a promise rejects, it will wait for all
|
|
44
|
+
* When set to `false`, instead of stopping when a promise rejects, it will wait for all
|
|
45
|
+
* the promises to settle and then reject with an
|
|
46
|
+
* [aggregated error](https://github.com/sindresorhus/aggregate-error) containing all the
|
|
47
|
+
* errors from the rejected promises.
|
|
23
48
|
*
|
|
24
49
|
* @default true
|
|
25
50
|
*/
|
|
@@ -35,20 +60,150 @@ export interface IterableMapperOptions {
|
|
|
35
60
|
*/
|
|
36
61
|
export type Mapper<Element = unknown, NewElement = unknown> = (element: Element, index: number) => NewElement | Promise<NewElement>;
|
|
37
62
|
/**
|
|
38
|
-
* Iterates over a source iterable with specified concurrency
|
|
63
|
+
* Iterates over a source iterable / generator with specified `concurrency`,
|
|
39
64
|
* calling the `mapper` on each iterated item, and storing the
|
|
40
|
-
* `mapper` result in a queue of
|
|
65
|
+
* `mapper` result in a queue of `maxUnread` size, before
|
|
41
66
|
* being iterated / read by the caller.
|
|
42
67
|
*
|
|
43
68
|
* @remarks
|
|
44
69
|
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
70
|
+
* ### Typical Use Case
|
|
71
|
+
* - Prefetching items from an async I/O source
|
|
72
|
+
* - In the simple sequential (`concurrency: 1`) case, allows items to be prefetched async, preserving order, while caller processes an item
|
|
73
|
+
* - Can allow parallel prefetches for sources that allow for out of order reads (`concurrency: 2+`)
|
|
74
|
+
* - Prevents the producer from racing ahead of the consumer if `maxUnread` is reached
|
|
75
|
+
*
|
|
76
|
+
* ### Error Handling
|
|
77
|
+
* The mapper should ideally handle all errors internally to enable error handling
|
|
78
|
+
* closest to where they occur. However, if errors do escape the mapper:
|
|
79
|
+
*
|
|
80
|
+
* When `stopOnMapperError` is true (default):
|
|
81
|
+
* - First error immediately stops processing
|
|
82
|
+
* - Error is thrown from the `AsyncIterator`'s next() call
|
|
83
|
+
*
|
|
84
|
+
* When `stopOnMapperError` is false:
|
|
85
|
+
* - Processing continues despite errors
|
|
86
|
+
* - All errors are collected and thrown together
|
|
87
|
+
* - Errors are thrown as `AggregateError` after all items complete
|
|
88
|
+
*
|
|
89
|
+
* ### Usage
|
|
90
|
+
* - Items are exposed to the `mapper` via an iterator or async iterator (this includes generator and async generator functions)
|
|
91
|
+
* - IMPORTANT: `mapper` method not be invoked when `maxUnread` is reached, until items are consumed
|
|
92
|
+
* - The iterable will set `done` when the `input` has indicated `done` and all `mapper` promises have resolved
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
*
|
|
96
|
+
* ### Typical Processing Loop without `IterableMapper`
|
|
97
|
+
*
|
|
98
|
+
* ```typescript
|
|
99
|
+
* const source = new SomeSource();
|
|
100
|
+
* const sourceIds = [1, 2,... 1000];
|
|
101
|
+
* const sink = new SomeSink();
|
|
102
|
+
* for (const sourceId of sourceIds) {
|
|
103
|
+
* const item = await source.read(sourceId); // takes 300 ms of I/O wait, no CPU
|
|
104
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
105
|
+
* await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
106
|
+
* }
|
|
107
|
+
* ```
|
|
108
|
+
*
|
|
109
|
+
* Each iteration takes 820ms total, but we waste time waiting for I/O.
|
|
110
|
+
* We could prefetch the next read (300ms) while processing (20ms) and writing (500ms),
|
|
111
|
+
* without changing the order of reads or writes.
|
|
112
|
+
*
|
|
113
|
+
* @example
|
|
48
114
|
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
115
|
+
* ### Using `IterableMapper` as Prefetcher with Blocking Sequential Writes
|
|
116
|
+
*
|
|
117
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads and and writes are sequential and blocking (unchanged).
|
|
118
|
+
*
|
|
119
|
+
* ```typescript
|
|
120
|
+
* const source = new SomeSource();
|
|
121
|
+
* const sourceIds = [1, 2,... 1000];
|
|
122
|
+
* // Pre-reads up to 8 items serially and releases in sequential order
|
|
123
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
124
|
+
* async (sourceId) => source.read(sourceId),
|
|
125
|
+
* { concurrency: 1, maxUnread: 10 }
|
|
126
|
+
* );
|
|
127
|
+
* const sink = new SomeSink();
|
|
128
|
+
* for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
129
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
130
|
+
* await sink.write(outputItem); // takes 500 ms of I/O wait, no CPU
|
|
131
|
+
* }
|
|
132
|
+
* ```
|
|
133
|
+
*
|
|
134
|
+
* This reduces iteration time to 520ms by overlapping reads with processing/writing.
|
|
135
|
+
*
|
|
136
|
+
* @example
|
|
137
|
+
*
|
|
138
|
+
* ### Using `IterableMapper` as Prefetcher with Background Sequential Writes with `IterableQueueMapperSimple`
|
|
139
|
+
*
|
|
140
|
+
* `concurrency: 1` on the prefetcher preserves the order of the reads.
|
|
141
|
+
* `concurrency: 1` on the flusher preserves the order of the writes, but allows the loop to iterate while last write is completing.
|
|
142
|
+
*
|
|
143
|
+
* ```typescript
|
|
144
|
+
* const source = new SomeSource();
|
|
145
|
+
* const sourceIds = [1, 2,... 1000];
|
|
146
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
147
|
+
* async (sourceId) => source.read(sourceId),
|
|
148
|
+
* { concurrency: 1, maxUnread: 10 }
|
|
149
|
+
* );
|
|
150
|
+
* const sink = new SomeSink();
|
|
151
|
+
* const flusher = new IterableQueueMapperSimple(
|
|
152
|
+
* async (outputItem) => sink.write(outputItem),
|
|
153
|
+
* { concurrency: 1 }
|
|
154
|
+
* );
|
|
155
|
+
* for await (const item of sourcePrefetcher) { // may not block for fast sources
|
|
156
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
157
|
+
* await flusher.enqueue(outputItem); // will periodically block for portion of write time
|
|
158
|
+
* }
|
|
159
|
+
* // Wait for all writes to complete
|
|
160
|
+
* await flusher.onIdle();
|
|
161
|
+
* // Check for errors
|
|
162
|
+
* if (flusher.errors.length > 0) {
|
|
163
|
+
* // ...
|
|
164
|
+
* }
|
|
165
|
+
* ```
|
|
166
|
+
*
|
|
167
|
+
* This reduces iteration time to about `max((max(readTime, writeTime) - cpuOpTime, cpuOpTime))`
|
|
168
|
+
* by overlapping reads and writes with the CPU processing step.
|
|
169
|
+
* In this contrived example, the loop time is reduced to 500ms - 20ms = 480ms.
|
|
170
|
+
* In cases where the CPU usage time is higher, the impact can be greater.
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
*
|
|
174
|
+
* ### Using `IterableMapper` as Prefetcher with Out of Order Reads and Background Out of Order Writes with `IterableQueueMapperSimple`
|
|
175
|
+
*
|
|
176
|
+
* For maximum throughput, allow out of order reads and writes with
|
|
177
|
+
* `IterableQueueMapper` (to iterate results with backpressure when too many unread items) or
|
|
178
|
+
* `IterableQueueMapperSimple` (to handle errors at end without custom iteration and applying backpressure to block further enqueues when `concurrency` items are in process):
|
|
179
|
+
*
|
|
180
|
+
* ```typescript
|
|
181
|
+
* const source = new SomeSource();
|
|
182
|
+
* const sourceIds = [1, 2,... 1000];
|
|
183
|
+
* const sourcePrefetcher = new IterableMapper(sourceIds,
|
|
184
|
+
* async (sourceId) => source.read(sourceId),
|
|
185
|
+
* { concurrency: 10, maxUnread: 20 }
|
|
186
|
+
* );
|
|
187
|
+
* const sink = new SomeSink();
|
|
188
|
+
* const flusher = new IterableQueueMapperSimple(
|
|
189
|
+
* async (outputItem) => sink.write(outputItem),
|
|
190
|
+
* { concurrency: 10 }
|
|
191
|
+
* );
|
|
192
|
+
* for await (const item of sourcePrefetcher) { // typically will not block
|
|
193
|
+
* const outputItem = doSomeOperation(item); // takes 20 ms of CPU
|
|
194
|
+
* await flusher.enqueue(outputItem); // typically will not block
|
|
195
|
+
* }
|
|
196
|
+
* // Wait for all writes to complete
|
|
197
|
+
* await flusher.onIdle();
|
|
198
|
+
* // Check for errors
|
|
199
|
+
* if (flusher.errors.length > 0) {
|
|
200
|
+
* // ...
|
|
201
|
+
* }
|
|
202
|
+
* ```
|
|
203
|
+
*
|
|
204
|
+
* This reduces iteration time to about 20ms by overlapping reads and writes with the CPU processing step.
|
|
205
|
+
* In this contrived (but common) example we would get a 41x improvement in throughput, removing 97.5% of
|
|
206
|
+
* the time to process each item and fully utilizing the CPU time available in the JS event loop.
|
|
52
207
|
*
|
|
53
208
|
* @category Iterable Input
|
|
54
209
|
*/
|
|
@@ -68,23 +223,11 @@ export declare class IterableMapper<Element, NewElement> implements AsyncIterabl
|
|
|
68
223
|
/**
|
|
69
224
|
* Create a new `IterableMapper`
|
|
70
225
|
*
|
|
71
|
-
* @param input Iterated over concurrently in the `mapper` function.
|
|
72
|
-
* @param mapper Function
|
|
73
|
-
* Expected to return a `Promise` or value.
|
|
74
|
-
*
|
|
75
|
-
* The `mapper` *should* handle all errors and not allow an error to be thrown
|
|
76
|
-
* out of the `mapper` function as this enables the best handling of errors
|
|
77
|
-
* closest to the time that they occur.
|
|
78
|
-
*
|
|
79
|
-
* If the `mapper` function does allow an error to be thrown then the
|
|
80
|
-
* `stopOnMapperError` option controls the behavior:
|
|
81
|
-
* - `stopOnMapperError`: `true` - will throw the error
|
|
82
|
-
* out of `next` or the `AsyncIterator` returned from `[Symbol.asyncIterator]`
|
|
83
|
-
* and stop processing.
|
|
84
|
-
* - `stopOnMapperError`: `false` - will continue processing
|
|
85
|
-
* and accumulate the errors to be thrown from `next` or the `AsyncIterator`
|
|
86
|
-
* returned from `[Symbol.asyncIterator]` when all items have been processed.
|
|
226
|
+
* @param input Iterated over concurrently, or serially, in the `mapper` function.
|
|
227
|
+
* @param mapper Function called for every item in `input`. Returns a `Promise` or value.
|
|
87
228
|
* @param options IterableMapper options
|
|
229
|
+
*
|
|
230
|
+
* @see {@link IterableQueueMapper} for full class documentation
|
|
88
231
|
*/
|
|
89
232
|
constructor(input: AsyncIterable<Element> | Iterable<Element>, mapper: Mapper<Element, NewElement>, options?: IterableMapperOptions);
|
|
90
233
|
[Symbol.asyncIterator](): AsyncIterator<NewElement>;
|
|
@@ -106,7 +249,7 @@ export declare class IterableMapper<Element, NewElement> implements AsyncIterabl
|
|
|
106
249
|
*/
|
|
107
250
|
private throwIfError;
|
|
108
251
|
/**
|
|
109
|
-
* Get the next item from the
|
|
252
|
+
* Get the next item from the `input` iterable.
|
|
110
253
|
*
|
|
111
254
|
* @remarks
|
|
112
255
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterable-mapper.d.ts","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC
|
|
1
|
+
{"version":3,"file":"iterable-mapper.d.ts","sourceRoot":"","sources":["../../src/iterable-mapper.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,MAAM,CAAC,OAAO,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO,IAAI,CAC5D,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,KACV,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAUtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AACH,qBAAa,cAAc,CAAC,OAAO,EAAE,UAAU,CAAE,YAAW,aAAa,CAAC,UAAU,CAAC;IACnF,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,QAAQ,CAAkC;IAElD,OAAO,CAAC,YAAY,CAA+C;IAEnE,OAAO,CAAC,SAAS,CAA6C;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,sBAAsB,CAAS;IAEvC;;;;;;;;OAQG;gBAED,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,EACjD,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,EACnC,OAAO,GAAE,qBAA0B;IAwF9B,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,UAAU,CAAC;IAI1D;;;;;OAKG;IACU,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;IAyBxD,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,oBAAoB;IAuB5B,OAAO,CAAC,kBAAkB;IAwB1B,OAAO,CAAC,SAAS;IAiBjB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;;;;;;;;;;OAcG;YACW,UAAU;CAiHzB"}
|