@shutterstock/p-map-iterable 1.1.1 → 1.2.0-prerelease.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +79 -17
  2. package/dist/src/blocking-queue.js +6 -20
  3. package/dist/src/blocking-queue.js.map +1 -1
  4. package/dist/src/index.d.ts +3 -2
  5. package/dist/src/index.js +4 -1
  6. package/dist/src/index.js.map +1 -1
  7. package/dist/src/iterable-mapper.d.ts +18 -13
  8. package/dist/src/iterable-mapper.js +28 -45
  9. package/dist/src/iterable-mapper.js.map +1 -1
  10. package/dist/src/iterable-queue-mapper-simple.d.ts +28 -21
  11. package/dist/src/iterable-queue-mapper-simple.js +29 -23
  12. package/dist/src/iterable-queue-mapper-simple.js.map +1 -1
  13. package/dist/src/iterable-queue-mapper.d.ts +18 -9
  14. package/dist/src/iterable-queue-mapper.js +17 -8
  15. package/dist/src/iterable-queue-mapper.js.map +1 -1
  16. package/dist/src/iterable-queue.js.map +1 -1
  17. package/dist/src/queue.js.map +1 -1
  18. package/package.json +25 -19
  19. package/dist/src/blocking-queue.d.ts.map +0 -1
  20. package/dist/src/blocking-queue.test.d.ts +0 -2
  21. package/dist/src/blocking-queue.test.d.ts.map +0 -1
  22. package/dist/src/blocking-queue.test.js +0 -174
  23. package/dist/src/blocking-queue.test.js.map +0 -1
  24. package/dist/src/index.d.ts.map +0 -1
  25. package/dist/src/iterable-mapper.d.ts.map +0 -1
  26. package/dist/src/iterable-mapper.test.d.ts +0 -2
  27. package/dist/src/iterable-mapper.test.d.ts.map +0 -1
  28. package/dist/src/iterable-mapper.test.js +0 -898
  29. package/dist/src/iterable-mapper.test.js.map +0 -1
  30. package/dist/src/iterable-queue-mapper-simple.d.ts.map +0 -1
  31. package/dist/src/iterable-queue-mapper-simple.test.d.ts +0 -2
  32. package/dist/src/iterable-queue-mapper-simple.test.d.ts.map +0 -1
  33. package/dist/src/iterable-queue-mapper-simple.test.js +0 -102
  34. package/dist/src/iterable-queue-mapper-simple.test.js.map +0 -1
  35. package/dist/src/iterable-queue-mapper.d.ts.map +0 -1
  36. package/dist/src/iterable-queue-mapper.test.d.ts +0 -2
  37. package/dist/src/iterable-queue-mapper.test.d.ts.map +0 -1
  38. package/dist/src/iterable-queue-mapper.test.js +0 -119
  39. package/dist/src/iterable-queue-mapper.test.js.map +0 -1
  40. package/dist/src/iterable-queue.d.ts.map +0 -1
  41. package/dist/src/iterable-queue.test.d.ts +0 -2
  42. package/dist/src/iterable-queue.test.d.ts.map +0 -1
  43. package/dist/src/iterable-queue.test.js +0 -143
  44. package/dist/src/iterable-queue.test.js.map +0 -1
  45. package/dist/src/queue.d.ts.map +0 -1
  46. package/dist/src/queue.test.d.ts +0 -2
  47. package/dist/src/queue.test.d.ts.map +0 -1
  48. package/dist/src/queue.test.js +0 -75
  49. package/dist/src/queue.test.js.map +0 -1
@@ -1,898 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- /* eslint-disable @typescript-eslint/no-explicit-any */
4
- /// <reference types="jest" />
5
- const iterable_mapper_1 = require("./iterable-mapper");
6
- const util_1 = require("util");
7
- const sleep = (0, util_1.promisify)(setTimeout);
8
- async function mapper({ value, ms }) {
9
- await sleep(ms);
10
- return value;
11
- }
12
- class SleepIterator {
13
- constructor(max) {
14
- this._current = 1;
15
- this._max = max;
16
- }
17
- async *[Symbol.asyncIterator]() {
18
- for (let i = 0; i < this._max; i++) {
19
- await sleep(1 * (i % 10));
20
- if (this._current <= this._max) {
21
- yield this._current;
22
- }
23
- this._current++;
24
- }
25
- }
26
- }
27
- class ThrowingIterator {
28
- constructor(max, throwOnIndex, delayMS) {
29
- this._max = max;
30
- this._throwOnIndex = throwOnIndex;
31
- this._delayMS = delayMS;
32
- }
33
- [Symbol.asyncIterator]() {
34
- let index = 0;
35
- const max = this._max;
36
- const delayMS = this._delayMS;
37
- const throwOnIndex = this._throwOnIndex;
38
- return {
39
- async next() {
40
- await sleep(delayMS);
41
- if (index === throwOnIndex) {
42
- throw new Error(`throwing on index ${index}`);
43
- }
44
- const item = { value: index, done: index === max };
45
- index++;
46
- return item;
47
- },
48
- };
49
- }
50
- }
51
- describe('IterableMapper', () => {
52
- const sourceNextSpy = jest.spyOn(iterable_mapper_1.IterableMapper.prototype, 'sourceNext');
53
- const startAnotherRunnerSpy = jest.spyOn(iterable_mapper_1.IterableMapper.prototype, 'startAnotherRunner');
54
- beforeAll(() => {
55
- // nothing
56
- });
57
- beforeEach(() => {
58
- jest.clearAllMocks();
59
- });
60
- describe('simple tests', () => {
61
- it('simple mapper works - sync mapper', async () => {
62
- const max = 100;
63
- const iterator = new SleepIterator(max);
64
- let total = 0;
65
- let callCount = 0;
66
- const prefetcher = new iterable_mapper_1.IterableMapper(iterator, (value) => {
67
- callCount++;
68
- total += value;
69
- return total;
70
- });
71
- let lastTotal = 0;
72
- let loopCount = 0;
73
- for await (const item of prefetcher) {
74
- loopCount++;
75
- if (item > lastTotal) {
76
- lastTotal = item;
77
- }
78
- }
79
- expect(callCount).toBe(max);
80
- expect(loopCount).toBe(max);
81
- expect(total).toBe((max * (max + 1)) / 2);
82
- expect(lastTotal).toBe((max * (max + 1)) / 2);
83
- });
84
- it('simple mapper works - async mapper', async () => {
85
- const max = 100;
86
- const iterator = new SleepIterator(max);
87
- let total = 0;
88
- let callCount = 0;
89
- const prefetcher = new iterable_mapper_1.IterableMapper(iterator, async (value) => {
90
- callCount++;
91
- total += value;
92
- await sleep(10 * (callCount % 6));
93
- return total;
94
- });
95
- let lastTotal = 0;
96
- let loopCount = 0;
97
- for await (const item of prefetcher) {
98
- loopCount++;
99
- if (item > lastTotal) {
100
- lastTotal = item;
101
- }
102
- }
103
- expect(callCount).toBe(max);
104
- expect(loopCount).toBe(max);
105
- expect(total).toBe((max * (max + 1)) / 2);
106
- expect(lastTotal).toBe((max * (max + 1)) / 2);
107
- });
108
- it('async mapper works - parallel next() calls', async () => {
109
- const max = 100;
110
- const inputIterator = new SleepIterator(max);
111
- let total = 0;
112
- let callCount = 0;
113
- const prefetcher = new iterable_mapper_1.IterableMapper(inputIterator, async (value) => {
114
- callCount++;
115
- total += value;
116
- await sleep(10 * (callCount % 6));
117
- return total;
118
- });
119
- let lastTotal = 0;
120
- let loopCount = 0;
121
- const nextPromises = [];
122
- const iterator = prefetcher[Symbol.asyncIterator]();
123
- for (let i = 0; i < max + 1; i++) {
124
- nextPromises.push(iterator.next());
125
- }
126
- // Wait for all the readers to resolve
127
- await Promise.all(nextPromises);
128
- for await (const item of nextPromises) {
129
- if (item.done === true) {
130
- break;
131
- }
132
- loopCount++;
133
- if (item.value > lastTotal) {
134
- lastTotal = item.value;
135
- }
136
- }
137
- expect(callCount).toBe(max);
138
- expect(loopCount).toBe(max);
139
- expect(total).toBe((max * (max + 1)) / 2);
140
- expect(lastTotal).toBe((max * (max + 1)) / 2);
141
- });
142
- it('right number run in parallel - simple', async () => {
143
- const startTime = Date.now();
144
- const max = 8;
145
- const delayBetweenMs = 200;
146
- const input = [
147
- { value: 1, ms: delayBetweenMs },
148
- { value: 2, ms: delayBetweenMs },
149
- { value: 3, ms: delayBetweenMs },
150
- { value: 4, ms: delayBetweenMs },
151
- { value: 5, ms: delayBetweenMs },
152
- { value: 6, ms: delayBetweenMs },
153
- { value: 7, ms: delayBetweenMs },
154
- { value: 8, ms: delayBetweenMs },
155
- ];
156
- const prefetcher = new iterable_mapper_1.IterableMapper(input, mapper, { concurrency: 2, maxUnread: 4 });
157
- let lastSeen = 0;
158
- let loopCount = 0;
159
- for await (const item of prefetcher) {
160
- loopCount++;
161
- if (item > lastSeen) {
162
- lastSeen = item;
163
- }
164
- }
165
- expect(loopCount).toBe(max);
166
- expect(lastSeen).toBe(max);
167
- // Should require at least 2 batches
168
- expect(Date.now() - startTime).toBeLessThan(5 * delayBetweenMs);
169
- expect(Date.now() - startTime).toBeGreaterThan(4 * delayBetweenMs);
170
- // The runners should never stop because the items are consumed immediately
171
- expect(startAnotherRunnerSpy).toHaveBeenCalledTimes(0);
172
- });
173
- // T0 - Mapping - Start 1, 2, waiting 200 ms
174
- // Q: []
175
- // Runners - 2 running
176
- // T200 - Mapping - Finished 1, 2
177
- // Q: [1, 2]
178
- // Iterate - Start 1, waiting 300 ms
179
- // Q: [2]
180
- // Mapping - Start 3, 4, waiting 200 ms
181
- // T400 - Mapping - Finished 3, 4
182
- // Q: [2, 3, 4]
183
- // Runners - 1 exits, 1 running
184
- // Mapping - Start 5, waiting 200 ms
185
- // T500 - Iterate - Start 2, waiting 300 ms
186
- // Q: [3, 4]
187
- // Runners - ***1 starts, 2 running after
188
- // Mapping - Start 6, waiting 200 ms
189
- // T600 - Mapping - Finished 5
190
- // Q: [3, 4, 5]
191
- // Runners - 1 exits, 1 running after
192
- // T700 - Mapping - Finished 6
193
- // Q: [3, 4, 5, 6]
194
- // Runners - 1 exits, 0 running after
195
- // T800 - Iterate - Finished 2
196
- // Iterate - Start 3, waiting 300 ms
197
- // Q: [4, 5, 6]
198
- // Runners - ***1 starts, 1 running after
199
- // Mapping - Start 7, waiting 200 ms
200
- // T1000 - Mapping - Finished 7
201
- // Q: [4, 5, 6, 7]
202
- // Runners - 1 exits, 0 running after
203
- // T1100 - Iterate - Finished 3
204
- // Iterate - Start 4, waiting 300 ms
205
- // Q: [5, 6, 7]
206
- // Runners - ***1 starts, 1 running after
207
- // T1101 - Mapping - Iteration Done
208
- // Runners - 1 exits, 0 running after
209
- // Q: [5, 6, 7]
210
- // T1400 - Iterate - Finished 4
211
- // Iterate - Start 5, waiting 300 ms
212
- // Q: [6, 7]
213
- // T1700 - Iterate - Finished 5
214
- // Iterate - Start 6, waiting 300 ms
215
- // Q: [7]
216
- // T2000 - Iterate - Finished 6
217
- // Iterate - Start 7, waiting 300 ms
218
- // Q: []
219
- // T2300 - Iterate - Finished 7
220
- // Q: []
221
- it('right number run in parallel - delay in reading', async () => {
222
- const startTime = Date.now();
223
- const max = 7;
224
- const mapDelayMs = 200;
225
- const readDelayMs = 300;
226
- const input = [
227
- { value: 1, ms: mapDelayMs },
228
- { value: 2, ms: mapDelayMs },
229
- { value: 3, ms: mapDelayMs },
230
- { value: 4, ms: mapDelayMs },
231
- { value: 5, ms: mapDelayMs },
232
- { value: 6, ms: mapDelayMs },
233
- { value: 7, ms: mapDelayMs },
234
- ];
235
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
236
- const prefetcher = new iterable_mapper_1.IterableMapper(input, mapper, { concurrency: 2, maxUnread: 4 });
237
- let lastSeen = 0;
238
- let loopCount = 0;
239
- for await (const item of prefetcher) {
240
- loopCount++;
241
- if (item > lastSeen) {
242
- lastSeen = item;
243
- }
244
- await sleep(readDelayMs);
245
- }
246
- expect(loopCount).toBe(max);
247
- expect(lastSeen).toBe(max);
248
- // 1st we wait 1 mapDelayMs for the 1st item to be ready
249
- // Then, as we iterate the mapped items, we wait readDelayMs for "processing"
250
- // This means that all the subsequent mapper calls never delay us because the
251
- // prefetch queue is always full.
252
- // Because of the prime numbers (2, 3, and 7) we know that we didn't accidentally
253
- // cause delays of a multiple of the wrong number.
254
- expect(Date.now() - startTime).toBeGreaterThan(mapDelayMs + max * readDelayMs);
255
- expect(sourceNextSpy).toHaveBeenCalledTimes(max + 1);
256
- expect(startAnotherRunnerSpy).toHaveBeenCalledTimes(3);
257
- });
258
- });
259
- describe('empty iterables', () => {
260
- it('empty sync iterable works - sync mapper', async () => {
261
- let total = 0;
262
- let callCount = 0;
263
- const prefetcher = new iterable_mapper_1.IterableMapper([], (value) => {
264
- callCount++;
265
- total += value;
266
- return total;
267
- });
268
- let lastTotal = 0;
269
- let loopCount = 0;
270
- for await (const item of prefetcher) {
271
- loopCount++;
272
- if (item > lastTotal) {
273
- lastTotal = item;
274
- }
275
- }
276
- expect(callCount).toBe(0);
277
- expect(loopCount).toBe(0);
278
- expect(total).toBe(0);
279
- expect(lastTotal).toBe(0);
280
- });
281
- it('empty sync iterable works - async mapper', async () => {
282
- let total = 0;
283
- let callCount = 0;
284
- const prefetcher = new iterable_mapper_1.IterableMapper([], async (value) => {
285
- await sleep(10);
286
- callCount++;
287
- total += value;
288
- return total;
289
- });
290
- let lastTotal = 0;
291
- let loopCount = 0;
292
- for await (const item of prefetcher) {
293
- loopCount++;
294
- if (item > lastTotal) {
295
- lastTotal = item;
296
- }
297
- }
298
- expect(callCount).toBe(0);
299
- expect(loopCount).toBe(0);
300
- expect(total).toBe(0);
301
- expect(lastTotal).toBe(0);
302
- });
303
- it('empty async iterable works - sync mapper', async () => {
304
- let total = 0;
305
- let callCount = 0;
306
- const prefetcher = new iterable_mapper_1.IterableMapper((() => {
307
- return {
308
- [Symbol.asyncIterator]() {
309
- return {
310
- async next() {
311
- await sleep(100);
312
- return { value: undefined, done: true };
313
- },
314
- };
315
- },
316
- };
317
- })(), (value) => {
318
- callCount++;
319
- total += value;
320
- return total;
321
- });
322
- let lastTotal = 0;
323
- let loopCount = 0;
324
- for await (const item of prefetcher) {
325
- loopCount++;
326
- if (item > lastTotal) {
327
- lastTotal = item;
328
- }
329
- }
330
- expect(callCount).toBe(0);
331
- expect(loopCount).toBe(0);
332
- expect(total).toBe(0);
333
- expect(lastTotal).toBe(0);
334
- });
335
- it('empty async iterable works - async mapper', async () => {
336
- let total = 0;
337
- let callCount = 0;
338
- const prefetcher = new iterable_mapper_1.IterableMapper((() => {
339
- return {
340
- [Symbol.asyncIterator]() {
341
- return {
342
- async next() {
343
- await sleep(100);
344
- return { value: undefined, done: true };
345
- },
346
- };
347
- },
348
- };
349
- })(), async (value) => {
350
- await sleep(10);
351
- callCount++;
352
- total += value;
353
- return total;
354
- });
355
- let lastTotal = 0;
356
- let loopCount = 0;
357
- for await (const item of prefetcher) {
358
- loopCount++;
359
- if (item > lastTotal) {
360
- lastTotal = item;
361
- }
362
- }
363
- expect(callCount).toBe(0);
364
- expect(loopCount).toBe(0);
365
- expect(total).toBe(0);
366
- expect(lastTotal).toBe(0);
367
- });
368
- });
369
- describe('order preservation tests', () => {
370
- it('preserves exact sequential order under high load with concurrency 1, maxUnread 8', async () => {
371
- const size = 1000;
372
- const input = Array.from({ length: size }, (_, i) => ({
373
- value: i + 1,
374
- ms: Math.floor(Math.random() * 20), // Random delay 0-19ms
375
- }));
376
- const mappedOrder = [];
377
- const iteratedOrder = [];
378
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async ({ value, ms }) => {
379
- await sleep(ms);
380
- mappedOrder.push(value);
381
- return value;
382
- }, { concurrency: 1, maxUnread: 8 });
383
- for await (const value of prefetcher) {
384
- iteratedOrder.push(value);
385
- // Add some random delay in consuming to create backpressure
386
- await sleep(Math.floor(Math.random() * 20));
387
- }
388
- // Verify exact sequential order preservation
389
- expect(iteratedOrder).toHaveLength(size);
390
- expect(mappedOrder).toHaveLength(size);
391
- // Verify that both mapped and iterated orders match the input sequence
392
- const expectedOrder = Array.from({ length: size }, (_, i) => i + 1);
393
- expect(mappedOrder).toEqual(expectedOrder);
394
- expect(iteratedOrder).toEqual(expectedOrder);
395
- }, 30000);
396
- });
397
- describe('concurrency 1, maxUnread 1', () => {
398
- const concurrency = 1;
399
- const maxUnread = 1;
400
- it('catches exception from mapper - 1st item - stopOnMapperError true', async () => {
401
- const input = [1, async () => sleep(300, 2), 3];
402
- const mappedValues = [];
403
- const iteratedValues = [];
404
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
405
- value = typeof value === 'function' ? await value() : value;
406
- mappedValues.push(value);
407
- if (value === 1) {
408
- await sleep(100);
409
- throw new Error('throw on 1st');
410
- }
411
- return value;
412
- }, { concurrency, maxUnread });
413
- let loopCount = 0;
414
- await expect(async () => {
415
- for await (const value of prefetcher) {
416
- loopCount++;
417
- iteratedValues.push(value);
418
- }
419
- }).rejects.toThrowError('throw on 1st');
420
- await sleep(500);
421
- expect(loopCount).toBe(0);
422
- expect(mappedValues).toEqual([1]);
423
- expect(iteratedValues).toEqual([]);
424
- });
425
- it('catches exception from mapper - 2nd item - stopOnMapperError true', async () => {
426
- const input = [1, async () => sleep(300, 2), 3];
427
- const mappedValues = [];
428
- const iteratedValues = [];
429
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
430
- value = typeof value === 'function' ? await value() : value;
431
- mappedValues.push(value);
432
- if (value === 2) {
433
- throw new Error('throw on 2nd');
434
- }
435
- await sleep(100);
436
- return value;
437
- }, { concurrency, maxUnread });
438
- let loopCount = 0;
439
- await expect(async () => {
440
- for await (const value of prefetcher) {
441
- loopCount++;
442
- iteratedValues.push(value);
443
- }
444
- }).rejects.toThrowError('throw on 2nd');
445
- await sleep(300);
446
- expect(loopCount).toBe(1);
447
- expect(mappedValues).toEqual([1, 2]);
448
- expect(iteratedValues).toEqual([1]);
449
- });
450
- it('catches exception from mapper - 1st item - stopOnMapperError false', async () => {
451
- const input = [1, async () => sleep(300, 2), 3];
452
- const mappedValues = [];
453
- const iteratedValues = [];
454
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
455
- value = typeof value === 'function' ? await value() : value;
456
- mappedValues.push(value);
457
- if (value === 1) {
458
- throw new Error('throw on 1st');
459
- }
460
- await sleep(100);
461
- return value;
462
- }, { concurrency: 1, stopOnMapperError: false });
463
- let loopCount = 0;
464
- await expect(async () => {
465
- for await (const value of prefetcher) {
466
- loopCount++;
467
- iteratedValues.push(value);
468
- }
469
- }).rejects.toThrowError('throw on 1st');
470
- await sleep(300);
471
- expect(loopCount).toBe(2);
472
- expect(mappedValues).toEqual([1, 2, 3]);
473
- expect(iteratedValues).toEqual([2, 3]);
474
- });
475
- it('catches exception from mapper - 2nd item - stopOnMapperError false', async () => {
476
- const input = [1, async () => sleep(300, 2), 3];
477
- const mappedValues = [];
478
- const iteratedValues = [];
479
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
480
- value = typeof value === 'function' ? await value() : value;
481
- mappedValues.push(value);
482
- if (value === 2) {
483
- throw new Error('throw on 2nd');
484
- }
485
- await sleep(100);
486
- return value;
487
- }, { concurrency, maxUnread, stopOnMapperError: false });
488
- let loopCount = 0;
489
- await expect(async () => {
490
- for await (const value of prefetcher) {
491
- loopCount++;
492
- iteratedValues.push(value);
493
- }
494
- }).rejects.toThrowError('throw on 2nd');
495
- await sleep(300);
496
- expect(loopCount).toBe(2);
497
- expect(mappedValues).toEqual([1, 2, 3]);
498
- expect(iteratedValues).toEqual([1, 3]);
499
- });
500
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError false', async () => {
501
- const input = new SleepIterator(10);
502
- const mappedValues = [];
503
- const iteratedValues = [];
504
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
505
- mappedValues.push(value);
506
- throw new Error('throw on each');
507
- }, { concurrency, maxUnread, stopOnMapperError: false });
508
- let loopCount = 0;
509
- await expect(async () => {
510
- for await (const value of prefetcher) {
511
- loopCount++;
512
- iteratedValues.push(value);
513
- }
514
- }).rejects.toThrowError('throw on each');
515
- expect(loopCount).toBe(0);
516
- expect(mappedValues).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
517
- expect(iteratedValues).toEqual([]);
518
- });
519
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError true', async () => {
520
- const input = new SleepIterator(10);
521
- const mappedValues = [];
522
- const iteratedValues = [];
523
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
524
- mappedValues.push(value);
525
- throw new Error('throw on each');
526
- }, { concurrency, maxUnread, stopOnMapperError: true });
527
- let loopCount = 0;
528
- await expect(async () => {
529
- for await (const value of prefetcher) {
530
- loopCount++;
531
- iteratedValues.push(value);
532
- }
533
- }).rejects.toThrowError('throw on each');
534
- expect(loopCount).toBe(0);
535
- expect(mappedValues).toEqual([1]);
536
- expect(iteratedValues).toEqual([]);
537
- });
538
- it('catches exception from source iterator - 1st item', async () => {
539
- const input = new ThrowingIterator(100, 0, 100);
540
- const mappedValues = [];
541
- const iteratedValues = [];
542
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
543
- mappedValues.push(value);
544
- await sleep(100);
545
- return value;
546
- }, { concurrency, maxUnread, stopOnMapperError: false });
547
- let loopCount = 0;
548
- await expect(async () => {
549
- for await (const value of prefetcher) {
550
- loopCount++;
551
- iteratedValues.push(value);
552
- }
553
- }).rejects.toThrowError('throwing on index 0');
554
- await sleep(500);
555
- expect(loopCount).toBe(0);
556
- expect(mappedValues).toEqual([]);
557
- expect(iteratedValues).toEqual([]);
558
- });
559
- // The 2nd iterable item throwing is distinct from the 1st when concurrency is 1 because
560
- // it means that the sourceNext() is invoked from next() or sourceNext() and not from
561
- // the constructor
562
- it('catches exception from source iterator - 2nd item', async () => {
563
- const input = new ThrowingIterator(100, 1, 100);
564
- const mappedValues = [];
565
- const iteratedValues = [];
566
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
567
- mappedValues.push(value);
568
- await sleep(100);
569
- return value;
570
- }, { concurrency, maxUnread, stopOnMapperError: false });
571
- let loopCount = 0;
572
- await expect(async () => {
573
- for await (const value of prefetcher) {
574
- loopCount++;
575
- iteratedValues.push(value);
576
- }
577
- }).rejects.toThrowError('throwing on index 1');
578
- await sleep(500);
579
- expect(loopCount).toBe(1);
580
- expect(mappedValues).toEqual([0]);
581
- expect(iteratedValues).toEqual([0]);
582
- });
583
- });
584
- describe('concurrency 2, maxUnread 4', () => {
585
- const concurrency = 2;
586
- const maxUnread = 4;
587
- it('catches exception from mapper - 1st item - stopOnMapperError true', async () => {
588
- const input = [1, async () => sleep(300, 2), 3];
589
- const mappedValues = [];
590
- const iteratedValues = [];
591
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
592
- value = typeof value === 'function' ? await value() : value;
593
- mappedValues.push(value);
594
- if (value === 1) {
595
- await sleep(100);
596
- throw new Error('throw on 1st');
597
- }
598
- return value;
599
- }, { concurrency, maxUnread });
600
- let loopCount = 0;
601
- await expect(async () => {
602
- for await (const value of prefetcher) {
603
- loopCount++;
604
- iteratedValues.push(value);
605
- }
606
- }).rejects.toThrowError('throw on 1st');
607
- await sleep(500);
608
- expect(loopCount).toBe(0);
609
- expect(mappedValues).toEqual([1, 2]);
610
- expect(iteratedValues).toEqual([]);
611
- });
612
- it('catches exception from mapper - 2nd item - stopOnMapperError true', async () => {
613
- const input = [1, async () => sleep(300, 2), 3];
614
- const mappedValues = [];
615
- const iteratedValues = [];
616
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
617
- value = typeof value === 'function' ? await value() : value;
618
- mappedValues.push(value);
619
- if (value === 2) {
620
- throw new Error('throw on 2nd');
621
- }
622
- await sleep(100);
623
- return value;
624
- }, { concurrency, maxUnread });
625
- let loopCount = 0;
626
- await expect(async () => {
627
- for await (const value of prefetcher) {
628
- loopCount++;
629
- iteratedValues.push(value);
630
- }
631
- }).rejects.toThrowError('throw on 2nd');
632
- await sleep(300);
633
- expect(loopCount).toBe(2);
634
- expect(mappedValues).toEqual([1, 3, 2]);
635
- expect(iteratedValues).toEqual([1, 3]);
636
- });
637
- it('catches exception from mapper - 1st item - stopOnMapperError false', async () => {
638
- const input = [1, async () => sleep(300, 2), 3];
639
- const mappedValues = [];
640
- const iteratedValues = [];
641
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
642
- value = typeof value === 'function' ? await value() : value;
643
- mappedValues.push(value);
644
- if (value === 1) {
645
- throw new Error('throw on 1st');
646
- }
647
- await sleep(100);
648
- return value;
649
- }, { concurrency, maxUnread, stopOnMapperError: false });
650
- let loopCount = 0;
651
- await expect(async () => {
652
- for await (const value of prefetcher) {
653
- loopCount++;
654
- iteratedValues.push(value);
655
- }
656
- }).rejects.toThrowError('throw on 1st');
657
- await sleep(300);
658
- expect(loopCount).toBe(2);
659
- expect(mappedValues).toEqual([1, 3, 2]);
660
- expect(iteratedValues).toEqual([3, 2]);
661
- });
662
- it('catches exception from mapper - 2nd item - stopOnMapperError false', async () => {
663
- const input = [1, async () => sleep(300, 2), 3];
664
- const mappedValues = [];
665
- const iteratedValues = [];
666
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
667
- value = typeof value === 'function' ? await value() : value;
668
- mappedValues.push(value);
669
- if (value === 2) {
670
- throw new Error('throw on 2nd');
671
- }
672
- await sleep(100);
673
- return value;
674
- }, { concurrency, maxUnread, stopOnMapperError: false });
675
- let loopCount = 0;
676
- await expect(async () => {
677
- for await (const value of prefetcher) {
678
- loopCount++;
679
- iteratedValues.push(value);
680
- }
681
- }).rejects.toThrowError('throw on 2nd');
682
- await sleep(300);
683
- expect(loopCount).toBe(2);
684
- expect(mappedValues).toEqual([1, 3, 2]);
685
- expect(iteratedValues).toEqual([1, 3]);
686
- });
687
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError false', async () => {
688
- const input = new SleepIterator(10);
689
- const mappedValues = [];
690
- const iteratedValues = [];
691
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
692
- mappedValues.push(value);
693
- throw new Error('throw on each');
694
- }, { concurrency, maxUnread, stopOnMapperError: false });
695
- let loopCount = 0;
696
- await expect(async () => {
697
- for await (const value of prefetcher) {
698
- loopCount++;
699
- iteratedValues.push(value);
700
- }
701
- }).rejects.toThrowError('throw on each');
702
- expect(loopCount).toBe(0);
703
- expect(mappedValues).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
704
- expect(iteratedValues).toEqual([]);
705
- });
706
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError true', async () => {
707
- const input = new SleepIterator(10);
708
- const mappedValues = [];
709
- const iteratedValues = [];
710
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
711
- mappedValues.push(value);
712
- throw new Error('throw on each');
713
- }, { concurrency, maxUnread, stopOnMapperError: true });
714
- let loopCount = 0;
715
- await expect(async () => {
716
- for await (const value of prefetcher) {
717
- loopCount++;
718
- iteratedValues.push(value);
719
- }
720
- }).rejects.toThrowError('throw on each');
721
- expect(loopCount).toBe(0);
722
- expect(mappedValues).toEqual([1]);
723
- expect(iteratedValues).toEqual([]);
724
- });
725
- it('catches exception from source iterator - 1st item', async () => {
726
- const input = new ThrowingIterator(100, 0, 100);
727
- const mappedValues = [];
728
- const iteratedValues = [];
729
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
730
- mappedValues.push(value);
731
- await sleep(100);
732
- return value;
733
- }, { concurrency, maxUnread, stopOnMapperError: false });
734
- let loopCount = 0;
735
- await expect(async () => {
736
- for await (const value of prefetcher) {
737
- loopCount++;
738
- iteratedValues.push(value);
739
- }
740
- }).rejects.toThrowError('throwing on index 0');
741
- await sleep(500);
742
- expect(loopCount).toBe(0);
743
- expect(mappedValues).toEqual([]);
744
- expect(iteratedValues).toEqual([]);
745
- });
746
- // The 2nd iterable item throwing is distinct from the 1st when concurrency is 1 because
747
- // it means that the sourceNext() is invoked from next() or sourceNext() and not from
748
- // the constructor
749
- it('catches exception from source iterator - 2nd item', async () => {
750
- const input = new ThrowingIterator(100, 1, 100);
751
- const mappedValues = [];
752
- const iteratedValues = [];
753
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
754
- mappedValues.push(value);
755
- await sleep(100);
756
- return value;
757
- }, { concurrency, maxUnread, stopOnMapperError: false });
758
- let loopCount = 0;
759
- await expect(async () => {
760
- for await (const value of prefetcher) {
761
- loopCount++;
762
- iteratedValues.push(value);
763
- }
764
- }).rejects.toThrowError('throwing on index 1');
765
- await sleep(500);
766
- expect(loopCount).toBe(1);
767
- expect(mappedValues).toEqual([0]);
768
- expect(iteratedValues).toEqual([0]);
769
- });
770
- });
771
- describe('concurrency Number.POSITIVE_INFINITY, maxUnread Number.POSITIVE_INFINITY', () => {
772
- const concurrency = Number.POSITIVE_INFINITY;
773
- const maxUnread = Number.POSITIVE_INFINITY;
774
- it('catches exception from mapper - 1st item - stopOnMapperError true', async () => {
775
- const input = [1, async () => sleep(300, 2), 3];
776
- const mappedValues = [];
777
- const iteratedValues = [];
778
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
779
- value = typeof value === 'function' ? await value() : value;
780
- mappedValues.push(value);
781
- if (value === 1) {
782
- await sleep(100);
783
- throw new Error('throw on 1st');
784
- }
785
- return value;
786
- }, { concurrency, maxUnread });
787
- let loopCount = 0;
788
- await expect(async () => {
789
- for await (const value of prefetcher) {
790
- loopCount++;
791
- iteratedValues.push(value);
792
- }
793
- }).rejects.toThrowError('throw on 1st');
794
- await sleep(500);
795
- expect(loopCount).toBe(1);
796
- expect(mappedValues).toEqual([1, 3, 2]);
797
- // NOTE: This seems wrong - Item 2 succeeds at being mapped, but after the 1st item throws.
798
- // Since item 2 was mapped you would think it should be iterated with the exception for
799
- // item 1 delayed until all items are read.
800
- // However, since the mapper for 2 could throw, this could lead to 2 exceptions wanting to
801
- // throw if we waited like this... so we would have to throw an AggregateError... and
802
- // that just leads to guidance that the caller should catch exceptions in their mapper
803
- // and return them as part of their NewElement object type.
804
- // If a consumer catches and returns the exception then re-throws in their `for await`
805
- // then this behavior would match their results exactly (the 1st item would cause the iteration
806
- // loop to break out).
807
- expect(iteratedValues).toEqual([3]);
808
- });
809
- it('catches exception from mapper - 2nd item - stopOnMapperError true', async () => {
810
- const input = [1, async () => sleep(300, 2), 3];
811
- const mappedValues = [];
812
- const iteratedValues = [];
813
- const prefetcher = new iterable_mapper_1.IterableMapper(input, async (value) => {
814
- value = typeof value === 'function' ? await value() : value;
815
- mappedValues.push(value);
816
- if (value === 2) {
817
- throw new Error('throw on 2nd');
818
- }
819
- await sleep(100);
820
- return value;
821
- }, { concurrency, maxUnread });
822
- let loopCount = 0;
823
- await expect(async () => {
824
- for await (const value of prefetcher) {
825
- loopCount++;
826
- iteratedValues.push(value);
827
- }
828
- }).rejects.toThrowError('throw on 2nd');
829
- await sleep(300);
830
- expect(loopCount).toBe(2);
831
- expect(mappedValues).toEqual([1, 3, 2]);
832
- expect(iteratedValues).toEqual([1, 3]);
833
- });
834
- it('async mapper works', async () => {
835
- const max = 100;
836
- const iterator = new SleepIterator(max);
837
- let total = 0;
838
- let callCount = 0;
839
- const prefetcher = new iterable_mapper_1.IterableMapper(iterator, async (value) => {
840
- callCount++;
841
- total += value;
842
- await sleep(10 * (callCount % 6));
843
- return total;
844
- }, { concurrency, maxUnread });
845
- let lastTotal = 0;
846
- let loopCount = 0;
847
- for await (const item of prefetcher) {
848
- loopCount++;
849
- if (item > lastTotal) {
850
- lastTotal = item;
851
- }
852
- }
853
- expect(callCount).toBe(max);
854
- expect(loopCount).toBe(max);
855
- expect(total).toBe((max * (max + 1)) / 2);
856
- expect(lastTotal).toBe((max * (max + 1)) / 2);
857
- });
858
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError true', async () => {
859
- const input = new SleepIterator(10);
860
- const mappedValues = [];
861
- const iteratedValues = [];
862
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
863
- mappedValues.push(value);
864
- throw new Error('throw on each');
865
- }, { concurrency, maxUnread, stopOnMapperError: true });
866
- let loopCount = 0;
867
- await expect(async () => {
868
- for await (const value of prefetcher) {
869
- loopCount++;
870
- iteratedValues.push(value);
871
- }
872
- }).rejects.toThrowError('throw on each');
873
- expect(loopCount).toBe(0);
874
- expect(mappedValues).toEqual([1]);
875
- expect(iteratedValues).toEqual([]);
876
- });
877
- it('catches exception from mapper - 10 of 10 items - stopOnMapperError false', async () => {
878
- const input = new SleepIterator(10);
879
- const mappedValues = [];
880
- const iteratedValues = [];
881
- const prefetcher = new iterable_mapper_1.IterableMapper(input, (value) => {
882
- mappedValues.push(value);
883
- throw new Error('throw on each');
884
- }, { concurrency, maxUnread, stopOnMapperError: false });
885
- let loopCount = 0;
886
- await expect(async () => {
887
- for await (const value of prefetcher) {
888
- loopCount++;
889
- iteratedValues.push(value);
890
- }
891
- }).rejects.toThrowError('throw on each');
892
- expect(loopCount).toBe(0);
893
- expect(mappedValues).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
894
- expect(iteratedValues).toEqual([]);
895
- });
896
- });
897
- });
898
- //# sourceMappingURL=iterable-mapper.test.js.map