dynamic-throttled-queue 2.1.0 → 2.1.1-rc.bcc09d3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -38,13 +38,15 @@ Set `concurrency` to bound callbacks that are still awaiting asynchronous comple
|
|
|
38
38
|
| `max_rpi` | `number` | `min_rpi` | Maximum requests per interval |
|
|
39
39
|
| `interval` | `number` | *required* | Milliseconds between each batch of requests |
|
|
40
40
|
| `evenly_spaced` | `boolean` | `true` | Distribute requests evenly throughout the interval |
|
|
41
|
-
| `errors_per_interval` | `number` | `5` |
|
|
41
|
+
| `errors_per_interval` | `number` | `5` | Positive integer error threshold per interval before adjusting rate |
|
|
42
42
|
| `back_off` | `boolean` | `false` | Back off for 1 full interval when error threshold is hit |
|
|
43
|
-
| `retry` | `number` | `0` |
|
|
43
|
+
| `retry` | `number` | `0` | Non-negative integer number of times to retry failed callbacks |
|
|
44
44
|
| `concurrency` | `number` | — | Maximum callbacks awaiting asynchronous completion; omit for no limit |
|
|
45
|
-
| `compact_threshold` | `number` | `512` |
|
|
45
|
+
| `compact_threshold` | `number` | `512` | Non-negative integer minimum dead slots before internal queue compaction triggers; `0` compacts at the earliest eligible point |
|
|
46
46
|
| `onRateChange` | `(rate: number) => void` | — | Called when the current rate changes |
|
|
47
47
|
|
|
48
|
+
`retry`, `errors_per_interval`, and `compact_threshold` reject fractional and non-finite values. `errors_per_interval` must be at least `1`; `retry` and `compact_threshold` may be `0`.
|
|
49
|
+
|
|
48
50
|
## Handle API
|
|
49
51
|
|
|
50
52
|
`createThrottledQueue` returns a function with additional properties:
|
|
@@ -63,6 +65,8 @@ console.log(throttle.pending); // number of queued callbacks
|
|
|
63
65
|
throttle.stop(); // halt processing
|
|
64
66
|
```
|
|
65
67
|
|
|
68
|
+
`stop()` clears scheduler timers and retains callbacks that have not started. It cannot cancel an active asynchronous callback; if that callback later succeeds, returns `false`, or rejects, its settlement does not restart scheduling. A retry created by a failed active callback is retained with the pending work. Enqueueing another callback resumes the queue.
|
|
69
|
+
|
|
66
70
|
## Examples
|
|
67
71
|
|
|
68
72
|
### Basic
|
|
@@ -5,13 +5,14 @@ export type ThrottleOptions = {
|
|
|
5
5
|
interval: number;
|
|
6
6
|
max_rpi?: number;
|
|
7
7
|
evenly_spaced?: boolean;
|
|
8
|
-
/**
|
|
8
|
+
/** Positive integer error threshold per interval before rate decrease. Default 5. */
|
|
9
9
|
errors_per_interval?: number;
|
|
10
10
|
back_off?: boolean;
|
|
11
|
+
/** Non-negative integer retries for each failed callback. Default 0. */
|
|
11
12
|
retry?: number;
|
|
12
13
|
/** Maximum number of callbacks awaiting asynchronous settlement. Omit for no limit. */
|
|
13
14
|
concurrency?: number;
|
|
14
|
-
/**
|
|
15
|
+
/** Non-negative integer dead slots before queue compaction triggers. Default 512. */
|
|
15
16
|
compact_threshold?: number;
|
|
16
17
|
onRateChange?: (rate: number) => void;
|
|
17
18
|
};
|
|
@@ -13,6 +13,15 @@ export function createThrottledQueue(options) {
|
|
|
13
13
|
if (concurrency !== undefined && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
14
14
|
throw new Error("concurrency must be a positive integer");
|
|
15
15
|
}
|
|
16
|
+
if (!Number.isInteger(errors_per_interval) || errors_per_interval < 1) {
|
|
17
|
+
throw new Error("errors_per_interval must be a positive integer");
|
|
18
|
+
}
|
|
19
|
+
if (!Number.isInteger(retry) || retry < 0) {
|
|
20
|
+
throw new Error("retry must be a non-negative integer");
|
|
21
|
+
}
|
|
22
|
+
if (!Number.isInteger(compact_threshold) || compact_threshold < 0) {
|
|
23
|
+
throw new Error("compact_threshold must be a non-negative integer");
|
|
24
|
+
}
|
|
16
25
|
let current_rpi = Math.ceil((max_rpi + min_rpi) / 2);
|
|
17
26
|
let dyn_interval = evenly_spaced ? interval / current_rpi : interval;
|
|
18
27
|
let dyn_requests_per_interval = evenly_spaced ? 1 : current_rpi;
|
|
@@ -23,11 +32,12 @@ export function createThrottledQueue(options) {
|
|
|
23
32
|
let timeout;
|
|
24
33
|
let dynTimeout;
|
|
25
34
|
let active_count = 0;
|
|
35
|
+
let isStopped = false;
|
|
26
36
|
const max_concurrency = concurrency ?? Infinity;
|
|
27
37
|
const queue = [];
|
|
28
38
|
let head = 0;
|
|
29
39
|
/** Halts timers. Retains unprocessed queue items; next enqueue resumes from where it left off. */
|
|
30
|
-
function
|
|
40
|
+
function halt() {
|
|
31
41
|
isRunning = false;
|
|
32
42
|
skippedLast = false;
|
|
33
43
|
clearTimeout(timeout);
|
|
@@ -39,12 +49,16 @@ export function createThrottledQueue(options) {
|
|
|
39
49
|
head = 0;
|
|
40
50
|
}
|
|
41
51
|
}
|
|
52
|
+
function stop() {
|
|
53
|
+
isStopped = true;
|
|
54
|
+
halt();
|
|
55
|
+
}
|
|
42
56
|
function handleResult(item, result) {
|
|
43
57
|
if (result === false) {
|
|
44
58
|
error_count++;
|
|
45
59
|
if (item.retries > 0) {
|
|
46
60
|
queue.push({ fn: item.fn, retries: item.retries - 1 });
|
|
47
|
-
if (!isRunning && queue.length > head) {
|
|
61
|
+
if (!isRunning && !isStopped && queue.length > head) {
|
|
48
62
|
start();
|
|
49
63
|
}
|
|
50
64
|
}
|
|
@@ -96,7 +110,7 @@ export function createThrottledQueue(options) {
|
|
|
96
110
|
head = 0;
|
|
97
111
|
}
|
|
98
112
|
if (head >= queue.length) {
|
|
99
|
-
|
|
113
|
+
halt();
|
|
100
114
|
return;
|
|
101
115
|
}
|
|
102
116
|
if (active_count >= max_concurrency) {
|
|
@@ -150,6 +164,7 @@ export function createThrottledQueue(options) {
|
|
|
150
164
|
}
|
|
151
165
|
function enqueue(callback) {
|
|
152
166
|
queue.push({ fn: callback, retries: retry });
|
|
167
|
+
isStopped = false;
|
|
153
168
|
if (!isRunning) {
|
|
154
169
|
start();
|
|
155
170
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-throttled-queue.js","sourceRoot":"","sources":["../src/dynamic-throttled-queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dynamic-throttled-queue.js","sourceRoot":"","sources":["../src/dynamic-throttled-queue.ts"],"names":[],"mappings":"AA6BA,MAAM,UAAU,oBAAoB,CAAC,OAAwB;IAC3D,MAAM,EACJ,OAAO,EACP,QAAQ,EACR,OAAO,GAAG,OAAO,EACjB,aAAa,GAAG,IAAI,EACpB,QAAQ,GAAG,KAAK,EAChB,KAAK,GAAG,CAAC,EACT,WAAW,EACX,iBAAiB,GAAG,GAAG,EACvB,YAAY,GACb,GAAG,OAAO,CAAC;IAEZ,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,CAAC,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrE,IAAI,yBAAyB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAChE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,OAAkD,CAAC;IACvD,IAAI,UAAqD,CAAC;IAC1D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,eAAe,GAAG,WAAW,IAAI,QAAQ,CAAC;IAEhD,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,kGAAkG;IAClG,SAAS,IAAI;QACX,SAAS,GAAG,KAAK,CAAC;QAClB,WAAW,GAAG,KAAK,CAAC;QACpB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,OAAO,GAAG,SAAS,CAAC;QACpB,YAAY,CAAC,UAAU,CAAC,CAAC;QACzB,UAAU,GAAG,SAAS,CAAC;QACvB,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACjB,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,SAAS,IAAI;QACX,SAAS,GAAG,IAAI,CAAC;QACjB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,SAAS,YAAY,CAAC,IAAe,EAAE,MAAsB;QAC3D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACrB,WAAW,EAAE,CAAC;YACd,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;oBACpD,KAAK,EAAE,CAAC;gBACV,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS,gBAAgB,CAAC,IAAe,EAAE,MAAsB,EAAE,MAAM,GAAG,KAAK;QAC/E,YAAY,EAAE,CAAC;QACf,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAE3B,gFAAgF;QAChF,IAAI,MAAM,IAAI,SAAS,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC/D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,SAAS,OAAO;QACd,MAAM,SAAS,GAAG,WAAW,GAAG,YAAY,CAAC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,SAAS,GAAG,GAAG,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,4FAA4F;QAC5F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,yBAAyB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,IAAI,GAAG,GAAG,IAAI,YAAY,GAAG,eAAe,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAE,CAAC;YAC5B,YAAY,EAAE,CAAC;YACf,IAAI,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;gBACpB,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,CAAC;YAED,IAAI,MAAoC,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACrB,CAAC;YACD,MAAM,CAAC;gBACL,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YACD,IAAI,MAAM,YAAY,OAAO,EAAE,CAAC;gBAC9B,KAAK,MAAM,CAAC,IAAI,CACd,KAAK,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,EAC5C,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAC1C,CAAC;YACJ,CAAC;iBACI,CAAC;gBACJ,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,mFAAmF;QACnF,IAAI,IAAI,GAAG,iBAAiB,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACtB,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;QAED,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,IAAI,YAAY,IAAI,eAAe,EAAE,CAAC;YACpC,OAAO;QACT,CAAC;QACD,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9C,CAAC;IACD,SAAS,SAAS,CAAC,MAAc;QAC/B,IAAI,MAAM,KAAK,WAAW;YAAE,OAAO;QACnC,WAAW,GAAG,MAAM,CAAC;QACrB,YAAY,EAAE,CAAC,WAAW,CAAC,CAAC;QAC5B,IAAI,aAAa,EAAE,CAAC;YAClB,YAAY,GAAG,QAAQ,GAAG,WAAW,CAAC;QACxC,CAAC;aACI,CAAC;YACJ,yBAAyB,GAAG,WAAW,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,kHAAkH;IAClH,SAAS,UAAU;QACjB,UAAU,GAAG,SAAS,CAAC;QACvB,MAAM,UAAU,GAAG,WAAW,CAAC;QAC/B,WAAW,GAAG,KAAK,CAAC;QAEpB,IAAI,WAAW,IAAI,mBAAmB,EAAE,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9C,IAAI,SAAS,IAAI,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;gBACzC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,YAAY,GAAG,QAAQ,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;aACI,IAAI,CAAC,UAAU,IAAI,WAAW,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YACjE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,WAAW,GAAG,CAAC,CAAC;QAEhB,IAAI,SAAS,EAAE,CAAC;YACd,UAAU,GAAG,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,SAAS,KAAK;QACZ,IAAI,WAAW;YAAE,OAAO;QACxB,SAAS,GAAG,IAAI,CAAC;QACjB,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAC5C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,SAAS,OAAO,CAAC,QAA0B;QACzC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,SAAS,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IACpB,uEAAuE;IACvE,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;IAE9E,OAAO,OAAyB,CAAC;AACnC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dynamic-throttled-queue",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.1-rc.bcc09d3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Dynamically throttles arbitrary code to execute between a minimum and maximum number of times per interval. Best for making throttled API requests.",
|
|
6
6
|
"files": [
|