dynamic-throttled-queue 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.idea/dynamic-throttled-queue.iml +12 -0
- package/.idea/encodings.xml +6 -0
- package/.idea/git_toolbox_blame.xml +6 -0
- package/.idea/git_toolbox_prj.xml +15 -0
- package/.idea/inspectionProfiles/Project_Default.xml +7 -0
- package/.idea/jsLinters/eslint.xml +6 -0
- package/.idea/misc.xml +79 -0
- package/.idea/modules.xml +8 -0
- package/.idea/prettier.xml +7 -0
- package/.idea/vcs.xml +6 -0
- package/README.md +5 -5
- package/dynamic-throttled-queue.js +7 -2
- package/package.json +1 -1
- package/test/tests.js +12 -8
- package/gulpfile.js +0 -13
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<module type="WEB_MODULE" version="4">
|
|
3
|
+
<component name="NewModuleRootManager">
|
|
4
|
+
<content url="file://$MODULE_DIR$">
|
|
5
|
+
<excludeFolder url="file://$MODULE_DIR$/.tmp" />
|
|
6
|
+
<excludeFolder url="file://$MODULE_DIR$/temp" />
|
|
7
|
+
<excludeFolder url="file://$MODULE_DIR$/tmp" />
|
|
8
|
+
</content>
|
|
9
|
+
<orderEntry type="inheritedJdk" />
|
|
10
|
+
<orderEntry type="sourceFolder" forTests="false" />
|
|
11
|
+
</component>
|
|
12
|
+
</module>
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project version="4">
|
|
3
|
+
<component name="GitToolBoxProjectSettings">
|
|
4
|
+
<option name="commitMessageIssueKeyValidationOverride">
|
|
5
|
+
<BoolValueOverride>
|
|
6
|
+
<option name="enabled" value="true" />
|
|
7
|
+
</BoolValueOverride>
|
|
8
|
+
</option>
|
|
9
|
+
<option name="commitMessageValidationEnabledOverride">
|
|
10
|
+
<BoolValueOverride>
|
|
11
|
+
<option name="enabled" value="true" />
|
|
12
|
+
</BoolValueOverride>
|
|
13
|
+
</option>
|
|
14
|
+
</component>
|
|
15
|
+
</project>
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
<component name="InspectionProjectProfileManager">
|
|
2
|
+
<profile version="1.0">
|
|
3
|
+
<option name="myName" value="Project Default" />
|
|
4
|
+
<inspection_tool class="CssBrowserCompatibilityForProperties" enabled="true" level="WARNING" enabled_by_default="true" />
|
|
5
|
+
<inspection_tool class="t" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
|
|
6
|
+
</profile>
|
|
7
|
+
</component>
|
package/.idea/misc.xml
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project version="4">
|
|
3
|
+
<component name="JavaScriptSettings">
|
|
4
|
+
<option name="languageLevel" value="ES6" />
|
|
5
|
+
</component>
|
|
6
|
+
<component name="ProjectInspectionProfilesVisibleTreeState">
|
|
7
|
+
<entry key="Project Default">
|
|
8
|
+
<profile-state>
|
|
9
|
+
<expanded-state>
|
|
10
|
+
<State>
|
|
11
|
+
<id />
|
|
12
|
+
</State>
|
|
13
|
+
<State>
|
|
14
|
+
<id>Code style issuesJavaScript</id>
|
|
15
|
+
</State>
|
|
16
|
+
<State>
|
|
17
|
+
<id>CoffeeScript</id>
|
|
18
|
+
</State>
|
|
19
|
+
<State>
|
|
20
|
+
<id>Control flow issuesJavaScript</id>
|
|
21
|
+
</State>
|
|
22
|
+
<State>
|
|
23
|
+
<id>ECMAScript 6 migration aidsJavaScript</id>
|
|
24
|
+
</State>
|
|
25
|
+
<State>
|
|
26
|
+
<id>General</id>
|
|
27
|
+
</State>
|
|
28
|
+
<State>
|
|
29
|
+
<id>GeneralJavaScript</id>
|
|
30
|
+
</State>
|
|
31
|
+
<State>
|
|
32
|
+
<id>JavaScript</id>
|
|
33
|
+
</State>
|
|
34
|
+
<State>
|
|
35
|
+
<id>JavaScript validity issuesJavaScript</id>
|
|
36
|
+
</State>
|
|
37
|
+
<State>
|
|
38
|
+
<id>Meteor</id>
|
|
39
|
+
</State>
|
|
40
|
+
<State>
|
|
41
|
+
<id>Node.jsJavaScript</id>
|
|
42
|
+
</State>
|
|
43
|
+
<State>
|
|
44
|
+
<id>Perl5</id>
|
|
45
|
+
</State>
|
|
46
|
+
<State>
|
|
47
|
+
<id>Perl5 POD</id>
|
|
48
|
+
</State>
|
|
49
|
+
<State>
|
|
50
|
+
<id>Potentially confusing code constructsJavaScript</id>
|
|
51
|
+
</State>
|
|
52
|
+
<State>
|
|
53
|
+
<id>Probable bugsCoffeeScript</id>
|
|
54
|
+
</State>
|
|
55
|
+
<State>
|
|
56
|
+
<id>Probable bugsJavaScript</id>
|
|
57
|
+
</State>
|
|
58
|
+
<State>
|
|
59
|
+
<id>Template Toolkit 2</id>
|
|
60
|
+
</State>
|
|
61
|
+
<State>
|
|
62
|
+
<id>Vue</id>
|
|
63
|
+
</State>
|
|
64
|
+
<State>
|
|
65
|
+
<id>XPath</id>
|
|
66
|
+
</State>
|
|
67
|
+
<State>
|
|
68
|
+
<id>XSLT</id>
|
|
69
|
+
</State>
|
|
70
|
+
</expanded-state>
|
|
71
|
+
<selected-state>
|
|
72
|
+
<State>
|
|
73
|
+
<id>AngularJS</id>
|
|
74
|
+
</State>
|
|
75
|
+
</selected-state>
|
|
76
|
+
</profile-state>
|
|
77
|
+
</entry>
|
|
78
|
+
</component>
|
|
79
|
+
</project>
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project version="4">
|
|
3
|
+
<component name="ProjectModuleManager">
|
|
4
|
+
<modules>
|
|
5
|
+
<module fileurl="file://$PROJECT_DIR$/.idea/dynamic-throttled-queue.iml" filepath="$PROJECT_DIR$/.idea/dynamic-throttled-queue.iml" />
|
|
6
|
+
</modules>
|
|
7
|
+
</component>
|
|
8
|
+
</project>
|
package/.idea/vcs.xml
ADDED
package/README.md
CHANGED
|
@@ -78,7 +78,7 @@ Wherever the `throttle` instance is used, your action will be placed into the sa
|
|
|
78
78
|
and be subject to the same rate limits.
|
|
79
79
|
|
|
80
80
|
```js
|
|
81
|
-
const throttledQueue = require('throttled-queue');
|
|
81
|
+
const throttledQueue = require('dynamic-throttled-queue');
|
|
82
82
|
const throttle = throttledQueue({min_rpi:1, interval:60 * 1000}); // at most make 1 request every minute.
|
|
83
83
|
|
|
84
84
|
for (let x = 0; x < 50; x++) {
|
|
@@ -100,7 +100,7 @@ for (let y = 0; y < 50; y++) {
|
|
|
100
100
|
By specifying a number higher than 1 for the min_rpi, and setting `evenly_spaced: false` you can dequeue multiple actions within the given interval:
|
|
101
101
|
|
|
102
102
|
```js
|
|
103
|
-
var throttledQueue = require('throttled-queue');
|
|
103
|
+
var throttledQueue = require('dynamic-throttled-queue');
|
|
104
104
|
var throttle = throttledQueue({min_rpi:10, interval:1000, evenly_spaced: true}); // at most make 10 requests every second.
|
|
105
105
|
|
|
106
106
|
for (let x = 0; x < 100; x++) {
|
|
@@ -115,7 +115,7 @@ for (let x = 0; x < 100; x++) {
|
|
|
115
115
|
By default your actions are evenly distributed over the interval `evenly_spaced: true`:
|
|
116
116
|
|
|
117
117
|
```js
|
|
118
|
-
const throttledQueue = require('throttled-queue');
|
|
118
|
+
const throttledQueue = require('dynamic-throttled-queue');
|
|
119
119
|
const throttle = throttledQueue({min_rpi: 10, interval: 1000, evenly_spaced:true}); // at most make 10 requests every second, but evenly spaced.
|
|
120
120
|
|
|
121
121
|
for (let x = 0; x < 100; x++) {
|
|
@@ -133,7 +133,7 @@ By suppling a `min_rpi` & `max_rpi` value to the options object, you will be abl
|
|
|
133
133
|
The second part of this is the `errors_per_second` option. This is set by default to 5 errors per second. Every X seconds, a check is made to see how many errors we have seen (through the use of `return false`) and if we see X or more, then the current requests per interval will decrease until we hit the `min_rpi` value. If between 0 - X errors are seen then we keep with the current requests per interval as is. Finally if there are 0 errors in the last check period then we will increase the current requests per interval until we reach `max_rpi`.
|
|
134
134
|
|
|
135
135
|
```js
|
|
136
|
-
const throttledQueue = require('throttled-queue');
|
|
136
|
+
const throttledQueue = require('dynamic-throttled-queue');
|
|
137
137
|
const throttle = throttledQueue({min_rpi: 1, max_rpi:5, interval: 1000}); // at most make 5 requests every second.
|
|
138
138
|
|
|
139
139
|
for (let x = 0; x < 100; x++) {
|
|
@@ -148,7 +148,7 @@ for (let x = 0; x < 100; x++) {
|
|
|
148
148
|
By suppling `backoff:true` in the options, every time we hit the `errors_per_second` mark, we will backoff from the next batch of calls for 1 inteveral
|
|
149
149
|
|
|
150
150
|
```js
|
|
151
|
-
const throttledQueue = require('throttled-queue');
|
|
151
|
+
const throttledQueue = require('dynamic-throttled-queue');
|
|
152
152
|
const throttle = throttledQueue({min_rpi: 10, interval: 1000, backoff:true, errors_per_second:2});
|
|
153
153
|
// at most make 10 requests every second, if more than 2 errors per second, then back off for 1 full interval of 1 second.
|
|
154
154
|
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
* @param errors_per_second [number=5] - Number of errors per second before deciding to either increase or decrease the current rpi
|
|
34
34
|
* @param back_off [boolean=true] - If true and we hit the errors_per_interval watermark, we will back off for 1 interval
|
|
35
35
|
* @param retry [number=0] - If greater than 0, any failed callbacks, will be put back onto the queue to retry upto X times
|
|
36
|
+
* @param cb [Function] - Sends back the current rate
|
|
36
37
|
*
|
|
37
38
|
* @returns {Function}
|
|
38
39
|
*/
|
|
@@ -45,7 +46,8 @@
|
|
|
45
46
|
evenly_spaced = true,
|
|
46
47
|
errors_per_second = 5,
|
|
47
48
|
back_off = false,
|
|
48
|
-
retry = 0
|
|
49
|
+
retry = 0,
|
|
50
|
+
cb = (rate) => undefined
|
|
49
51
|
} = options;
|
|
50
52
|
|
|
51
53
|
debug = typeOf(options.debug) !== "undefined" ? options.debug === true : false;
|
|
@@ -156,6 +158,7 @@
|
|
|
156
158
|
if (error_count >= errors_per_second) {
|
|
157
159
|
debugFn('Decreasing rate limit');
|
|
158
160
|
current_rpi = Math.max(min_rpi, current_rpi - 1);
|
|
161
|
+
cb && cb(current_rpi);
|
|
159
162
|
//decrease dyn count by 1
|
|
160
163
|
if (evenly_spaced) {
|
|
161
164
|
dyn_interval = interval / current_rpi;
|
|
@@ -180,6 +183,7 @@
|
|
|
180
183
|
} else if (!bSkippedLast && error_count === 0 && queue.length > 0) {
|
|
181
184
|
debugFn('Increasing rate limit');
|
|
182
185
|
current_rpi = Math.min(max_rpi, current_rpi + 1);
|
|
186
|
+
cb && cb(current_rpi);
|
|
183
187
|
//increase dyn count by 1
|
|
184
188
|
if (evenly_spaced) {
|
|
185
189
|
dyn_interval = interval / current_rpi;
|
|
@@ -188,6 +192,7 @@
|
|
|
188
192
|
}
|
|
189
193
|
}
|
|
190
194
|
error_count = 0;
|
|
195
|
+
cb && cb(current_rpi);
|
|
191
196
|
debugFn(`current_rpi: ${current_rpi}`);
|
|
192
197
|
dyn_timeout = setTimeout(alterDYN_rpi, interval);
|
|
193
198
|
};
|
|
@@ -234,4 +239,4 @@
|
|
|
234
239
|
root.throttledQueue = throttledQueue;
|
|
235
240
|
}
|
|
236
241
|
|
|
237
|
-
}).call(this);
|
|
242
|
+
}).call(this);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dynamic-throttled-queue",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"description": "Forked from shaunpersad/throttled-queue. Dynamically throttles arbitrary code to execute a minimum/maximum number of times per interval. Best for making throttled API requests.",
|
|
5
5
|
"main": "dynamic-throttled-queue.js",
|
|
6
6
|
"directories": {
|
package/test/tests.js
CHANGED
|
@@ -4,13 +4,17 @@ function calculateRPMS(num_requests, time_started) {
|
|
|
4
4
|
|
|
5
5
|
}
|
|
6
6
|
|
|
7
|
+
function cb(rate) {
|
|
8
|
+
console.log('Rate:', rate);
|
|
9
|
+
}
|
|
10
|
+
|
|
7
11
|
describe('dynamic-throttled-queue', function() {
|
|
8
12
|
|
|
9
13
|
it('should queue all callbacks', function(done) {
|
|
10
14
|
|
|
11
15
|
var requests_per_interval = 1;
|
|
12
16
|
var interval = 200;
|
|
13
|
-
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval});
|
|
17
|
+
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval, cb});
|
|
14
18
|
var num_requests = 0;
|
|
15
19
|
var request_limit = 100;
|
|
16
20
|
for (var x = 0; x < request_limit; x++) {
|
|
@@ -31,7 +35,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
31
35
|
|
|
32
36
|
var requests_per_interval = 1;
|
|
33
37
|
var interval = 200;
|
|
34
|
-
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval});
|
|
38
|
+
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval, cb});
|
|
35
39
|
var last_executed = Date.now();
|
|
36
40
|
|
|
37
41
|
var num_requests = 0;
|
|
@@ -61,7 +65,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
61
65
|
|
|
62
66
|
var requests_per_interval = 1;
|
|
63
67
|
var interval = 500;
|
|
64
|
-
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval});
|
|
68
|
+
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval, cb});
|
|
65
69
|
var time_started = Date.now();
|
|
66
70
|
var max_rpms = requests_per_interval / interval;
|
|
67
71
|
|
|
@@ -89,7 +93,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
89
93
|
|
|
90
94
|
var requests_per_interval = 3;
|
|
91
95
|
var interval = 1000;
|
|
92
|
-
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval});
|
|
96
|
+
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval, cb});
|
|
93
97
|
var time_started = Date.now();
|
|
94
98
|
var max_rpms = requests_per_interval / interval;
|
|
95
99
|
|
|
@@ -117,7 +121,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
117
121
|
|
|
118
122
|
var requests_per_interval = 3;
|
|
119
123
|
var interval = 1000;
|
|
120
|
-
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval}, true);
|
|
124
|
+
var throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval, cb}, true);
|
|
121
125
|
var time_started = Date.now();
|
|
122
126
|
var max_rpms = requests_per_interval / interval;
|
|
123
127
|
|
|
@@ -145,7 +149,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
145
149
|
const requests_per_interval = 5;
|
|
146
150
|
const interval = 200;
|
|
147
151
|
const retry = 2;
|
|
148
|
-
const throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval,evenly_spaced:false, retry}, true);
|
|
152
|
+
const throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval,evenly_spaced:false, retry, cb}, true);
|
|
149
153
|
|
|
150
154
|
let num_requests = 0;
|
|
151
155
|
const request_limit = 5;
|
|
@@ -173,7 +177,7 @@ describe('dynamic-throttled-queue', function() {
|
|
|
173
177
|
const requests_per_interval = 5;
|
|
174
178
|
const interval = 200;
|
|
175
179
|
const retry = 2;
|
|
176
|
-
const throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval,evenly_spaced:false, retry}, true);
|
|
180
|
+
const throttle = throttledQueue({min_rpi:requests_per_interval, interval:interval,evenly_spaced:false, retry, cb}, true);
|
|
177
181
|
|
|
178
182
|
let num_requests = 0;
|
|
179
183
|
const request_limit = 5;
|
|
@@ -196,4 +200,4 @@ describe('dynamic-throttled-queue', function() {
|
|
|
196
200
|
});
|
|
197
201
|
}
|
|
198
202
|
});
|
|
199
|
-
});
|
|
203
|
+
});
|
package/gulpfile.js
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
var gulp = require('gulp');
|
|
2
|
-
|
|
3
|
-
var uglify = require('gulp-uglify');
|
|
4
|
-
var rename = require('gulp-rename');
|
|
5
|
-
babel = require('gulp-babel');
|
|
6
|
-
|
|
7
|
-
gulp.task('default', function() {
|
|
8
|
-
return gulp.src('./dynamic-throttled-queue.js')
|
|
9
|
-
.pipe(rename('dynamic-throttled-queue.min.js'))
|
|
10
|
-
.pipe(babel({presets: ['@babel/env']}))
|
|
11
|
-
.pipe(uglify())
|
|
12
|
-
.pipe(gulp.dest('./'));
|
|
13
|
-
});
|