@crawlee/core 3.16.1-beta.5 → 3.16.1-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +10 -0
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/client_load_signal.d.ts +26 -0
- package/autoscaling/client_load_signal.d.ts.map +1 -0
- package/autoscaling/client_load_signal.js +40 -0
- package/autoscaling/client_load_signal.js.map +1 -0
- package/autoscaling/cpu_load_signal.d.ts +29 -0
- package/autoscaling/cpu_load_signal.d.ts.map +1 -0
- package/autoscaling/cpu_load_signal.js +27 -0
- package/autoscaling/cpu_load_signal.js.map +1 -0
- package/autoscaling/event_loop_load_signal.d.ts +24 -0
- package/autoscaling/event_loop_load_signal.d.ts.map +1 -0
- package/autoscaling/event_loop_load_signal.js +39 -0
- package/autoscaling/event_loop_load_signal.js.map +1 -0
- package/autoscaling/index.d.ts +5 -0
- package/autoscaling/index.d.ts.map +1 -1
- package/autoscaling/index.js +5 -0
- package/autoscaling/index.js.map +1 -1
- package/autoscaling/load_signal.d.ts +100 -0
- package/autoscaling/load_signal.d.ts.map +1 -0
- package/autoscaling/load_signal.js +151 -0
- package/autoscaling/load_signal.js.map +1 -0
- package/autoscaling/memory_load_signal.d.ts +44 -0
- package/autoscaling/memory_load_signal.d.ts.map +1 -0
- package/autoscaling/memory_load_signal.js +159 -0
- package/autoscaling/memory_load_signal.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +26 -60
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +76 -225
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +21 -32
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +46 -92
- package/autoscaling/system_status.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +8 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +9 -8
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -0
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +8 -7
- package/events/local_event_manager.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +6 -6
- package/storages/dataset.d.ts +1 -1
- package/storages/request_provider.d.ts +17 -0
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +46 -17
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +0 -1
- package/storages/request_queue_v2.js.map +1 -1
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SystemStatus = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const utils_1 = require("@crawlee/utils");
|
|
6
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
|
+
const load_signal_1 = require("./load_signal");
|
|
7
7
|
const snapshotter_1 = require("./snapshotter");
|
|
8
|
+
/** The four built-in signal names that map to typed `SystemInfo` fields. */
|
|
9
|
+
const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
|
|
8
10
|
/**
|
|
9
11
|
* Provides a simple interface to reading system status from a {@link Snapshotter} instance.
|
|
10
12
|
* It only exposes two functions {@link SystemStatus.getCurrentStatus}
|
|
@@ -35,31 +37,24 @@ class SystemStatus {
|
|
|
35
37
|
writable: true,
|
|
36
38
|
value: void 0
|
|
37
39
|
});
|
|
38
|
-
Object.defineProperty(this, "
|
|
39
|
-
enumerable: true,
|
|
40
|
-
configurable: true,
|
|
41
|
-
writable: true,
|
|
42
|
-
value: void 0
|
|
43
|
-
});
|
|
44
|
-
Object.defineProperty(this, "maxEventLoopOverloadedRatio", {
|
|
45
|
-
enumerable: true,
|
|
46
|
-
configurable: true,
|
|
47
|
-
writable: true,
|
|
48
|
-
value: void 0
|
|
49
|
-
});
|
|
50
|
-
Object.defineProperty(this, "maxCpuOverloadedRatio", {
|
|
40
|
+
Object.defineProperty(this, "snapshotter", {
|
|
51
41
|
enumerable: true,
|
|
52
42
|
configurable: true,
|
|
53
43
|
writable: true,
|
|
54
44
|
value: void 0
|
|
55
45
|
});
|
|
56
|
-
Object.defineProperty(this, "
|
|
46
|
+
Object.defineProperty(this, "signals", {
|
|
57
47
|
enumerable: true,
|
|
58
48
|
configurable: true,
|
|
59
49
|
writable: true,
|
|
60
50
|
value: void 0
|
|
61
51
|
});
|
|
62
|
-
|
|
52
|
+
/**
|
|
53
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
54
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
55
|
+
* `overloadedRatio`.
|
|
56
|
+
*/
|
|
57
|
+
Object.defineProperty(this, "ratioOverrides", {
|
|
63
58
|
enumerable: true,
|
|
64
59
|
configurable: true,
|
|
65
60
|
writable: true,
|
|
@@ -72,15 +67,21 @@ class SystemStatus {
|
|
|
72
67
|
maxCpuOverloadedRatio: ow_1.default.optional.number,
|
|
73
68
|
maxClientOverloadedRatio: ow_1.default.optional.number,
|
|
74
69
|
snapshotter: ow_1.default.optional.object,
|
|
70
|
+
loadSignals: ow_1.default.optional.array,
|
|
75
71
|
config: ow_1.default.optional.object,
|
|
76
72
|
}));
|
|
77
|
-
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
|
|
73
|
+
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], config, } = options;
|
|
78
74
|
this.currentHistoryMillis = currentHistorySecs * 1000;
|
|
79
|
-
this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
|
|
80
|
-
this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
|
|
81
|
-
this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
|
|
82
|
-
this.maxClientOverloadedRatio = maxClientOverloadedRatio;
|
|
83
75
|
this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
|
|
76
|
+
// Built-in signals from the snapshotter + any custom signals
|
|
77
|
+
this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
|
|
78
|
+
// Allow legacy options to override the built-in signal ratios
|
|
79
|
+
this.ratioOverrides = {
|
|
80
|
+
memInfo: maxMemoryOverloadedRatio,
|
|
81
|
+
eventLoopInfo: maxEventLoopOverloadedRatio,
|
|
82
|
+
cpuInfo: maxCpuOverloadedRatio,
|
|
83
|
+
clientInfo: maxClientOverloadedRatio,
|
|
84
|
+
};
|
|
84
85
|
}
|
|
85
86
|
/**
|
|
86
87
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -124,80 +125,33 @@ class SystemStatus {
|
|
|
124
125
|
* Returns a system status object.
|
|
125
126
|
*/
|
|
126
127
|
_isSystemIdle(sampleDurationMillis) {
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
!eventLoopInfo.isOverloaded &&
|
|
134
|
-
!cpuInfo.isOverloaded &&
|
|
135
|
-
!clientInfo.isOverloaded,
|
|
136
|
-
memInfo,
|
|
137
|
-
eventLoopInfo,
|
|
138
|
-
cpuInfo,
|
|
139
|
-
clientInfo,
|
|
128
|
+
const result = {
|
|
129
|
+
isSystemIdle: true,
|
|
130
|
+
memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
131
|
+
eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
132
|
+
cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
133
|
+
clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
140
134
|
};
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Returns an object with an isOverloaded property set to true
|
|
160
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
161
|
-
*/
|
|
162
|
-
_isCpuOverloaded(sampleDurationMillis) {
|
|
163
|
-
const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
|
|
164
|
-
return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
|
|
165
|
-
}
|
|
166
|
-
/**
|
|
167
|
-
* Returns an object with an isOverloaded property set to true
|
|
168
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
169
|
-
*/
|
|
170
|
-
_isClientOverloaded(sampleDurationMillis) {
|
|
171
|
-
const sample = this.snapshotter.getClientSample(sampleDurationMillis);
|
|
172
|
-
return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
|
|
173
|
-
}
|
|
174
|
-
/**
|
|
175
|
-
* Returns an object with sample information and an isOverloaded property
|
|
176
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
177
|
-
*/
|
|
178
|
-
_isSampleOverloaded(sample, ratio) {
|
|
179
|
-
if (sample.length === 0) {
|
|
180
|
-
return {
|
|
181
|
-
isOverloaded: false,
|
|
182
|
-
limitRatio: ratio,
|
|
183
|
-
actualRatio: 0,
|
|
184
|
-
};
|
|
135
|
+
let loadSignalInfo;
|
|
136
|
+
for (const signal of this.signals) {
|
|
137
|
+
const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
|
|
138
|
+
const sample = signal.getSample(sampleDurationMillis);
|
|
139
|
+
const info = (0, load_signal_1.evaluateLoadSignalSample)(sample, ratio);
|
|
140
|
+
if (info.isOverloaded) {
|
|
141
|
+
result.isSystemIdle = false;
|
|
142
|
+
}
|
|
143
|
+
if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
|
|
144
|
+
result[signal.name] = info;
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
loadSignalInfo ?? (loadSignalInfo = {});
|
|
148
|
+
loadSignalInfo[signal.name] = info;
|
|
149
|
+
}
|
|
185
150
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
for (let i = 1; i < sample.length; i++) {
|
|
189
|
-
const previous = sample[i - 1];
|
|
190
|
-
const current = sample[i];
|
|
191
|
-
const weight = +current.createdAt - +previous.createdAt;
|
|
192
|
-
weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
|
|
193
|
-
values.push(+current.isOverloaded);
|
|
151
|
+
if (loadSignalInfo) {
|
|
152
|
+
result.loadSignalInfo = loadSignalInfo;
|
|
194
153
|
}
|
|
195
|
-
|
|
196
|
-
return {
|
|
197
|
-
isOverloaded: wAvg > ratio,
|
|
198
|
-
limitRatio: ratio,
|
|
199
|
-
actualRatio: Math.round(wAvg * 1000) / 1000,
|
|
200
|
-
};
|
|
154
|
+
return result;
|
|
201
155
|
}
|
|
202
156
|
}
|
|
203
157
|
exports.SystemStatus = SystemStatus;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system_status.js","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":";;;;AAAA,
|
|
1
|
+
{"version":3,"file":"system_status.js","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAIpB,+CAAyD;AACzD,+CAA4C;AA4G5C,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;AAE5F;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAa,YAAY;IAYrB,YAAY,UAA+B,EAAE;QAX5B;;;;;WAA6B;QAC7B;;;;;WAAyB;QACzB;;;;;WAAsB;QAEvC;;;;WAIG;QACK;;;;;WAAuC;QAG3C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC9B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,kBAAkB,GAAG,CAAC,EACtB,wBAAwB,GAAG,GAAG,EAC9B,2BAA2B,GAAG,GAAG,EACjC,qBAAqB,GAAG,GAAG,EAC3B,wBAAwB,GAAG,GAAG,EAC9B,WAAW,EACX,WAAW,GAAG,EAAE,EAChB,MAAM,GACT,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACtD,IAAI,CAAC,WAAW,GAAG,WAAW,IAAI,IAAI,yBAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9D,6DAA6D;QAC7D,IAAI,CAAC,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE,GAAG,WAAW,CAAC,CAAC;QAEtE,8DAA8D;QAC9D,IAAI,CAAC,cAAc,GAAG;YAClB,OAAO,EAAE,wBAAwB;YACjC,aAAa,EAAE,2BAA2B;YAC1C,OAAO,EAAE,qBAAqB;YAC9B,UAAU,EAAE,wBAAwB;SACvC,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB;QACf,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,oBAA6B;QACjD,MAAM,MAAM,GAAe;YACvB,YAAY,EAAE,IAAI;YAClB,OAAO,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;YAC/D,aAAa,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;YACrE,OAAO,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;YAC/D,UAAU,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;SACrE,CAAC;QAEF,IAAI,cAAsD,CAAC;QAE3D,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC;YACzE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;YACtD,MAAM,IAAI,GAAG,IAAA,sCAAwB,EAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAErD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC;YAChC,CAAC;YAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvC,MAAc,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACJ,cAAc,KAAd,cAAc,GAAK,EAAE,EAAC;gBACtB,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC;QAC3C,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAlID,oCAkIC"}
|
|
@@ -139,6 +139,14 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
139
139
|
* If provided, disallowed URLs will be ignored.
|
|
140
140
|
*/
|
|
141
141
|
robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
|
|
142
|
+
/**
|
|
143
|
+
* Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
|
|
144
|
+
* `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
|
|
145
|
+
* {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
|
|
146
|
+
*/
|
|
147
|
+
respectRobotsTxtFile?: boolean | {
|
|
148
|
+
userAgent?: string;
|
|
149
|
+
};
|
|
142
150
|
/**
|
|
143
151
|
* When a request is skipped for some reason, you can use this callback to act on it.
|
|
144
152
|
* This is currently fired for requests skipped
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI7C,OAAO,KAAK,EAAE,OAAO,EAAkB,MAAM,YAAY,CAAC;AAC1D,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,4BAA4B,EAC/B,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,sBAAsB,EAGzB,MAAM,UAAU,CAAC;AAUlB,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEjD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,GAAG;IACtE,YAAY,EAAE;QACV,kBAAkB,EAAE,CAChB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,KACjC,OAAO,CAAC,wBAAwB,CAAC,CAAC;KAC1C,CAAC;CACL,GACF,OAAO,CAAC,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI7C,OAAO,KAAK,EAAE,OAAO,EAAkB,MAAM,YAAY,CAAC;AAC1D,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,4BAA4B,EAC/B,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,sBAAsB,EAGzB,MAAM,UAAU,CAAC;AAUlB,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEjD;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,GAAG;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAExD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,GAAG;IACtE,YAAY,EAAE;QACV,kBAAkB,EAAE,CAChB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,KACjC,OAAO,CAAC,wBAAwB,CAAC,CAAC;KAC1C,CAAC;CACL,GACF,OAAO,CAAC,sBAAsB,CAAC,CAiOjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -95,6 +95,7 @@ async function enqueueLinks(options) {
|
|
|
95
95
|
urls: ow_1.default.array.ofType(ow_1.default.string),
|
|
96
96
|
requestQueue: ow_1.default.object.hasKeys('addRequestsBatched'),
|
|
97
97
|
robotsTxtFile: ow_1.default.optional.object.hasKeys('isAllowed'),
|
|
98
|
+
respectRobotsTxtFile: ow_1.default.optional.any(ow_1.default.boolean, ow_1.default.object.exactShape({ userAgent: ow_1.default.optional.string })),
|
|
98
99
|
onSkippedRequest: ow_1.default.optional.function,
|
|
99
100
|
forefront: ow_1.default.optional.boolean,
|
|
100
101
|
skipNavigation: ow_1.default.optional.boolean,
|
|
@@ -184,10 +185,11 @@ async function enqueueLinks(options) {
|
|
|
184
185
|
}
|
|
185
186
|
}
|
|
186
187
|
let requestOptions = (0, shared_1.createRequestOptions)(urls, options);
|
|
187
|
-
if (robotsTxtFile) {
|
|
188
|
+
if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
|
|
189
|
+
const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
|
|
188
190
|
const skippedRequests = [];
|
|
189
191
|
requestOptions = requestOptions.filter((request) => {
|
|
190
|
-
if (robotsTxtFile.isAllowed(request.url)) {
|
|
192
|
+
if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
|
|
191
193
|
return true;
|
|
192
194
|
}
|
|
193
195
|
skippedRequests.push(request);
|
|
@@ -221,15 +223,14 @@ async function enqueueLinks(options) {
|
|
|
221
223
|
await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
|
|
222
224
|
return filtered;
|
|
223
225
|
}
|
|
224
|
-
|
|
225
|
-
if (typeof limit === 'number' && limit < requests.length) {
|
|
226
|
-
await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
|
|
227
|
-
requests = requests.slice(0, limit);
|
|
228
|
-
}
|
|
229
|
-
const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
|
|
226
|
+
const { addedRequests, requestsOverLimit } = await requestQueue.addRequestsBatched(await createFilteredRequests(), {
|
|
230
227
|
forefront,
|
|
231
228
|
waitForAllRequestsToBeAdded,
|
|
229
|
+
maxNewRequests: limit,
|
|
232
230
|
});
|
|
231
|
+
if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
|
|
232
|
+
await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
|
|
233
|
+
}
|
|
233
234
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
234
235
|
}
|
|
235
236
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;AAwRA,oCA0OC;AAUD,wFAoCC;;AA9iBD,oDAAoB;AACpB,iCAAkC;AAGlC,6DAA6B;AAkB7B,qCAOkB;AA0KlB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,+BAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAC9B,OAOC;IAED,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC;QACrD,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1G,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,KAAK,UAAU,qBAAqB,CAChC,eAAwE,EACxE,MAA4B;QAE5B,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC;oBACpB,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM;iBAC1C,CAAkB,CAAC;YACxB,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,IAAI,OAAO,CAAC,oBAAoB,KAAK,KAAK,EAAE,CAAC;QAC1D,MAAM,eAAe,GACjB,OAAO,OAAO,CAAC,oBAAoB,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC7G,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,eAAe,CAAC,EAAE,CAAC;gBACxD,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,wBAAwB,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACb,MAAM,kBAAkB,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;YAC7D,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACtB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,kBAAkB,CAAC;QAC9B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAqB,CAAC;QAEnD,MAAM,qBAAqB,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,UAAU,sBAAsB;QACjC,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAA,uBAAc,EACjB,cAAc,EACd,uBAAuB,EACvB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,IAAA,uBAAc,EACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACF,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,IAAA,iCAAwB,EAAC,gCAAgC,EAAE,uBAAuB,EAAE,CAAC,GAAG,EAAE,EAAE,CACzG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAC5B,CAAC;QAEF,MAAM,qBAAqB,CACvB,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EACvC,SAAS,CACZ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,MAAM,sBAAsB,EAAE,EAAE;QAC/G,SAAS;QACT,2BAA2B;QAC3B,cAAc,EAAE,KAAK;KACxB,CAAC,CAAC;IAEH,IAAI,iBAAiB,EAAE,MAAM,KAAK,SAAS,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1E,MAAM,qBAAqB,CACvB,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAI,EAAE,CAAC,CAAC,EAC3E,cAAc,CACjB,CAAC;IACN,CAAC;IAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAoBb,gBAAgB;
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAoBb,gBAAgB;YAahB,aAAa;CAM9B"}
|
|
@@ -96,14 +96,9 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
96
96
|
}
|
|
97
97
|
async createMemoryInfo() {
|
|
98
98
|
try {
|
|
99
|
-
|
|
100
|
-
const memInfo = await (0, utils_1.getMemoryInfoV2)(await this.isContainerizedWrapper());
|
|
101
|
-
return {
|
|
102
|
-
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
const memInfo = await (0, utils_1.getMemoryInfo)();
|
|
99
|
+
const memInfo = await this.getMemoryInfo();
|
|
106
100
|
return {
|
|
101
|
+
memTotalBytes: memInfo.totalBytes,
|
|
107
102
|
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
108
103
|
};
|
|
109
104
|
}
|
|
@@ -112,6 +107,12 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
112
107
|
return {};
|
|
113
108
|
}
|
|
114
109
|
}
|
|
110
|
+
async getMemoryInfo() {
|
|
111
|
+
if (this.config.get('systemInfoV2')) {
|
|
112
|
+
return (0, utils_1.getMemoryInfoV2)(await this.isContainerizedWrapper());
|
|
113
|
+
}
|
|
114
|
+
return (0, utils_1.getMemoryInfo)();
|
|
115
|
+
}
|
|
115
116
|
}
|
|
116
117
|
exports.LocalEventManager = LocalEventManager;
|
|
117
118
|
//# sourceMappingURL=local_event_manager.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AAEzB,0CAAuG;AAEvG,6DAA6B;AAC7B,gDAA0E;AAG1E,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACY;;;;mBAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AAEzB,0CAAuG;AAEvG,6DAA6B;AAC7B,gDAA0E;AAG1E,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACY;;;;mBAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IAgHlD,CAAC;IA9GG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,IAAA,uBAAe,GAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,IAAA,4BAAoB,EAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3C,OAAO;gBACH,aAAa,EAAE,OAAO,CAAC,UAAU;gBACjC,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,OAAO,IAAA,uBAAe,EAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAjHD,8CAiHC"}
|
package/index.mjs
CHANGED
|
@@ -26,6 +26,7 @@ export const LoggerJson = mod.LoggerJson;
|
|
|
26
26
|
export const LoggerText = mod.LoggerText;
|
|
27
27
|
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
28
28
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
29
|
+
export const MemoryLoadSignal = mod.MemoryLoadSignal;
|
|
29
30
|
export const MissingRouteError = mod.MissingRouteError;
|
|
30
31
|
export const NonRetryableError = mod.NonRetryableError;
|
|
31
32
|
export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
|
|
@@ -53,6 +54,7 @@ export const Session = mod.Session;
|
|
|
53
54
|
export const SessionError = mod.SessionError;
|
|
54
55
|
export const SessionPool = mod.SessionPool;
|
|
55
56
|
export const SitemapRequestList = mod.SitemapRequestList;
|
|
57
|
+
export const SnapshotStore = mod.SnapshotStore;
|
|
56
58
|
export const Snapshotter = mod.Snapshotter;
|
|
57
59
|
export const Statistics = mod.Statistics;
|
|
58
60
|
export const StorageManager = mod.StorageManager;
|
|
@@ -65,11 +67,15 @@ export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
|
65
67
|
export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
|
|
66
68
|
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
67
69
|
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
70
|
+
export const createClientLoadSignal = mod.createClientLoadSignal;
|
|
71
|
+
export const createCpuLoadSignal = mod.createCpuLoadSignal;
|
|
68
72
|
export const createDeserialize = mod.createDeserialize;
|
|
73
|
+
export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
|
|
69
74
|
export const createRequestOptions = mod.createRequestOptions;
|
|
70
75
|
export const createRequests = mod.createRequests;
|
|
71
76
|
export const deserializeArray = mod.deserializeArray;
|
|
72
77
|
export const enqueueLinks = mod.enqueueLinks;
|
|
78
|
+
export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
|
|
73
79
|
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
74
80
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
75
81
|
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.16.1-beta.
|
|
3
|
+
"version": "3.16.1-beta.50",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,14 +59,14 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.16.1-beta.
|
|
63
|
-
"@crawlee/types": "3.16.1-beta.
|
|
64
|
-
"@crawlee/utils": "3.16.1-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.16.1-beta.50",
|
|
63
|
+
"@crawlee/types": "3.16.1-beta.50",
|
|
64
|
+
"@crawlee/utils": "3.16.1-beta.50",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
67
67
|
"csv-stringify": "^6.2.0",
|
|
68
68
|
"fs-extra": "^11.0.0",
|
|
69
|
-
"got-scraping": "^4.
|
|
69
|
+
"got-scraping": "^4.2.1",
|
|
70
70
|
"json5": "^2.2.3",
|
|
71
71
|
"minimatch": "^9.0.0",
|
|
72
72
|
"ow": "^0.28.1",
|
|
@@ -83,5 +83,5 @@
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
},
|
|
86
|
-
"gitHead": "
|
|
86
|
+
"gitHead": "6d241a21c39b0415f276ec10b1d9e207580faebb"
|
|
87
87
|
}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -314,7 +314,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
314
314
|
* If the dataset is empty, reduce will return undefined.
|
|
315
315
|
*
|
|
316
316
|
* @param iteratee
|
|
317
|
-
* @param memo Unset parameter,
|
|
317
|
+
* @param memo Unset parameter, necessary to be able to pass options
|
|
318
318
|
* @param [options] An object containing extra options for `reduce()`
|
|
319
319
|
*/
|
|
320
320
|
reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
|
|
@@ -345,6 +345,17 @@ export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions
|
|
|
345
345
|
* @default 1000
|
|
346
346
|
*/
|
|
347
347
|
waitBetweenBatchesMillis?: number;
|
|
348
|
+
/**
|
|
349
|
+
* If set, only this many *actually new* requests (i.e. not already present in the queue) will be added.
|
|
350
|
+
* Once the budget is reached, remaining requests from the iterable will be collected in
|
|
351
|
+
* {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} instead.
|
|
352
|
+
*
|
|
353
|
+
* This is useful in combination with `maxRequestsPerCrawl` to avoid duplicate URLs consuming the budget.
|
|
354
|
+
*
|
|
355
|
+
* **Note:** Setting this option implicitly enables {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`},
|
|
356
|
+
* since all batches must complete before leftover requests can be accurately reported.
|
|
357
|
+
*/
|
|
358
|
+
maxNewRequests?: number;
|
|
348
359
|
}
|
|
349
360
|
export interface AddRequestsBatchedResult {
|
|
350
361
|
addedRequests: ProcessedRequest[];
|
|
@@ -365,6 +376,12 @@ export interface AddRequestsBatchedResult {
|
|
|
365
376
|
* ```
|
|
366
377
|
*/
|
|
367
378
|
waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
|
|
379
|
+
/**
|
|
380
|
+
* Requests from the input that were not added to the queue because the
|
|
381
|
+
* {@link AddRequestsBatchedOptions.maxNewRequests|`maxNewRequests`} budget was reached.
|
|
382
|
+
* Empty when `maxNewRequests` is not set.
|
|
383
|
+
*/
|
|
384
|
+
requestsOverLimit?: Source[];
|
|
368
385
|
}
|
|
369
386
|
export {};
|
|
370
387
|
//# sourceMappingURL=request_provider.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAYxB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAIzE,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAmClE,QAAQ,CAAC,MAAM;IAlCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAYxB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAIzE,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAmClE,QAAQ,CAAC,MAAM;IAlCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IA8KpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAMtD;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAQjC;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC;CA6BlH;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;;OASG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAEzD;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAChC"}
|
|
@@ -348,6 +348,7 @@ class RequestProvider {
|
|
|
348
348
|
waitForAllRequestsToBeAdded: ow_1.default.optional.boolean,
|
|
349
349
|
batchSize: ow_1.default.optional.number,
|
|
350
350
|
waitBetweenBatchesMillis: ow_1.default.optional.number,
|
|
351
|
+
maxNewRequests: ow_1.default.optional.number,
|
|
351
352
|
}));
|
|
352
353
|
const addRequest = this.addRequest.bind(this);
|
|
353
354
|
async function* generateRequests() {
|
|
@@ -375,8 +376,14 @@ class RequestProvider {
|
|
|
375
376
|
}
|
|
376
377
|
}
|
|
377
378
|
}
|
|
378
|
-
const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
|
|
379
|
-
|
|
379
|
+
const { batchSize = 1000, waitBetweenBatchesMillis = 1000, maxNewRequests } = options;
|
|
380
|
+
let remainingBudget = maxNewRequests ?? Infinity;
|
|
381
|
+
const requestsOverLimit = [];
|
|
382
|
+
// If there's a limit on the number of added requests, do not send batches bigger than the limit
|
|
383
|
+
const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
|
|
384
|
+
// Hold onto the underlying iterator so we can drain leftovers from it in buildResult
|
|
385
|
+
const requestIterator = generateRequests();
|
|
386
|
+
const chunks = (0, utils_1.peekableAsyncIterable)((0, utils_1.chunkedAsyncIterable)(requestIterator, effectiveChunkSize));
|
|
380
387
|
const chunksIterator = chunks[Symbol.asyncIterator]();
|
|
381
388
|
const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
|
|
382
389
|
const resultsToReturn = [];
|
|
@@ -388,25 +395,50 @@ class RequestProvider {
|
|
|
388
395
|
}
|
|
389
396
|
return resultsToReturn;
|
|
390
397
|
};
|
|
391
|
-
|
|
398
|
+
/**
|
|
399
|
+
* Process a chunk: send it to the queue, then update the remaining budget if maxNewRequests is active.
|
|
400
|
+
*/
|
|
401
|
+
const processChunk = async (chunk, cache = true) => {
|
|
402
|
+
const results = await attemptToAddToQueueAndAddAnyUnprocessed(chunk, cache);
|
|
403
|
+
if (maxNewRequests !== undefined) {
|
|
404
|
+
remainingBudget -= results.filter((r) => !r.wasAlreadyPresent).length;
|
|
405
|
+
}
|
|
406
|
+
return results;
|
|
407
|
+
};
|
|
408
|
+
/**
|
|
409
|
+
* Build the final result. When maxNewRequests is set, drains any remaining items
|
|
410
|
+
* from the underlying request iterator into requestsOverLimit.
|
|
411
|
+
*
|
|
412
|
+
* We accept the iterator explicitly (rather than closing over it) to make it obvious
|
|
413
|
+
* that this is the *same* iterator that `chunkedAsyncIterable` has been consuming —
|
|
414
|
+
* so only unconsumed items are drained. We drain `requestIterator` (not `chunks`)
|
|
415
|
+
* because `chunkedAsyncIterable` stops yielding when the budget-based chunk size
|
|
416
|
+
* drops to 0, leaving unconsumed items in the underlying iterator.
|
|
417
|
+
*/
|
|
418
|
+
const buildResult = async (addedRequests, waitForAllRequestsToBeAdded, unconsumedIterator) => {
|
|
419
|
+
if (maxNewRequests !== undefined) {
|
|
420
|
+
for await (const request of unconsumedIterator) {
|
|
421
|
+
requestsOverLimit.push(request);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
return { addedRequests, waitForAllRequestsToBeAdded, requestsOverLimit };
|
|
425
|
+
};
|
|
426
|
+
// Add initial batch to process right away
|
|
392
427
|
const initialChunk = await chunksIterator.peek();
|
|
393
428
|
if (initialChunk === undefined) {
|
|
394
|
-
return
|
|
429
|
+
return buildResult([], Promise.resolve([]), requestIterator);
|
|
395
430
|
}
|
|
396
|
-
const addedRequests = await
|
|
431
|
+
const addedRequests = await processChunk(initialChunk);
|
|
397
432
|
await chunksIterator.next();
|
|
398
|
-
// If we have no more requests to add, return immediately
|
|
433
|
+
// If we have no more requests to add (either exhausted or budget hit), return immediately
|
|
399
434
|
if ((await chunksIterator.peek()) === undefined) {
|
|
400
|
-
return
|
|
401
|
-
addedRequests,
|
|
402
|
-
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
403
|
-
};
|
|
435
|
+
return buildResult(addedRequests, Promise.resolve([]), requestIterator);
|
|
404
436
|
}
|
|
405
437
|
// eslint-disable-next-line no-async-promise-executor
|
|
406
438
|
const promise = new Promise(async (resolve) => {
|
|
407
439
|
const finalAddedRequests = [];
|
|
408
440
|
for await (const requestChunk of chunks) {
|
|
409
|
-
finalAddedRequests.push(...(await
|
|
441
|
+
finalAddedRequests.push(...(await processChunk(requestChunk, false)));
|
|
410
442
|
await (0, utils_1.sleep)(waitBetweenBatchesMillis);
|
|
411
443
|
}
|
|
412
444
|
resolve(finalAddedRequests);
|
|
@@ -415,14 +447,11 @@ class RequestProvider {
|
|
|
415
447
|
void promise.finally(() => {
|
|
416
448
|
this.inProgressRequestBatchCount -= 1;
|
|
417
449
|
});
|
|
418
|
-
//
|
|
419
|
-
if (options.waitForAllRequestsToBeAdded) {
|
|
450
|
+
// When maxNewRequests is set, we must wait for all batches so we can accurately report skipped requests.
|
|
451
|
+
if (options.waitForAllRequestsToBeAdded || maxNewRequests !== undefined) {
|
|
420
452
|
addedRequests.push(...(await promise));
|
|
421
453
|
}
|
|
422
|
-
return
|
|
423
|
-
addedRequests,
|
|
424
|
-
waitForAllRequestsToBeAdded: promise,
|
|
425
|
-
};
|
|
454
|
+
return buildResult(addedRequests, promise, requestIterator);
|
|
426
455
|
}
|
|
427
456
|
/**
|
|
428
457
|
* Gets the request from the queue specified by ID.
|