@crawlee/core 4.0.0-beta.71 → 4.0.0-beta.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -1
- package/autoscaling/autoscaled_pool.js +9 -2
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +101 -0
- package/autoscaling/snapshotter.d.ts +36 -63
- package/autoscaling/snapshotter.js +83 -175
- package/autoscaling/system_status.d.ts +21 -33
- package/autoscaling/system_status.js +45 -82
- package/configuration.d.ts +0 -1
- package/configuration.js +0 -1
- package/cookie_utils.d.ts +0 -1
- package/cookie_utils.js +2 -1
- package/crawlers/context_pipeline.d.ts +0 -1
- package/crawlers/context_pipeline.js +0 -1
- package/crawlers/crawler_commons.d.ts +0 -1
- package/crawlers/crawler_commons.js +0 -1
- package/crawlers/crawler_utils.d.ts +0 -1
- package/crawlers/crawler_utils.js +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -1
- package/crawlers/error_snapshotter.js +0 -1
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +0 -1
- package/crawlers/index.js +0 -1
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +6 -1
- package/crawlers/statistics.js +8 -1
- package/enqueue_links/enqueue_links.d.ts +8 -1
- package/enqueue_links/enqueue_links.js +9 -9
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +0 -1
- package/enqueue_links/shared.js +0 -1
- package/errors.d.ts +20 -1
- package/errors.js +22 -1
- package/events/event_manager.d.ts +0 -1
- package/events/event_manager.js +0 -1
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +9 -6
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/log.d.ts +0 -1
- package/log.js +0 -1
- package/memory-storage/consts.d.ts +0 -1
- package/memory-storage/consts.js +0 -1
- package/memory-storage/index.d.ts +0 -1
- package/memory-storage/index.js +0 -1
- package/memory-storage/memory-storage.d.ts +0 -1
- package/memory-storage/memory-storage.js +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
- package/memory-storage/resource-clients/common/base-client.js +0 -1
- package/memory-storage/resource-clients/dataset.d.ts +0 -1
- package/memory-storage/resource-clients/dataset.js +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
- package/memory-storage/resource-clients/key-value-store.js +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts +0 -1
- package/memory-storage/resource-clients/request-queue.js +0 -1
- package/memory-storage/utils.d.ts +6 -1
- package/memory-storage/utils.js +15 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts +0 -1
- package/proxy_configuration.js +0 -1
- package/recoverable_state.d.ts +0 -1
- package/recoverable_state.js +0 -1
- package/request.d.ts +0 -1
- package/request.js +0 -1
- package/router.d.ts +117 -15
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +0 -1
- package/service_locator.d.ts +0 -1
- package/service_locator.js +0 -1
- package/session_pool/consts.d.ts +0 -1
- package/session_pool/consts.js +0 -1
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +0 -1
- package/session_pool/fingerprint.js +0 -1
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.js +0 -1
- package/session_pool/session.d.ts +0 -1
- package/session_pool/session.js +0 -1
- package/session_pool/session_pool.d.ts +0 -1
- package/session_pool/session_pool.js +0 -1
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +0 -1
- package/storages/dataset.d.ts +1 -2
- package/storages/dataset.js +0 -1
- package/storages/index.d.ts +0 -1
- package/storages/index.js +0 -1
- package/storages/key_value_store.d.ts +0 -1
- package/storages/key_value_store.js +0 -1
- package/storages/key_value_store_codec.d.ts +0 -1
- package/storages/key_value_store_codec.js +0 -1
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +6 -1
- package/storages/request_list.js +17 -3
- package/storages/request_loader.d.ts +0 -1
- package/storages/request_loader.js +0 -1
- package/storages/request_manager.d.ts +0 -1
- package/storages/request_manager.js +0 -1
- package/storages/request_manager_tandem.d.ts +0 -1
- package/storages/request_manager_tandem.js +0 -1
- package/storages/request_queue.d.ts +24 -1
- package/storages/request_queue.js +80 -27
- package/storages/sitemap_request_loader.d.ts +0 -1
- package/storages/sitemap_request_loader.js +0 -1
- package/storages/storage_instance_manager.d.ts +0 -1
- package/storages/storage_instance_manager.js +0 -1
- package/storages/storage_stats.d.ts +0 -1
- package/storages/storage_stats.js +0 -1
- package/storages/utils.d.ts +0 -1
- package/storages/utils.js +0 -1
- package/typedefs.d.ts +0 -1
- package/typedefs.js +0 -1
- package/validators.d.ts +0 -1
- package/validators.js +0 -1
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/memory-storage/consts.d.ts.map +0 -1
- package/memory-storage/consts.js.map +0 -1
- package/memory-storage/index.d.ts.map +0 -1
- package/memory-storage/index.js.map +0 -1
- package/memory-storage/memory-storage.d.ts.map +0 -1
- package/memory-storage/memory-storage.js.map +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
- package/memory-storage/resource-clients/common/base-client.js.map +0 -1
- package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
- package/memory-storage/resource-clients/dataset.js.map +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
- package/memory-storage/resource-clients/key-value-store.js.map +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
- package/memory-storage/resource-clients/request-queue.js.map +0 -1
- package/memory-storage/utils.d.ts.map +0 -1
- package/memory-storage/utils.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/service_locator.d.ts.map +0 -1
- package/service_locator.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/fingerprint.d.ts.map +0 -1
- package/session_pool/fingerprint.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/key_value_store_codec.d.ts.map +0 -1
- package/storages/key_value_store_codec.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_loader.d.ts.map +0 -1
- package/storages/request_loader.js.map +0 -1
- package/storages/request_manager.d.ts.map +0 -1
- package/storages/request_manager.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/sitemap_request_loader.d.ts.map +0 -1
- package/storages/sitemap_request_loader.js.map +0 -1
- package/storages/storage_instance_manager.d.ts.map +0 -1
- package/storages/storage_instance_manager.js.map +0 -1
- package/storages/storage_stats.d.ts.map +0 -1
- package/storages/storage_stats.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import { weightedAvg } from '@crawlee/utils';
|
|
2
1
|
import ow from 'ow';
|
|
2
|
+
import { evaluateLoadSignalSample } from './load_signal.js';
|
|
3
3
|
import { Snapshotter } from './snapshotter.js';
|
|
4
|
+
/** The four built-in signal names that map to typed `SystemInfo` fields. */
|
|
5
|
+
const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
|
|
4
6
|
/**
|
|
5
7
|
* Provides a simple interface to reading system status from a {@link Snapshotter} instance.
|
|
6
8
|
* It only exposes two functions {@link SystemStatus.getCurrentStatus}
|
|
@@ -25,11 +27,14 @@ import { Snapshotter } from './snapshotter.js';
|
|
|
25
27
|
*/
|
|
26
28
|
export class SystemStatus {
|
|
27
29
|
currentHistoryMillis;
|
|
28
|
-
maxMemoryOverloadedRatio;
|
|
29
|
-
maxEventLoopOverloadedRatio;
|
|
30
|
-
maxCpuOverloadedRatio;
|
|
31
|
-
maxClientOverloadedRatio;
|
|
32
30
|
snapshotter;
|
|
31
|
+
signals;
|
|
32
|
+
/**
|
|
33
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
34
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
35
|
+
* `overloadedRatio`.
|
|
36
|
+
*/
|
|
37
|
+
ratioOverrides;
|
|
33
38
|
constructor(options = {}) {
|
|
34
39
|
ow(options, ow.object.exactShape({
|
|
35
40
|
currentHistorySecs: ow.optional.number,
|
|
@@ -38,14 +43,20 @@ export class SystemStatus {
|
|
|
38
43
|
maxCpuOverloadedRatio: ow.optional.number,
|
|
39
44
|
maxClientOverloadedRatio: ow.optional.number,
|
|
40
45
|
snapshotter: ow.optional.object,
|
|
46
|
+
loadSignals: ow.optional.array,
|
|
41
47
|
}));
|
|
42
|
-
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, } = options;
|
|
48
|
+
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
|
|
43
49
|
this.currentHistoryMillis = currentHistorySecs * 1000;
|
|
44
|
-
this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
|
|
45
|
-
this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
|
|
46
|
-
this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
|
|
47
|
-
this.maxClientOverloadedRatio = maxClientOverloadedRatio;
|
|
48
50
|
this.snapshotter = snapshotter || new Snapshotter();
|
|
51
|
+
// Built-in signals from the snapshotter + any custom signals
|
|
52
|
+
this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
|
|
53
|
+
// Allow legacy options to override the built-in signal ratios
|
|
54
|
+
this.ratioOverrides = {
|
|
55
|
+
memInfo: maxMemoryOverloadedRatio,
|
|
56
|
+
eventLoopInfo: maxEventLoopOverloadedRatio,
|
|
57
|
+
cpuInfo: maxCpuOverloadedRatio,
|
|
58
|
+
clientInfo: maxClientOverloadedRatio,
|
|
59
|
+
};
|
|
49
60
|
}
|
|
50
61
|
/**
|
|
51
62
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -89,80 +100,32 @@ export class SystemStatus {
|
|
|
89
100
|
* Returns a system status object.
|
|
90
101
|
*/
|
|
91
102
|
_isSystemIdle(sampleDurationMillis) {
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
!eventLoopInfo.isOverloaded &&
|
|
99
|
-
!cpuInfo.isOverloaded &&
|
|
100
|
-
!clientInfo.isOverloaded,
|
|
101
|
-
memInfo,
|
|
102
|
-
eventLoopInfo,
|
|
103
|
-
cpuInfo,
|
|
104
|
-
clientInfo,
|
|
103
|
+
const result = {
|
|
104
|
+
isSystemIdle: true,
|
|
105
|
+
memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
106
|
+
eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
107
|
+
cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
108
|
+
clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
105
109
|
};
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
|
|
122
|
-
}
|
|
123
|
-
/**
|
|
124
|
-
* Returns an object with an isOverloaded property set to true
|
|
125
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
126
|
-
*/
|
|
127
|
-
_isCpuOverloaded(sampleDurationMillis) {
|
|
128
|
-
const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
|
|
129
|
-
return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Returns an object with an isOverloaded property set to true
|
|
133
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
134
|
-
*/
|
|
135
|
-
_isClientOverloaded(sampleDurationMillis) {
|
|
136
|
-
const sample = this.snapshotter.getClientSample(sampleDurationMillis);
|
|
137
|
-
return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
|
|
138
|
-
}
|
|
139
|
-
/**
|
|
140
|
-
* Returns an object with sample information and an isOverloaded property
|
|
141
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
142
|
-
*/
|
|
143
|
-
_isSampleOverloaded(sample, ratio) {
|
|
144
|
-
if (sample.length === 0) {
|
|
145
|
-
return {
|
|
146
|
-
isOverloaded: false,
|
|
147
|
-
limitRatio: ratio,
|
|
148
|
-
actualRatio: 0,
|
|
149
|
-
};
|
|
110
|
+
let loadSignalInfo;
|
|
111
|
+
for (const signal of this.signals) {
|
|
112
|
+
const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
|
|
113
|
+
const sample = signal.getSample(sampleDurationMillis);
|
|
114
|
+
const info = evaluateLoadSignalSample(sample, ratio);
|
|
115
|
+
if (info.isOverloaded) {
|
|
116
|
+
result.isSystemIdle = false;
|
|
117
|
+
}
|
|
118
|
+
if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
|
|
119
|
+
result[signal.name] = info;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
loadSignalInfo ??= {};
|
|
123
|
+
loadSignalInfo[signal.name] = info;
|
|
124
|
+
}
|
|
150
125
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
for (let i = 1; i < sample.length; i++) {
|
|
154
|
-
const previous = sample[i - 1];
|
|
155
|
-
const current = sample[i];
|
|
156
|
-
const weight = +current.createdAt - +previous.createdAt;
|
|
157
|
-
weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
|
|
158
|
-
values.push(+current.isOverloaded);
|
|
126
|
+
if (loadSignalInfo) {
|
|
127
|
+
result.loadSignalInfo = loadSignalInfo;
|
|
159
128
|
}
|
|
160
|
-
|
|
161
|
-
return {
|
|
162
|
-
isOverloaded: wAvg > ratio,
|
|
163
|
-
limitRatio: ratio,
|
|
164
|
-
actualRatio: Math.round(wAvg * 1000) / 1000,
|
|
165
|
-
};
|
|
129
|
+
return result;
|
|
166
130
|
}
|
|
167
131
|
}
|
|
168
|
-
//# sourceMappingURL=system_status.js.map
|
package/configuration.d.ts
CHANGED
package/configuration.js
CHANGED
package/cookie_utils.d.ts
CHANGED
package/cookie_utils.js
CHANGED
|
@@ -103,6 +103,8 @@ export function mergeCookies(url, sourceCookies) {
|
|
|
103
103
|
if (!cookieString)
|
|
104
104
|
continue;
|
|
105
105
|
const cookie = Cookie.parse(cookieString);
|
|
106
|
+
if (!cookie)
|
|
107
|
+
throw new CookieParseError(cookieString);
|
|
106
108
|
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
107
109
|
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
108
110
|
});
|
|
@@ -116,4 +118,3 @@ export function mergeCookies(url, sourceCookies) {
|
|
|
116
118
|
}
|
|
117
119
|
return jar.getCookieStringSync(url);
|
|
118
120
|
}
|
|
119
|
-
//# sourceMappingURL=cookie_utils.js.map
|
|
@@ -68,4 +68,3 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
|
|
|
68
68
|
*/
|
|
69
69
|
abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
|
|
70
70
|
}
|
|
71
|
-
//# sourceMappingURL=context_pipeline.d.ts.map
|
package/crawlers/index.d.ts
CHANGED
package/crawlers/index.js
CHANGED
package/crawlers/statistics.d.ts
CHANGED
|
@@ -86,6 +86,12 @@ export declare class Statistics {
|
|
|
86
86
|
* @ignore
|
|
87
87
|
*/
|
|
88
88
|
failJob(id: number | string, retryCount: number): void;
|
|
89
|
+
/**
|
|
90
|
+
* Discards a started job without affecting the finished/failed counters, e.g. when a request
|
|
91
|
+
* turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
|
|
92
|
+
* @ignore
|
|
93
|
+
*/
|
|
94
|
+
discardJob(id: number | string): void;
|
|
89
95
|
/**
|
|
90
96
|
* Calculate the current statistics
|
|
91
97
|
*/
|
|
@@ -201,4 +207,3 @@ export interface StatisticState {
|
|
|
201
207
|
retryErrors: Record<string, unknown>;
|
|
202
208
|
requestsWithStatusCode: Record<string, number>;
|
|
203
209
|
}
|
|
204
|
-
//# sourceMappingURL=statistics.d.ts.map
|
package/crawlers/statistics.js
CHANGED
|
@@ -195,6 +195,14 @@ export class Statistics {
|
|
|
195
195
|
this._saveRetryCountForJob(retryCount);
|
|
196
196
|
this.requestsInProgress.delete(id);
|
|
197
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Discards a started job without affecting the finished/failed counters, e.g. when a request
|
|
200
|
+
* turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
|
|
201
|
+
* @ignore
|
|
202
|
+
*/
|
|
203
|
+
discardJob(id) {
|
|
204
|
+
this.requestsInProgress.delete(id);
|
|
205
|
+
}
|
|
198
206
|
/**
|
|
199
207
|
* Calculate the current statistics
|
|
200
208
|
*/
|
|
@@ -340,4 +348,3 @@ export class Statistics {
|
|
|
340
348
|
return result;
|
|
341
349
|
}
|
|
342
350
|
}
|
|
343
|
-
//# sourceMappingURL=statistics.js.map
|
|
@@ -147,6 +147,14 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
147
147
|
* If provided, disallowed URLs will be ignored.
|
|
148
148
|
*/
|
|
149
149
|
robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
|
|
150
|
+
/**
|
|
151
|
+
* Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
|
|
152
|
+
* `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
|
|
153
|
+
* {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
|
|
154
|
+
*/
|
|
155
|
+
respectRobotsTxtFile?: boolean | {
|
|
156
|
+
userAgent?: string;
|
|
157
|
+
};
|
|
150
158
|
/**
|
|
151
159
|
* When a request is skipped for some reason, you can use this callback to act on it.
|
|
152
160
|
* This is currently fired for requests skipped
|
|
@@ -254,4 +262,3 @@ export interface ResolveBaseUrl {
|
|
|
254
262
|
originalRequestUrl: string;
|
|
255
263
|
finalRequestUrl?: string;
|
|
256
264
|
}
|
|
257
|
-
//# sourceMappingURL=enqueue_links.d.ts.map
|
|
@@ -90,6 +90,7 @@ export async function enqueueLinks(options) {
|
|
|
90
90
|
urls: ow.array.ofType(ow.string),
|
|
91
91
|
requestManager: ow.object.hasKeys('addRequestsBatched'),
|
|
92
92
|
robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
|
|
93
|
+
respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
|
|
93
94
|
onSkippedRequest: ow.optional.function,
|
|
94
95
|
forefront: ow.optional.boolean,
|
|
95
96
|
skipNavigation: ow.optional.boolean,
|
|
@@ -182,10 +183,11 @@ export async function enqueueLinks(options) {
|
|
|
182
183
|
}
|
|
183
184
|
}
|
|
184
185
|
let requestOptions = createRequestOptions(urls, options);
|
|
185
|
-
if (robotsTxtFile) {
|
|
186
|
+
if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
|
|
187
|
+
const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
|
|
186
188
|
const skippedRequests = [];
|
|
187
189
|
requestOptions = requestOptions.filter((request) => {
|
|
188
|
-
if (robotsTxtFile.isAllowed(request.url)) {
|
|
190
|
+
if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
|
|
189
191
|
return true;
|
|
190
192
|
}
|
|
191
193
|
skippedRequests.push(request);
|
|
@@ -217,15 +219,14 @@ export async function enqueueLinks(options) {
|
|
|
217
219
|
// Step 3: Create Request instances from the final request options
|
|
218
220
|
return filteredOptions.map((opts) => new Request(opts));
|
|
219
221
|
}
|
|
220
|
-
|
|
221
|
-
if (typeof limit === 'number' && limit < requests.length) {
|
|
222
|
-
await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
|
|
223
|
-
requests = requests.slice(0, limit);
|
|
224
|
-
}
|
|
225
|
-
const { addedRequests } = await requestManager.addRequestsBatched(requests, {
|
|
222
|
+
const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
|
|
226
223
|
forefront,
|
|
227
224
|
waitForAllRequestsToBeAdded,
|
|
225
|
+
maxNewRequests: limit,
|
|
228
226
|
});
|
|
227
|
+
if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
|
|
228
|
+
await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
|
|
229
|
+
}
|
|
229
230
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
230
231
|
}
|
|
231
232
|
/**
|
|
@@ -268,4 +269,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
|
|
|
268
269
|
function ignoreHttpSchema(pattern) {
|
|
269
270
|
return pattern.replace(/^(https?):\/\//, 'http{s,}://');
|
|
270
271
|
}
|
|
271
|
-
//# sourceMappingURL=enqueue_links.js.map
|
package/enqueue_links/index.d.ts
CHANGED
package/enqueue_links/index.js
CHANGED
|
@@ -81,4 +81,3 @@ export interface RequestTransform {
|
|
|
81
81
|
* @internal
|
|
82
82
|
*/
|
|
83
83
|
export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
|
|
84
|
-
//# sourceMappingURL=shared.d.ts.map
|
package/enqueue_links/shared.js
CHANGED
package/errors.d.ts
CHANGED
|
@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
|
|
|
14
14
|
*/
|
|
15
15
|
export declare class MissingRouteError extends CriticalError {
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
19
|
+
*
|
|
20
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
21
|
+
*/
|
|
22
|
+
export declare class RequestValidationError extends NonRetryableError {
|
|
23
|
+
readonly label: string | symbol;
|
|
24
|
+
readonly issues: readonly {
|
|
25
|
+
readonly message: string;
|
|
26
|
+
readonly path?: readonly (PropertyKey | {
|
|
27
|
+
key: PropertyKey;
|
|
28
|
+
})[];
|
|
29
|
+
}[];
|
|
30
|
+
constructor(label: string | symbol, issues: readonly {
|
|
31
|
+
readonly message: string;
|
|
32
|
+
readonly path?: readonly (PropertyKey | {
|
|
33
|
+
key: PropertyKey;
|
|
34
|
+
})[];
|
|
35
|
+
}[]);
|
|
36
|
+
}
|
|
17
37
|
/**
|
|
18
38
|
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
19
39
|
*
|
|
@@ -60,4 +80,3 @@ export declare class ServiceConflictError extends Error {
|
|
|
60
80
|
*/
|
|
61
81
|
export declare class NavigationSkippedError extends NonRetryableError {
|
|
62
82
|
}
|
|
63
|
-
//# sourceMappingURL=errors.d.ts.map
|
package/errors.js
CHANGED
|
@@ -15,6 +15,28 @@ export class CriticalError extends NonRetryableError {
|
|
|
15
15
|
*/
|
|
16
16
|
export class MissingRouteError extends CriticalError {
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
20
|
+
*
|
|
21
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
22
|
+
*/
|
|
23
|
+
export class RequestValidationError extends NonRetryableError {
|
|
24
|
+
label;
|
|
25
|
+
issues;
|
|
26
|
+
constructor(label, issues) {
|
|
27
|
+
const details = issues
|
|
28
|
+
.map((issue) => {
|
|
29
|
+
const path = (issue.path ?? [])
|
|
30
|
+
.map((segment) => (typeof segment === 'object' ? segment.key : segment))
|
|
31
|
+
.join('.');
|
|
32
|
+
return `- ${path ? `${path}: ` : ''}${issue.message}`;
|
|
33
|
+
})
|
|
34
|
+
.join('\n');
|
|
35
|
+
super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
|
|
36
|
+
this.label = label;
|
|
37
|
+
this.issues = issues;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
18
40
|
/**
|
|
19
41
|
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
20
42
|
*
|
|
@@ -78,4 +100,3 @@ export class ServiceConflictError extends Error {
|
|
|
78
100
|
*/
|
|
79
101
|
export class NavigationSkippedError extends NonRetryableError {
|
|
80
102
|
}
|
|
81
|
-
//# sourceMappingURL=errors.js.map
|
package/events/event_manager.js
CHANGED
package/events/index.d.ts
CHANGED
package/events/index.js
CHANGED
|
@@ -80,12 +80,9 @@ export class LocalEventManager extends EventManager {
|
|
|
80
80
|
}
|
|
81
81
|
async createMemoryInfo() {
|
|
82
82
|
try {
|
|
83
|
-
const
|
|
84
|
-
const memInfo = await getMemoryInfo({
|
|
85
|
-
containerized: await this.isContainerizedWrapper(),
|
|
86
|
-
logger: serviceLocator.getLogger(),
|
|
87
|
-
});
|
|
83
|
+
const memInfo = await this.getMemoryInfo();
|
|
88
84
|
return {
|
|
85
|
+
memTotalBytes: memInfo.totalBytes,
|
|
89
86
|
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
90
87
|
};
|
|
91
88
|
}
|
|
@@ -94,5 +91,11 @@ export class LocalEventManager extends EventManager {
|
|
|
94
91
|
return {};
|
|
95
92
|
}
|
|
96
93
|
}
|
|
94
|
+
async getMemoryInfo() {
|
|
95
|
+
const { getMemoryInfo } = await import('@crawlee/utils');
|
|
96
|
+
return getMemoryInfo({
|
|
97
|
+
containerized: await this.isContainerizedWrapper(),
|
|
98
|
+
logger: serviceLocator.getLogger(),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
97
101
|
}
|
|
98
|
-
//# sourceMappingURL=local_event_manager.js.map
|
package/index.d.ts
CHANGED
|
@@ -18,4 +18,3 @@ export * from './cookie_utils.js';
|
|
|
18
18
|
export * from './recoverable_state.js';
|
|
19
19
|
export { PseudoUrl } from '@apify/pseudo_url';
|
|
20
20
|
export type { Dictionary, Awaitable, Constructor, StorageBackend, Cookie, QueueOperationInfo } from '@crawlee/types';
|
|
21
|
-
//# sourceMappingURL=index.d.ts.map
|
package/index.js
CHANGED
package/log.d.ts
CHANGED
package/log.js
CHANGED
package/memory-storage/consts.js
CHANGED
package/memory-storage/index.js
CHANGED