@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +14 -15
- package/autoscaling/autoscaled_pool.js +37 -36
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +102 -0
- package/autoscaling/snapshotter.d.ts +29 -91
- package/autoscaling/snapshotter.js +65 -204
- package/autoscaling/system_status.d.ts +22 -37
- package/autoscaling/system_status.js +49 -87
- package/configuration.d.ts +88 -228
- package/configuration.js +162 -224
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +20 -13
- package/crawlers/context_pipeline.d.ts +70 -0
- package/crawlers/context_pipeline.js +122 -0
- package/crawlers/crawler_commons.d.ts +20 -30
- package/crawlers/crawler_commons.js +12 -21
- package/crawlers/crawler_utils.d.ts +2 -3
- package/crawlers/crawler_utils.js +1 -2
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.js +2 -3
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -2
- package/crawlers/index.js +1 -2
- package/crawlers/internals/types.d.ts +7 -0
- package/crawlers/internals/types.js +1 -0
- package/crawlers/statistics.d.ts +28 -23
- package/crawlers/statistics.js +38 -34
- package/enqueue_links/enqueue_links.d.ts +44 -22
- package/enqueue_links/enqueue_links.js +51 -30
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +25 -9
- package/enqueue_links/shared.js +69 -38
- package/errors.d.ts +53 -4
- package/errors.js +70 -5
- package/events/event_manager.d.ts +34 -8
- package/events/event_manager.js +8 -10
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -5
- package/events/local_event_manager.js +37 -40
- package/index.d.ts +4 -3
- package/index.js +3 -2
- package/log.d.ts +82 -3
- package/log.js +102 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +46 -0
- package/memory-storage/memory-storage.js +136 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +113 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +203 -0
- package/memory-storage/resource-clients/request-queue.d.ts +96 -0
- package/memory-storage/resource-clients/request-queue.js +421 -0
- package/memory-storage/utils.d.ts +17 -0
- package/memory-storage/utils.js +42 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +29 -152
- package/proxy_configuration.js +21 -173
- package/recoverable_state.d.ts +120 -0
- package/recoverable_state.js +141 -0
- package/request.d.ts +84 -13
- package/request.js +107 -28
- package/router.d.ts +118 -16
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +1 -2
- package/service_locator.d.ts +156 -0
- package/service_locator.js +238 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +37 -75
- package/session_pool/session.js +49 -102
- package/session_pool/session_pool.d.ts +85 -90
- package/session_pool/session_pool.js +131 -120
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +5 -2
- package/storages/dataset.d.ts +101 -51
- package/storages/dataset.js +168 -126
- package/storages/index.d.ts +8 -7
- package/storages/index.js +6 -7
- package/storages/key_value_store.d.ts +165 -36
- package/storages/key_value_store.js +269 -122
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +53 -97
- package/storages/request_list.js +99 -75
- package/storages/request_loader.d.ts +96 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +106 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +290 -47
- package/storages/request_queue.js +628 -215
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.js +258 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/utils.d.ts +51 -6
- package/storages/utils.js +60 -9
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +1 -2
- package/typedefs.js +0 -1
- package/validators.d.ts +8 -1
- package/validators.js +10 -3
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
|
|
2
1
|
import ow from 'ow';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
2
|
+
import { serviceLocator } from '../service_locator.js';
|
|
3
|
+
import { createClientLoadSignal } from './client_load_signal.js';
|
|
4
|
+
import { createCpuLoadSignal } from './cpu_load_signal.js';
|
|
5
|
+
import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
6
|
+
import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
9
7
|
/**
|
|
10
8
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
11
9
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -29,27 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
29
27
|
*
|
|
30
28
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
31
29
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
30
|
+
*
|
|
32
31
|
* @category Scaling
|
|
33
32
|
*/
|
|
34
33
|
export class Snapshotter {
|
|
35
34
|
log;
|
|
36
35
|
client;
|
|
37
36
|
config;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
37
|
+
memorySignal;
|
|
38
|
+
eventLoopSignal;
|
|
39
|
+
cpuSignal;
|
|
40
|
+
clientSignal;
|
|
41
|
+
/**
|
|
42
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
43
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
44
|
+
*/
|
|
45
|
+
getLoadSignals() {
|
|
46
|
+
return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
|
|
47
|
+
}
|
|
48
|
+
// Legacy public properties kept for backward compat (tests read these directly)
|
|
49
|
+
get cpuSnapshots() {
|
|
50
|
+
return this.cpuSignal.store.getAll();
|
|
51
|
+
}
|
|
52
|
+
get eventLoopSnapshots() {
|
|
53
|
+
return this.eventLoopSignal.store.getAll();
|
|
54
|
+
}
|
|
55
|
+
get memorySnapshots() {
|
|
56
|
+
return this.memorySignal.getMemorySnapshots();
|
|
57
|
+
}
|
|
58
|
+
get clientSnapshots() {
|
|
59
|
+
return this.clientSignal.store.getAll();
|
|
60
|
+
}
|
|
53
61
|
/**
|
|
54
62
|
* @param [options] All `Snapshotter` configuration options.
|
|
55
63
|
*/
|
|
@@ -65,58 +73,50 @@ export class Snapshotter {
|
|
|
65
73
|
client: ow.optional.object,
|
|
66
74
|
config: ow.optional.object,
|
|
67
75
|
}));
|
|
68
|
-
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log =
|
|
76
|
+
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
|
|
69
77
|
this.log = log.child({ prefix: 'Snapshotter' });
|
|
70
78
|
this.client = client;
|
|
71
79
|
this.config = config;
|
|
72
|
-
|
|
73
|
-
this.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
81
|
+
this.memorySignal = new MemoryLoadSignal({
|
|
82
|
+
maxUsedMemoryRatio,
|
|
83
|
+
snapshotHistoryMillis,
|
|
84
|
+
config: this.config,
|
|
85
|
+
log: this.log,
|
|
86
|
+
});
|
|
87
|
+
this.eventLoopSignal = createEventLoopLoadSignal({
|
|
88
|
+
eventLoopSnapshotIntervalSecs,
|
|
89
|
+
maxBlockedMillis,
|
|
90
|
+
snapshotHistoryMillis,
|
|
91
|
+
});
|
|
92
|
+
this.cpuSignal = createCpuLoadSignal({
|
|
93
|
+
snapshotHistoryMillis,
|
|
94
|
+
config: this.config,
|
|
95
|
+
});
|
|
96
|
+
this.clientSignal = createClientLoadSignal({
|
|
97
|
+
client: this.client,
|
|
98
|
+
clientSnapshotIntervalSecs,
|
|
99
|
+
maxClientErrors,
|
|
100
|
+
snapshotHistoryMillis,
|
|
101
|
+
});
|
|
82
102
|
}
|
|
83
103
|
/**
|
|
84
104
|
* Starts capturing snapshots at configured intervals.
|
|
85
105
|
*/
|
|
86
106
|
async start() {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
else {
|
|
92
|
-
let totalBytes;
|
|
93
|
-
if (this.config.get('systemInfoV2')) {
|
|
94
|
-
const containerized = this.config.get('containerized', await isContainerized());
|
|
95
|
-
const memInfo = await getMemoryInfoV2(containerized);
|
|
96
|
-
totalBytes = memInfo.totalBytes;
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
const memInfo = await getMemoryInfo();
|
|
100
|
-
totalBytes = memInfo.totalBytes;
|
|
101
|
-
}
|
|
102
|
-
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
|
|
103
|
-
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
104
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
105
|
-
}
|
|
106
|
-
// Start snapshotting.
|
|
107
|
-
this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
|
|
108
|
-
this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
|
|
109
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
|
|
110
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
107
|
+
await this.memorySignal.start();
|
|
108
|
+
await this.eventLoopSignal.start();
|
|
109
|
+
await this.cpuSignal.start();
|
|
110
|
+
await this.clientSignal.start();
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
113
|
* Stops all resource capturing.
|
|
114
114
|
*/
|
|
115
115
|
async stop() {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
this.
|
|
119
|
-
this.
|
|
116
|
+
await this.memorySignal.stop();
|
|
117
|
+
await this.eventLoopSignal.stop();
|
|
118
|
+
await this.cpuSignal.stop();
|
|
119
|
+
await this.clientSignal.stop();
|
|
120
120
|
// Allow microtask queue to unwind before stop returns.
|
|
121
121
|
await new Promise((resolve) => {
|
|
122
122
|
setImmediate(resolve);
|
|
@@ -127,166 +127,27 @@ export class Snapshotter {
|
|
|
127
127
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
128
128
|
*/
|
|
129
129
|
getMemorySample(sampleDurationMillis) {
|
|
130
|
-
return this.
|
|
130
|
+
return this.memorySignal.getSample(sampleDurationMillis);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
133
|
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
134
134
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
135
135
|
*/
|
|
136
136
|
getEventLoopSample(sampleDurationMillis) {
|
|
137
|
-
return this.
|
|
137
|
+
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
138
138
|
}
|
|
139
139
|
/**
|
|
140
140
|
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
141
141
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
142
142
|
*/
|
|
143
143
|
getCpuSample(sampleDurationMillis) {
|
|
144
|
-
return this.
|
|
144
|
+
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
145
145
|
}
|
|
146
146
|
/**
|
|
147
147
|
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
148
148
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
149
149
|
*/
|
|
150
150
|
getClientSample(sampleDurationMillis) {
|
|
151
|
-
return this.
|
|
152
|
-
}
|
|
153
|
-
/**
|
|
154
|
-
* Finds the latest snapshots by sampleDurationMillis in the provided array.
|
|
155
|
-
*/
|
|
156
|
-
_getSample(snapshots, sampleDurationMillis) {
|
|
157
|
-
if (!sampleDurationMillis)
|
|
158
|
-
return snapshots;
|
|
159
|
-
const sample = [];
|
|
160
|
-
let idx = snapshots.length;
|
|
161
|
-
if (!idx)
|
|
162
|
-
return sample;
|
|
163
|
-
const latestTime = snapshots[idx - 1].createdAt;
|
|
164
|
-
while (idx--) {
|
|
165
|
-
const snapshot = snapshots[idx];
|
|
166
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
167
|
-
sample.unshift(snapshot);
|
|
168
|
-
}
|
|
169
|
-
else {
|
|
170
|
-
break;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
return sample;
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Creates a snapshot of current memory usage
|
|
177
|
-
* using the Apify platform `systemInfo` event.
|
|
178
|
-
*/
|
|
179
|
-
_snapshotMemory(systemInfo) {
|
|
180
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
181
|
-
this._pruneSnapshots(this.memorySnapshots, createdAt);
|
|
182
|
-
const { memCurrentBytes } = systemInfo;
|
|
183
|
-
const snapshot = {
|
|
184
|
-
createdAt,
|
|
185
|
-
isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
186
|
-
usedBytes: memCurrentBytes,
|
|
187
|
-
};
|
|
188
|
-
this.memorySnapshots.push(snapshot);
|
|
189
|
-
this._memoryOverloadWarning(systemInfo);
|
|
190
|
-
}
|
|
191
|
-
/**
|
|
192
|
-
* Checks for critical memory overload and logs it to the console.
|
|
193
|
-
*/
|
|
194
|
-
_memoryOverloadWarning(systemInfo) {
|
|
195
|
-
const { memCurrentBytes } = systemInfo;
|
|
196
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
197
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
198
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
199
|
-
return;
|
|
200
|
-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
|
|
201
|
-
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
202
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
203
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
204
|
-
if (isCriticalOverload) {
|
|
205
|
-
const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
|
|
206
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
207
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
208
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
209
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* Creates a snapshot of current event loop delay.
|
|
214
|
-
*/
|
|
215
|
-
_snapshotEventLoop(intervalCallback) {
|
|
216
|
-
const now = new Date();
|
|
217
|
-
this._pruneSnapshots(this.eventLoopSnapshots, now);
|
|
218
|
-
const snapshot = {
|
|
219
|
-
createdAt: now,
|
|
220
|
-
isOverloaded: false,
|
|
221
|
-
exceededMillis: 0,
|
|
222
|
-
};
|
|
223
|
-
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
|
|
224
|
-
if (previousSnapshot) {
|
|
225
|
-
const { createdAt } = previousSnapshot;
|
|
226
|
-
const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
|
|
227
|
-
if (delta > this.maxBlockedMillis)
|
|
228
|
-
snapshot.isOverloaded = true;
|
|
229
|
-
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
230
|
-
}
|
|
231
|
-
this.eventLoopSnapshots.push(snapshot);
|
|
232
|
-
intervalCallback();
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
|
|
236
|
-
*/
|
|
237
|
-
_snapshotCpu(systemInfo) {
|
|
238
|
-
const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
|
|
239
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
240
|
-
this._pruneSnapshots(this.cpuSnapshots, createdAt);
|
|
241
|
-
this.cpuSnapshots.push({
|
|
242
|
-
createdAt,
|
|
243
|
-
isOverloaded: isCpuOverloaded,
|
|
244
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
245
|
-
});
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Creates a snapshot of current API state by checking for
|
|
249
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
250
|
-
* of the API call are considered for snapshotting since
|
|
251
|
-
* earlier errors may just be caused by a random spike in
|
|
252
|
-
* number of requests and do not necessarily signify API
|
|
253
|
-
* overloading.
|
|
254
|
-
*/
|
|
255
|
-
_snapshotClient(intervalCallback) {
|
|
256
|
-
const now = new Date();
|
|
257
|
-
this._pruneSnapshots(this.clientSnapshots, now);
|
|
258
|
-
const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
|
|
259
|
-
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
260
|
-
// Handle empty snapshots array
|
|
261
|
-
const snapshot = {
|
|
262
|
-
createdAt: now,
|
|
263
|
-
isOverloaded: false,
|
|
264
|
-
rateLimitErrorCount: currentErrCount,
|
|
265
|
-
};
|
|
266
|
-
const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
|
|
267
|
-
if (previousSnapshot) {
|
|
268
|
-
const { rateLimitErrorCount } = previousSnapshot;
|
|
269
|
-
const delta = currentErrCount - rateLimitErrorCount;
|
|
270
|
-
if (delta > this.maxClientErrors)
|
|
271
|
-
snapshot.isOverloaded = true;
|
|
272
|
-
}
|
|
273
|
-
this.clientSnapshots.push(snapshot);
|
|
274
|
-
intervalCallback();
|
|
275
|
-
}
|
|
276
|
-
/**
|
|
277
|
-
* Removes snapshots that are older than the snapshotHistorySecs option
|
|
278
|
-
* from the array (destructively - in place).
|
|
279
|
-
*/
|
|
280
|
-
_pruneSnapshots(snapshots, now) {
|
|
281
|
-
let oldCount = 0;
|
|
282
|
-
for (let i = 0; i < snapshots.length; i++) {
|
|
283
|
-
const { createdAt } = snapshots[i];
|
|
284
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
|
|
285
|
-
oldCount++;
|
|
286
|
-
else
|
|
287
|
-
break;
|
|
288
|
-
}
|
|
289
|
-
snapshots.splice(0, oldCount);
|
|
151
|
+
return this.clientSignal.getSample(sampleDurationMillis);
|
|
290
152
|
}
|
|
291
153
|
}
|
|
292
|
-
//# sourceMappingURL=snapshotter.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { LoadSignal } from './load_signal.js';
|
|
2
2
|
import { Snapshotter } from './snapshotter.js';
|
|
3
3
|
/**
|
|
4
4
|
* Represents the current status of the system.
|
|
@@ -10,6 +10,7 @@ export interface SystemInfo {
|
|
|
10
10
|
eventLoopInfo: ClientInfo;
|
|
11
11
|
cpuInfo: ClientInfo;
|
|
12
12
|
clientInfo: ClientInfo;
|
|
13
|
+
memTotalBytes?: number;
|
|
13
14
|
memCurrentBytes?: number;
|
|
14
15
|
/**
|
|
15
16
|
* Platform only property
|
|
@@ -26,6 +27,11 @@ export interface SystemInfo {
|
|
|
26
27
|
* @internal
|
|
27
28
|
*/
|
|
28
29
|
createdAt?: Date;
|
|
30
|
+
/**
|
|
31
|
+
* Status of additional load signals beyond the built-in four.
|
|
32
|
+
* Keys are `LoadSignal.name` values, values are overload info.
|
|
33
|
+
*/
|
|
34
|
+
loadSignalInfo?: Record<string, ClientInfo>;
|
|
29
35
|
}
|
|
30
36
|
export interface SystemStatusOptions {
|
|
31
37
|
/**
|
|
@@ -61,8 +67,13 @@ export interface SystemStatusOptions {
|
|
|
61
67
|
* The `Snapshotter` instance to be queried for `SystemStatus`.
|
|
62
68
|
*/
|
|
63
69
|
snapshotter?: Snapshotter;
|
|
64
|
-
/**
|
|
65
|
-
|
|
70
|
+
/**
|
|
71
|
+
* Additional load signals to include in the system status evaluation.
|
|
72
|
+
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
73
|
+
* and client signals. If any signal reports overload, the system is
|
|
74
|
+
* considered overloaded.
|
|
75
|
+
*/
|
|
76
|
+
loadSignals?: LoadSignal[];
|
|
66
77
|
}
|
|
67
78
|
export interface ClientInfo {
|
|
68
79
|
isOverloaded: boolean;
|
|
@@ -105,11 +116,14 @@ export interface FinalStatistics {
|
|
|
105
116
|
*/
|
|
106
117
|
export declare class SystemStatus {
|
|
107
118
|
private readonly currentHistoryMillis;
|
|
108
|
-
private readonly maxMemoryOverloadedRatio;
|
|
109
|
-
private readonly maxEventLoopOverloadedRatio;
|
|
110
|
-
private readonly maxCpuOverloadedRatio;
|
|
111
|
-
private readonly maxClientOverloadedRatio;
|
|
112
119
|
private readonly snapshotter;
|
|
120
|
+
private readonly signals;
|
|
121
|
+
/**
|
|
122
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
123
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
124
|
+
* `overloadedRatio`.
|
|
125
|
+
*/
|
|
126
|
+
private ratioOverrides;
|
|
113
127
|
constructor(options?: SystemStatusOptions);
|
|
114
128
|
/**
|
|
115
129
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -148,34 +162,5 @@ export declare class SystemStatus {
|
|
|
148
162
|
/**
|
|
149
163
|
* Returns a system status object.
|
|
150
164
|
*/
|
|
151
|
-
|
|
152
|
-
/**
|
|
153
|
-
* Returns an object with an isOverloaded property set to true
|
|
154
|
-
* if the memory has been overloaded in the last sampleDurationMillis.
|
|
155
|
-
*/
|
|
156
|
-
protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
157
|
-
/**
|
|
158
|
-
* Returns an object with an isOverloaded property set to true
|
|
159
|
-
* if the event loop has been overloaded in the last sampleDurationMillis.
|
|
160
|
-
*/
|
|
161
|
-
protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
162
|
-
/**
|
|
163
|
-
* Returns an object with an isOverloaded property set to true
|
|
164
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
165
|
-
*/
|
|
166
|
-
protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
167
|
-
/**
|
|
168
|
-
* Returns an object with an isOverloaded property set to true
|
|
169
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
170
|
-
*/
|
|
171
|
-
protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
172
|
-
/**
|
|
173
|
-
* Returns an object with sample information and an isOverloaded property
|
|
174
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
175
|
-
*/
|
|
176
|
-
protected _isSampleOverloaded<T extends {
|
|
177
|
-
createdAt: Date;
|
|
178
|
-
isOverloaded: boolean;
|
|
179
|
-
}>(sample: T[], ratio: number): ClientInfo;
|
|
165
|
+
private isSystemIdle;
|
|
180
166
|
}
|
|
181
|
-
//# sourceMappingURL=system_status.d.ts.map
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import { weightedAvg } from '@crawlee/utils';
|
|
2
1
|
import ow from 'ow';
|
|
2
|
+
import { evaluateLoadSignalSample } from './load_signal.js';
|
|
3
3
|
import { Snapshotter } from './snapshotter.js';
|
|
4
|
+
/** The four built-in signal names that map to typed `SystemInfo` fields. */
|
|
5
|
+
const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
|
|
4
6
|
/**
|
|
5
7
|
* Provides a simple interface to reading system status from a {@link Snapshotter} instance.
|
|
6
8
|
* It only exposes two functions {@link SystemStatus.getCurrentStatus}
|
|
@@ -25,11 +27,14 @@ import { Snapshotter } from './snapshotter.js';
|
|
|
25
27
|
*/
|
|
26
28
|
export class SystemStatus {
|
|
27
29
|
currentHistoryMillis;
|
|
28
|
-
maxMemoryOverloadedRatio;
|
|
29
|
-
maxEventLoopOverloadedRatio;
|
|
30
|
-
maxCpuOverloadedRatio;
|
|
31
|
-
maxClientOverloadedRatio;
|
|
32
30
|
snapshotter;
|
|
31
|
+
signals;
|
|
32
|
+
/**
|
|
33
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
34
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
35
|
+
* `overloadedRatio`.
|
|
36
|
+
*/
|
|
37
|
+
ratioOverrides;
|
|
33
38
|
constructor(options = {}) {
|
|
34
39
|
ow(options, ow.object.exactShape({
|
|
35
40
|
currentHistorySecs: ow.optional.number,
|
|
@@ -38,15 +43,20 @@ export class SystemStatus {
|
|
|
38
43
|
maxCpuOverloadedRatio: ow.optional.number,
|
|
39
44
|
maxClientOverloadedRatio: ow.optional.number,
|
|
40
45
|
snapshotter: ow.optional.object,
|
|
41
|
-
|
|
46
|
+
loadSignals: ow.optional.array,
|
|
42
47
|
}));
|
|
43
|
-
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter,
|
|
48
|
+
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
|
|
44
49
|
this.currentHistoryMillis = currentHistorySecs * 1000;
|
|
45
|
-
this.
|
|
46
|
-
|
|
47
|
-
this.
|
|
48
|
-
|
|
49
|
-
this.
|
|
50
|
+
this.snapshotter = snapshotter || new Snapshotter();
|
|
51
|
+
// Built-in signals from the snapshotter + any custom signals
|
|
52
|
+
this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
|
|
53
|
+
// Allow legacy options to override the built-in signal ratios
|
|
54
|
+
this.ratioOverrides = {
|
|
55
|
+
memInfo: maxMemoryOverloadedRatio,
|
|
56
|
+
eventLoopInfo: maxEventLoopOverloadedRatio,
|
|
57
|
+
cpuInfo: maxCpuOverloadedRatio,
|
|
58
|
+
clientInfo: maxClientOverloadedRatio,
|
|
59
|
+
};
|
|
50
60
|
}
|
|
51
61
|
/**
|
|
52
62
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -65,7 +75,7 @@ export class SystemStatus {
|
|
|
65
75
|
* and `true` otherwise.
|
|
66
76
|
*/
|
|
67
77
|
getCurrentStatus() {
|
|
68
|
-
return this.
|
|
78
|
+
return this.isSystemIdle(this.currentHistoryMillis);
|
|
69
79
|
}
|
|
70
80
|
/**
|
|
71
81
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -84,86 +94,38 @@ export class SystemStatus {
|
|
|
84
94
|
* (which is configurable in the {@link Snapshotter}) and `true` otherwise.
|
|
85
95
|
*/
|
|
86
96
|
getHistoricalStatus() {
|
|
87
|
-
return this.
|
|
97
|
+
return this.isSystemIdle();
|
|
88
98
|
}
|
|
89
99
|
/**
|
|
90
100
|
* Returns a system status object.
|
|
91
101
|
*/
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
!eventLoopInfo.isOverloaded &&
|
|
100
|
-
!cpuInfo.isOverloaded &&
|
|
101
|
-
!clientInfo.isOverloaded,
|
|
102
|
-
memInfo,
|
|
103
|
-
eventLoopInfo,
|
|
104
|
-
cpuInfo,
|
|
105
|
-
clientInfo,
|
|
102
|
+
isSystemIdle(sampleDurationMillis) {
|
|
103
|
+
const result = {
|
|
104
|
+
isSystemIdle: true,
|
|
105
|
+
memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
106
|
+
eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
107
|
+
cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
108
|
+
clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
106
109
|
};
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
|
|
123
|
-
}
|
|
124
|
-
/**
|
|
125
|
-
* Returns an object with an isOverloaded property set to true
|
|
126
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
127
|
-
*/
|
|
128
|
-
_isCpuOverloaded(sampleDurationMillis) {
|
|
129
|
-
const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
|
|
130
|
-
return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* Returns an object with an isOverloaded property set to true
|
|
134
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
135
|
-
*/
|
|
136
|
-
_isClientOverloaded(sampleDurationMillis) {
|
|
137
|
-
const sample = this.snapshotter.getClientSample(sampleDurationMillis);
|
|
138
|
-
return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* Returns an object with sample information and an isOverloaded property
|
|
142
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
143
|
-
*/
|
|
144
|
-
_isSampleOverloaded(sample, ratio) {
|
|
145
|
-
if (sample.length === 0) {
|
|
146
|
-
return {
|
|
147
|
-
isOverloaded: false,
|
|
148
|
-
limitRatio: ratio,
|
|
149
|
-
actualRatio: 0,
|
|
150
|
-
};
|
|
110
|
+
let loadSignalInfo;
|
|
111
|
+
for (const signal of this.signals) {
|
|
112
|
+
const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
|
|
113
|
+
const sample = signal.getSample(sampleDurationMillis);
|
|
114
|
+
const info = evaluateLoadSignalSample(sample, ratio);
|
|
115
|
+
if (info.isOverloaded) {
|
|
116
|
+
result.isSystemIdle = false;
|
|
117
|
+
}
|
|
118
|
+
if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
|
|
119
|
+
result[signal.name] = info;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
loadSignalInfo ??= {};
|
|
123
|
+
loadSignalInfo[signal.name] = info;
|
|
124
|
+
}
|
|
151
125
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
for (let i = 1; i < sample.length; i++) {
|
|
155
|
-
const previous = sample[i - 1];
|
|
156
|
-
const current = sample[i];
|
|
157
|
-
const weight = +current.createdAt - +previous.createdAt;
|
|
158
|
-
weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
|
|
159
|
-
values.push(+current.isOverloaded);
|
|
126
|
+
if (loadSignalInfo) {
|
|
127
|
+
result.loadSignalInfo = loadSignalInfo;
|
|
160
128
|
}
|
|
161
|
-
|
|
162
|
-
return {
|
|
163
|
-
isOverloaded: wAvg > ratio,
|
|
164
|
-
limitRatio: ratio,
|
|
165
|
-
actualRatio: Math.round(wAvg * 1000) / 1000,
|
|
166
|
-
};
|
|
129
|
+
return result;
|
|
167
130
|
}
|
|
168
131
|
}
|
|
169
|
-
//# sourceMappingURL=system_status.js.map
|