@crawlee/core 4.0.0-beta.71 → 4.0.0-beta.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -1
- package/autoscaling/autoscaled_pool.js +9 -2
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +101 -0
- package/autoscaling/snapshotter.d.ts +36 -63
- package/autoscaling/snapshotter.js +83 -175
- package/autoscaling/system_status.d.ts +21 -33
- package/autoscaling/system_status.js +45 -82
- package/configuration.d.ts +0 -1
- package/configuration.js +0 -1
- package/cookie_utils.d.ts +0 -1
- package/cookie_utils.js +2 -1
- package/crawlers/context_pipeline.d.ts +0 -1
- package/crawlers/context_pipeline.js +0 -1
- package/crawlers/crawler_commons.d.ts +0 -1
- package/crawlers/crawler_commons.js +0 -1
- package/crawlers/crawler_utils.d.ts +0 -1
- package/crawlers/crawler_utils.js +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -1
- package/crawlers/error_snapshotter.js +0 -1
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +0 -1
- package/crawlers/index.js +0 -1
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +6 -1
- package/crawlers/statistics.js +8 -1
- package/enqueue_links/enqueue_links.d.ts +8 -1
- package/enqueue_links/enqueue_links.js +9 -9
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +0 -1
- package/enqueue_links/shared.js +0 -1
- package/errors.d.ts +20 -1
- package/errors.js +22 -1
- package/events/event_manager.d.ts +0 -1
- package/events/event_manager.js +0 -1
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +9 -6
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/log.d.ts +0 -1
- package/log.js +0 -1
- package/memory-storage/consts.d.ts +0 -1
- package/memory-storage/consts.js +0 -1
- package/memory-storage/index.d.ts +0 -1
- package/memory-storage/index.js +0 -1
- package/memory-storage/memory-storage.d.ts +0 -1
- package/memory-storage/memory-storage.js +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
- package/memory-storage/resource-clients/common/base-client.js +0 -1
- package/memory-storage/resource-clients/dataset.d.ts +0 -1
- package/memory-storage/resource-clients/dataset.js +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
- package/memory-storage/resource-clients/key-value-store.js +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts +0 -1
- package/memory-storage/resource-clients/request-queue.js +0 -1
- package/memory-storage/utils.d.ts +6 -1
- package/memory-storage/utils.js +15 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts +0 -1
- package/proxy_configuration.js +0 -1
- package/recoverable_state.d.ts +0 -1
- package/recoverable_state.js +0 -1
- package/request.d.ts +0 -1
- package/request.js +0 -1
- package/router.d.ts +117 -15
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +0 -1
- package/service_locator.d.ts +0 -1
- package/service_locator.js +0 -1
- package/session_pool/consts.d.ts +0 -1
- package/session_pool/consts.js +0 -1
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +0 -1
- package/session_pool/fingerprint.js +0 -1
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.js +0 -1
- package/session_pool/session.d.ts +0 -1
- package/session_pool/session.js +0 -1
- package/session_pool/session_pool.d.ts +0 -1
- package/session_pool/session_pool.js +0 -1
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +0 -1
- package/storages/dataset.d.ts +1 -2
- package/storages/dataset.js +0 -1
- package/storages/index.d.ts +0 -1
- package/storages/index.js +0 -1
- package/storages/key_value_store.d.ts +0 -1
- package/storages/key_value_store.js +0 -1
- package/storages/key_value_store_codec.d.ts +0 -1
- package/storages/key_value_store_codec.js +0 -1
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +6 -1
- package/storages/request_list.js +17 -3
- package/storages/request_loader.d.ts +0 -1
- package/storages/request_loader.js +0 -1
- package/storages/request_manager.d.ts +0 -1
- package/storages/request_manager.js +0 -1
- package/storages/request_manager_tandem.d.ts +0 -1
- package/storages/request_manager_tandem.js +0 -1
- package/storages/request_queue.d.ts +24 -1
- package/storages/request_queue.js +80 -27
- package/storages/sitemap_request_loader.d.ts +0 -1
- package/storages/sitemap_request_loader.js +0 -1
- package/storages/storage_instance_manager.d.ts +0 -1
- package/storages/storage_instance_manager.js +0 -1
- package/storages/storage_stats.d.ts +0 -1
- package/storages/storage_stats.js +0 -1
- package/storages/utils.d.ts +0 -1
- package/storages/utils.js +0 -1
- package/typedefs.d.ts +0 -1
- package/typedefs.js +0 -1
- package/validators.d.ts +0 -1
- package/validators.js +0 -1
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/memory-storage/consts.d.ts.map +0 -1
- package/memory-storage/consts.js.map +0 -1
- package/memory-storage/index.d.ts.map +0 -1
- package/memory-storage/index.js.map +0 -1
- package/memory-storage/memory-storage.d.ts.map +0 -1
- package/memory-storage/memory-storage.js.map +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
- package/memory-storage/resource-clients/common/base-client.js.map +0 -1
- package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
- package/memory-storage/resource-clients/dataset.js.map +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
- package/memory-storage/resource-clients/key-value-store.js.map +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
- package/memory-storage/resource-clients/request-queue.js.map +0 -1
- package/memory-storage/utils.d.ts.map +0 -1
- package/memory-storage/utils.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/service_locator.d.ts.map +0 -1
- package/service_locator.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/fingerprint.d.ts.map +0 -1
- package/session_pool/fingerprint.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/key_value_store_codec.d.ts.map +0 -1
- package/storages/key_value_store_codec.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_loader.d.ts.map +0 -1
- package/storages/request_loader.js.map +0 -1
- package/storages/request_manager.d.ts.map +0 -1
- package/storages/request_manager.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/sitemap_request_loader.d.ts.map +0 -1
- package/storages/sitemap_request_loader.js.map +0 -1
- package/storages/storage_instance_manager.d.ts.map +0 -1
- package/storages/storage_instance_manager.js.map +0 -1
- package/storages/storage_stats.d.ts.map +0 -1
- package/storages/storage_stats.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { StorageBackend } from '@crawlee/types';
|
|
2
|
+
import type { Configuration } from '../configuration.js';
|
|
2
3
|
import type { CrawleeLogger } from '../log.js';
|
|
4
|
+
import type { ClientSnapshot } from './client_load_signal.js';
|
|
5
|
+
import type { CpuSnapshot } from './cpu_load_signal.js';
|
|
6
|
+
import type { EventLoopSnapshot } from './event_loop_load_signal.js';
|
|
7
|
+
import type { LoadSignal } from './load_signal.js';
|
|
8
|
+
import type { MemorySnapshot } from './memory_load_signal.js';
|
|
3
9
|
import type { SystemInfo } from './system_status.js';
|
|
4
10
|
export interface SnapshotterOptions {
|
|
5
11
|
/**
|
|
@@ -28,41 +34,21 @@ export interface SnapshotterOptions {
|
|
|
28
34
|
/**
|
|
29
35
|
* Defines the maximum number of new rate limit errors within
|
|
30
36
|
* the given interval.
|
|
31
|
-
* @default
|
|
37
|
+
* @default 3
|
|
32
38
|
*/
|
|
33
39
|
maxClientErrors?: number;
|
|
34
40
|
/**
|
|
35
41
|
* Sets the interval in seconds for which a history of resource snapshots
|
|
36
42
|
* will be kept. Increasing this to very high numbers will affect performance.
|
|
37
|
-
* @default
|
|
43
|
+
* @default 30
|
|
38
44
|
*/
|
|
39
45
|
snapshotHistorySecs?: number;
|
|
40
46
|
/** @internal */
|
|
41
47
|
log?: CrawleeLogger;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
usedBytes?: number;
|
|
47
|
-
}
|
|
48
|
-
interface CpuSnapshot {
|
|
49
|
-
createdAt: Date;
|
|
50
|
-
isOverloaded: boolean;
|
|
51
|
-
usedRatio: number;
|
|
52
|
-
ticks?: {
|
|
53
|
-
idle: number;
|
|
54
|
-
total: number;
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
interface EventLoopSnapshot {
|
|
58
|
-
createdAt: Date;
|
|
59
|
-
isOverloaded: boolean;
|
|
60
|
-
exceededMillis: number;
|
|
61
|
-
}
|
|
62
|
-
interface ClientSnapshot {
|
|
63
|
-
createdAt: Date;
|
|
64
|
-
isOverloaded: boolean;
|
|
65
|
-
rateLimitErrorCount: number;
|
|
48
|
+
/** @internal */
|
|
49
|
+
client?: StorageBackend;
|
|
50
|
+
/** @internal */
|
|
51
|
+
config?: Configuration;
|
|
66
52
|
}
|
|
67
53
|
/**
|
|
68
54
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
@@ -87,24 +73,26 @@ interface ClientSnapshot {
|
|
|
87
73
|
*
|
|
88
74
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
89
75
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
76
|
+
*
|
|
90
77
|
* @category Scaling
|
|
91
78
|
*/
|
|
92
79
|
export declare class Snapshotter {
|
|
93
80
|
log: CrawleeLogger;
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
81
|
+
client: StorageBackend;
|
|
82
|
+
config: Configuration;
|
|
83
|
+
private readonly memorySignal;
|
|
84
|
+
private readonly eventLoopSignal;
|
|
85
|
+
private readonly cpuSignal;
|
|
86
|
+
private readonly clientSignal;
|
|
87
|
+
/**
|
|
88
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
89
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
90
|
+
*/
|
|
91
|
+
getLoadSignals(): LoadSignal[];
|
|
92
|
+
get cpuSnapshots(): CpuSnapshot[];
|
|
93
|
+
get eventLoopSnapshots(): EventLoopSnapshot[];
|
|
94
|
+
get memorySnapshots(): MemorySnapshot[];
|
|
95
|
+
get clientSnapshots(): ClientSnapshot[];
|
|
108
96
|
/**
|
|
109
97
|
* @param [options] All `Snapshotter` configuration options.
|
|
110
98
|
*/
|
|
@@ -138,42 +126,27 @@ export declare class Snapshotter {
|
|
|
138
126
|
*/
|
|
139
127
|
getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
|
|
140
128
|
/**
|
|
141
|
-
*
|
|
142
|
-
*/
|
|
143
|
-
protected _getSample<T extends {
|
|
144
|
-
createdAt: Date;
|
|
145
|
-
}>(snapshots: T[], sampleDurationMillis?: number): T[];
|
|
146
|
-
/**
|
|
147
|
-
* Creates a snapshot of current memory usage
|
|
148
|
-
* using the Apify platform `systemInfo` event.
|
|
129
|
+
* @deprecated Kept for backward compatibility.
|
|
149
130
|
*/
|
|
150
131
|
protected _snapshotMemory(systemInfo: SystemInfo): void;
|
|
151
132
|
/**
|
|
152
|
-
*
|
|
133
|
+
* @deprecated Kept for backward compatibility.
|
|
153
134
|
*/
|
|
154
135
|
protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
|
|
155
136
|
/**
|
|
156
|
-
*
|
|
137
|
+
* @deprecated Kept for backward compatibility.
|
|
157
138
|
*/
|
|
158
139
|
protected _snapshotEventLoop(intervalCallback: () => unknown): void;
|
|
159
140
|
/**
|
|
160
|
-
*
|
|
141
|
+
* @deprecated Kept for backward compatibility.
|
|
161
142
|
*/
|
|
162
143
|
protected _snapshotCpu(systemInfo: SystemInfo): void;
|
|
163
144
|
/**
|
|
164
|
-
*
|
|
165
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
166
|
-
* of the API call are considered for snapshotting since
|
|
167
|
-
* earlier errors may just be caused by a random spike in
|
|
168
|
-
* number of requests and do not necessarily signify API
|
|
169
|
-
* overloading.
|
|
145
|
+
* @deprecated Kept for backward compatibility.
|
|
170
146
|
*/
|
|
171
147
|
protected _snapshotClient(intervalCallback: () => unknown): void;
|
|
172
148
|
/**
|
|
173
|
-
*
|
|
174
|
-
* from the array (destructively - in place).
|
|
149
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
175
150
|
*/
|
|
176
|
-
protected _pruneSnapshots(
|
|
151
|
+
protected _pruneSnapshots(_snapshots: any[], _now: Date): void;
|
|
177
152
|
}
|
|
178
|
-
export {};
|
|
179
|
-
//# sourceMappingURL=snapshotter.d.ts.map
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { getMemoryInfo, isContainerized } from '@crawlee/utils';
|
|
2
1
|
import ow from 'ow';
|
|
3
|
-
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
2
|
import { serviceLocator } from '../service_locator.js';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
import { createClientLoadSignal } from './client_load_signal.js';
|
|
4
|
+
import { createCpuLoadSignal } from './cpu_load_signal.js';
|
|
5
|
+
import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
6
|
+
import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
8
7
|
/**
|
|
9
8
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
10
9
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -28,24 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
28
27
|
*
|
|
29
28
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
30
29
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
30
|
+
*
|
|
31
31
|
* @category Scaling
|
|
32
32
|
*/
|
|
33
33
|
export class Snapshotter {
|
|
34
34
|
log;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
35
|
+
client;
|
|
36
|
+
config;
|
|
37
|
+
memorySignal;
|
|
38
|
+
eventLoopSignal;
|
|
39
|
+
cpuSignal;
|
|
40
|
+
clientSignal;
|
|
41
|
+
/**
|
|
42
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
43
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
44
|
+
*/
|
|
45
|
+
getLoadSignals() {
|
|
46
|
+
return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
|
|
47
|
+
}
|
|
48
|
+
// Legacy public properties kept for backward compat (tests read these directly)
|
|
49
|
+
get cpuSnapshots() {
|
|
50
|
+
return this.cpuSignal.store.getAll();
|
|
51
|
+
}
|
|
52
|
+
get eventLoopSnapshots() {
|
|
53
|
+
return this.eventLoopSignal.store.getAll();
|
|
54
|
+
}
|
|
55
|
+
get memorySnapshots() {
|
|
56
|
+
return this.memorySignal.getMemorySnapshots();
|
|
57
|
+
}
|
|
58
|
+
get clientSnapshots() {
|
|
59
|
+
return this.clientSignal.store.getAll();
|
|
60
|
+
}
|
|
49
61
|
/**
|
|
50
62
|
* @param [options] All `Snapshotter` configuration options.
|
|
51
63
|
*/
|
|
@@ -58,54 +70,53 @@ export class Snapshotter {
|
|
|
58
70
|
maxUsedMemoryRatio: ow.optional.number,
|
|
59
71
|
maxClientErrors: ow.optional.number,
|
|
60
72
|
log: ow.optional.object,
|
|
73
|
+
client: ow.optional.object,
|
|
74
|
+
config: ow.optional.object,
|
|
61
75
|
}));
|
|
62
|
-
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), } = options;
|
|
76
|
+
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
|
|
63
77
|
this.log = log.child({ prefix: 'Snapshotter' });
|
|
64
|
-
this.
|
|
65
|
-
this.
|
|
66
|
-
|
|
67
|
-
this.
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
78
|
+
this.client = client;
|
|
79
|
+
this.config = config;
|
|
80
|
+
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
81
|
+
this.memorySignal = new MemoryLoadSignal({
|
|
82
|
+
maxUsedMemoryRatio,
|
|
83
|
+
snapshotHistoryMillis,
|
|
84
|
+
config: this.config,
|
|
85
|
+
log: this.log,
|
|
86
|
+
});
|
|
87
|
+
this.eventLoopSignal = createEventLoopLoadSignal({
|
|
88
|
+
eventLoopSnapshotIntervalSecs,
|
|
89
|
+
maxBlockedMillis,
|
|
90
|
+
snapshotHistoryMillis,
|
|
91
|
+
});
|
|
92
|
+
this.cpuSignal = createCpuLoadSignal({
|
|
93
|
+
snapshotHistoryMillis,
|
|
94
|
+
config: this.config,
|
|
95
|
+
});
|
|
96
|
+
this.clientSignal = createClientLoadSignal({
|
|
97
|
+
client: this.client,
|
|
98
|
+
clientSnapshotIntervalSecs,
|
|
99
|
+
maxClientErrors,
|
|
100
|
+
snapshotHistoryMillis,
|
|
101
|
+
});
|
|
73
102
|
}
|
|
74
103
|
/**
|
|
75
104
|
* Starts capturing snapshots at configured intervals.
|
|
76
105
|
*/
|
|
77
106
|
async start() {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
else {
|
|
83
|
-
const containerized = serviceLocator.getConfiguration().containerized ?? (await isContainerized());
|
|
84
|
-
const memInfo = await getMemoryInfo({
|
|
85
|
-
containerized,
|
|
86
|
-
logger: serviceLocator.getLogger(),
|
|
87
|
-
});
|
|
88
|
-
const totalBytes = memInfo.totalBytes;
|
|
89
|
-
this.maxMemoryBytes = Math.ceil(totalBytes * serviceLocator.getConfiguration().availableMemoryRatio);
|
|
90
|
-
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
91
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
92
|
-
}
|
|
93
|
-
// Start snapshotting.
|
|
94
|
-
this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
|
|
95
|
-
this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
|
|
96
|
-
const events = serviceLocator.getEventManager();
|
|
97
|
-
events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
|
|
98
|
-
events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
107
|
+
await this.memorySignal.start();
|
|
108
|
+
await this.eventLoopSignal.start();
|
|
109
|
+
await this.cpuSignal.start();
|
|
110
|
+
await this.clientSignal.start();
|
|
99
111
|
}
|
|
100
112
|
/**
|
|
101
113
|
* Stops all resource capturing.
|
|
102
114
|
*/
|
|
103
115
|
async stop() {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
116
|
+
await this.memorySignal.stop();
|
|
117
|
+
await this.eventLoopSignal.stop();
|
|
118
|
+
await this.cpuSignal.stop();
|
|
119
|
+
await this.clientSignal.stop();
|
|
109
120
|
// Allow microtask queue to unwind before stop returns.
|
|
110
121
|
await new Promise((resolve) => {
|
|
111
122
|
setImmediate(resolve);
|
|
@@ -116,166 +127,63 @@ export class Snapshotter {
|
|
|
116
127
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
117
128
|
*/
|
|
118
129
|
getMemorySample(sampleDurationMillis) {
|
|
119
|
-
return this.
|
|
130
|
+
return this.memorySignal.getSample(sampleDurationMillis);
|
|
120
131
|
}
|
|
121
132
|
/**
|
|
122
133
|
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
123
134
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
124
135
|
*/
|
|
125
136
|
getEventLoopSample(sampleDurationMillis) {
|
|
126
|
-
return this.
|
|
137
|
+
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
127
138
|
}
|
|
128
139
|
/**
|
|
129
140
|
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
130
141
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
131
142
|
*/
|
|
132
143
|
getCpuSample(sampleDurationMillis) {
|
|
133
|
-
return this.
|
|
144
|
+
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
134
145
|
}
|
|
135
146
|
/**
|
|
136
147
|
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
137
148
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
138
149
|
*/
|
|
139
150
|
getClientSample(sampleDurationMillis) {
|
|
140
|
-
return this.
|
|
151
|
+
return this.clientSignal.getSample(sampleDurationMillis);
|
|
141
152
|
}
|
|
142
153
|
/**
|
|
143
|
-
*
|
|
144
|
-
*/
|
|
145
|
-
_getSample(snapshots, sampleDurationMillis) {
|
|
146
|
-
if (!sampleDurationMillis)
|
|
147
|
-
return snapshots;
|
|
148
|
-
const sample = [];
|
|
149
|
-
let idx = snapshots.length;
|
|
150
|
-
if (!idx)
|
|
151
|
-
return sample;
|
|
152
|
-
const latestTime = snapshots[idx - 1].createdAt;
|
|
153
|
-
while (idx--) {
|
|
154
|
-
const snapshot = snapshots[idx];
|
|
155
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
156
|
-
sample.unshift(snapshot);
|
|
157
|
-
}
|
|
158
|
-
else {
|
|
159
|
-
break;
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
return sample;
|
|
163
|
-
}
|
|
164
|
-
/**
|
|
165
|
-
* Creates a snapshot of current memory usage
|
|
166
|
-
* using the Apify platform `systemInfo` event.
|
|
154
|
+
* @deprecated Kept for backward compatibility.
|
|
167
155
|
*/
|
|
168
156
|
_snapshotMemory(systemInfo) {
|
|
169
|
-
|
|
170
|
-
this._pruneSnapshots(this.memorySnapshots, createdAt);
|
|
171
|
-
const { memCurrentBytes } = systemInfo;
|
|
172
|
-
const snapshot = {
|
|
173
|
-
createdAt,
|
|
174
|
-
isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
175
|
-
usedBytes: memCurrentBytes,
|
|
176
|
-
};
|
|
177
|
-
this.memorySnapshots.push(snapshot);
|
|
178
|
-
this._memoryOverloadWarning(systemInfo);
|
|
157
|
+
this.memorySignal._onSystemInfo(systemInfo);
|
|
179
158
|
}
|
|
180
159
|
/**
|
|
181
|
-
*
|
|
160
|
+
* @deprecated Kept for backward compatibility.
|
|
182
161
|
*/
|
|
183
162
|
_memoryOverloadWarning(systemInfo) {
|
|
184
|
-
|
|
185
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
186
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
187
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
188
|
-
return;
|
|
189
|
-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
|
|
190
|
-
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
191
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
192
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
193
|
-
if (isCriticalOverload) {
|
|
194
|
-
const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
|
|
195
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
196
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
197
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
198
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
199
|
-
}
|
|
163
|
+
this.memorySignal._memoryOverloadWarning(systemInfo);
|
|
200
164
|
}
|
|
201
165
|
/**
|
|
202
|
-
*
|
|
166
|
+
* @deprecated Kept for backward compatibility.
|
|
203
167
|
*/
|
|
204
168
|
_snapshotEventLoop(intervalCallback) {
|
|
205
|
-
|
|
206
|
-
this._pruneSnapshots(this.eventLoopSnapshots, now);
|
|
207
|
-
const snapshot = {
|
|
208
|
-
createdAt: now,
|
|
209
|
-
isOverloaded: false,
|
|
210
|
-
exceededMillis: 0,
|
|
211
|
-
};
|
|
212
|
-
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
|
|
213
|
-
if (previousSnapshot) {
|
|
214
|
-
const { createdAt } = previousSnapshot;
|
|
215
|
-
const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
|
|
216
|
-
if (delta > this.maxBlockedMillis)
|
|
217
|
-
snapshot.isOverloaded = true;
|
|
218
|
-
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
219
|
-
}
|
|
220
|
-
this.eventLoopSnapshots.push(snapshot);
|
|
221
|
-
intervalCallback();
|
|
169
|
+
this.eventLoopSignal.handle(intervalCallback);
|
|
222
170
|
}
|
|
223
171
|
/**
|
|
224
|
-
*
|
|
172
|
+
* @deprecated Kept for backward compatibility.
|
|
225
173
|
*/
|
|
226
174
|
_snapshotCpu(systemInfo) {
|
|
227
|
-
|
|
228
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
229
|
-
this._pruneSnapshots(this.cpuSnapshots, createdAt);
|
|
230
|
-
this.cpuSnapshots.push({
|
|
231
|
-
createdAt,
|
|
232
|
-
isOverloaded: isCpuOverloaded,
|
|
233
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
234
|
-
});
|
|
175
|
+
this.cpuSignal.handle(systemInfo);
|
|
235
176
|
}
|
|
236
177
|
/**
|
|
237
|
-
*
|
|
238
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
239
|
-
* of the API call are considered for snapshotting since
|
|
240
|
-
* earlier errors may just be caused by a random spike in
|
|
241
|
-
* number of requests and do not necessarily signify API
|
|
242
|
-
* overloading.
|
|
178
|
+
* @deprecated Kept for backward compatibility.
|
|
243
179
|
*/
|
|
244
180
|
_snapshotClient(intervalCallback) {
|
|
245
|
-
|
|
246
|
-
this._pruneSnapshots(this.clientSnapshots, now);
|
|
247
|
-
const allErrorCounts = serviceLocator.getStorageBackend().stats?.rateLimitErrors ?? []; // storage backend might not support this
|
|
248
|
-
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
249
|
-
// Handle empty snapshots array
|
|
250
|
-
const snapshot = {
|
|
251
|
-
createdAt: now,
|
|
252
|
-
isOverloaded: false,
|
|
253
|
-
rateLimitErrorCount: currentErrCount,
|
|
254
|
-
};
|
|
255
|
-
const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
|
|
256
|
-
if (previousSnapshot) {
|
|
257
|
-
const { rateLimitErrorCount } = previousSnapshot;
|
|
258
|
-
const delta = currentErrCount - rateLimitErrorCount;
|
|
259
|
-
if (delta > this.maxClientErrors)
|
|
260
|
-
snapshot.isOverloaded = true;
|
|
261
|
-
}
|
|
262
|
-
this.clientSnapshots.push(snapshot);
|
|
263
|
-
intervalCallback();
|
|
181
|
+
this.clientSignal.handle(intervalCallback);
|
|
264
182
|
}
|
|
265
183
|
/**
|
|
266
|
-
*
|
|
267
|
-
* from the array (destructively - in place).
|
|
184
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
268
185
|
*/
|
|
269
|
-
_pruneSnapshots(
|
|
270
|
-
|
|
271
|
-
for (let i = 0; i < snapshots.length; i++) {
|
|
272
|
-
const { createdAt } = snapshots[i];
|
|
273
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
|
|
274
|
-
oldCount++;
|
|
275
|
-
else
|
|
276
|
-
break;
|
|
277
|
-
}
|
|
278
|
-
snapshots.splice(0, oldCount);
|
|
186
|
+
_pruneSnapshots(_snapshots, _now) {
|
|
187
|
+
// no-op — signals prune themselves
|
|
279
188
|
}
|
|
280
189
|
}
|
|
281
|
-
//# sourceMappingURL=snapshotter.js.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LoadSignal } from './load_signal.js';
|
|
1
2
|
import { Snapshotter } from './snapshotter.js';
|
|
2
3
|
/**
|
|
3
4
|
* Represents the current status of the system.
|
|
@@ -9,6 +10,7 @@ export interface SystemInfo {
|
|
|
9
10
|
eventLoopInfo: ClientInfo;
|
|
10
11
|
cpuInfo: ClientInfo;
|
|
11
12
|
clientInfo: ClientInfo;
|
|
13
|
+
memTotalBytes?: number;
|
|
12
14
|
memCurrentBytes?: number;
|
|
13
15
|
/**
|
|
14
16
|
* Platform only property
|
|
@@ -25,6 +27,11 @@ export interface SystemInfo {
|
|
|
25
27
|
* @internal
|
|
26
28
|
*/
|
|
27
29
|
createdAt?: Date;
|
|
30
|
+
/**
|
|
31
|
+
* Status of additional load signals beyond the built-in four.
|
|
32
|
+
* Keys are `LoadSignal.name` values, values are overload info.
|
|
33
|
+
*/
|
|
34
|
+
loadSignalInfo?: Record<string, ClientInfo>;
|
|
28
35
|
}
|
|
29
36
|
export interface SystemStatusOptions {
|
|
30
37
|
/**
|
|
@@ -60,6 +67,13 @@ export interface SystemStatusOptions {
|
|
|
60
67
|
* The `Snapshotter` instance to be queried for `SystemStatus`.
|
|
61
68
|
*/
|
|
62
69
|
snapshotter?: Snapshotter;
|
|
70
|
+
/**
|
|
71
|
+
* Additional load signals to include in the system status evaluation.
|
|
72
|
+
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
73
|
+
* and client signals. If any signal reports overload, the system is
|
|
74
|
+
* considered overloaded.
|
|
75
|
+
*/
|
|
76
|
+
loadSignals?: LoadSignal[];
|
|
63
77
|
}
|
|
64
78
|
export interface ClientInfo {
|
|
65
79
|
isOverloaded: boolean;
|
|
@@ -102,11 +116,14 @@ export interface FinalStatistics {
|
|
|
102
116
|
*/
|
|
103
117
|
export declare class SystemStatus {
|
|
104
118
|
private readonly currentHistoryMillis;
|
|
105
|
-
private readonly maxMemoryOverloadedRatio;
|
|
106
|
-
private readonly maxEventLoopOverloadedRatio;
|
|
107
|
-
private readonly maxCpuOverloadedRatio;
|
|
108
|
-
private readonly maxClientOverloadedRatio;
|
|
109
119
|
private readonly snapshotter;
|
|
120
|
+
private readonly signals;
|
|
121
|
+
/**
|
|
122
|
+
* Per-signal ratio overrides. The built-in four get their overrides from
|
|
123
|
+
* the legacy `max*OverloadedRatio` options; custom signals use their own
|
|
124
|
+
* `overloadedRatio`.
|
|
125
|
+
*/
|
|
126
|
+
private ratioOverrides;
|
|
110
127
|
constructor(options?: SystemStatusOptions);
|
|
111
128
|
/**
|
|
112
129
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -146,33 +163,4 @@ export declare class SystemStatus {
|
|
|
146
163
|
* Returns a system status object.
|
|
147
164
|
*/
|
|
148
165
|
protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
|
|
149
|
-
/**
|
|
150
|
-
* Returns an object with an isOverloaded property set to true
|
|
151
|
-
* if the memory has been overloaded in the last sampleDurationMillis.
|
|
152
|
-
*/
|
|
153
|
-
protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
154
|
-
/**
|
|
155
|
-
* Returns an object with an isOverloaded property set to true
|
|
156
|
-
* if the event loop has been overloaded in the last sampleDurationMillis.
|
|
157
|
-
*/
|
|
158
|
-
protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
159
|
-
/**
|
|
160
|
-
* Returns an object with an isOverloaded property set to true
|
|
161
|
-
* if the CPU has been overloaded in the last sampleDurationMillis.
|
|
162
|
-
*/
|
|
163
|
-
protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
164
|
-
/**
|
|
165
|
-
* Returns an object with an isOverloaded property set to true
|
|
166
|
-
* if the client has been overloaded in the last sampleDurationMillis.
|
|
167
|
-
*/
|
|
168
|
-
protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
|
|
169
|
-
/**
|
|
170
|
-
* Returns an object with sample information and an isOverloaded property
|
|
171
|
-
* set to true if at least the ratio of snapshots in the sample are overloaded.
|
|
172
|
-
*/
|
|
173
|
-
protected _isSampleOverloaded<T extends {
|
|
174
|
-
createdAt: Date;
|
|
175
|
-
isOverloaded: boolean;
|
|
176
|
-
}>(sample: T[], ratio: number): ClientInfo;
|
|
177
166
|
}
|
|
178
|
-
//# sourceMappingURL=system_status.d.ts.map
|