@crawlee/core 4.0.0-beta.70 → 4.0.0-beta.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -1
- package/autoscaling/autoscaled_pool.js +9 -2
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +101 -0
- package/autoscaling/snapshotter.d.ts +36 -63
- package/autoscaling/snapshotter.js +83 -175
- package/autoscaling/system_status.d.ts +21 -33
- package/autoscaling/system_status.js +45 -82
- package/configuration.d.ts +0 -1
- package/configuration.js +0 -1
- package/cookie_utils.d.ts +0 -1
- package/cookie_utils.js +2 -1
- package/crawlers/context_pipeline.d.ts +0 -1
- package/crawlers/context_pipeline.js +0 -1
- package/crawlers/crawler_commons.d.ts +0 -1
- package/crawlers/crawler_commons.js +0 -1
- package/crawlers/crawler_utils.d.ts +0 -1
- package/crawlers/crawler_utils.js +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -1
- package/crawlers/error_snapshotter.js +0 -1
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +0 -1
- package/crawlers/index.js +0 -1
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +6 -1
- package/crawlers/statistics.js +8 -1
- package/enqueue_links/enqueue_links.d.ts +8 -1
- package/enqueue_links/enqueue_links.js +9 -9
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +0 -1
- package/enqueue_links/shared.js +0 -1
- package/errors.d.ts +20 -1
- package/errors.js +22 -1
- package/events/event_manager.d.ts +0 -1
- package/events/event_manager.js +0 -1
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +9 -6
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/log.d.ts +0 -1
- package/log.js +0 -1
- package/memory-storage/consts.d.ts +0 -1
- package/memory-storage/consts.js +0 -1
- package/memory-storage/index.d.ts +0 -1
- package/memory-storage/index.js +0 -1
- package/memory-storage/memory-storage.d.ts +0 -1
- package/memory-storage/memory-storage.js +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
- package/memory-storage/resource-clients/common/base-client.js +0 -1
- package/memory-storage/resource-clients/dataset.d.ts +0 -1
- package/memory-storage/resource-clients/dataset.js +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
- package/memory-storage/resource-clients/key-value-store.js +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts +0 -1
- package/memory-storage/resource-clients/request-queue.js +0 -1
- package/memory-storage/utils.d.ts +6 -1
- package/memory-storage/utils.js +15 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts +0 -1
- package/proxy_configuration.js +0 -1
- package/recoverable_state.d.ts +0 -1
- package/recoverable_state.js +0 -1
- package/request.d.ts +0 -1
- package/request.js +0 -1
- package/router.d.ts +117 -15
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +0 -1
- package/service_locator.d.ts +0 -1
- package/service_locator.js +0 -1
- package/session_pool/consts.d.ts +0 -1
- package/session_pool/consts.js +0 -1
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +0 -1
- package/session_pool/fingerprint.js +0 -1
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.js +0 -1
- package/session_pool/session.d.ts +0 -1
- package/session_pool/session.js +0 -1
- package/session_pool/session_pool.d.ts +0 -1
- package/session_pool/session_pool.js +0 -1
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +0 -1
- package/storages/dataset.d.ts +1 -2
- package/storages/dataset.js +0 -1
- package/storages/index.d.ts +0 -1
- package/storages/index.js +0 -1
- package/storages/key_value_store.d.ts +0 -1
- package/storages/key_value_store.js +0 -1
- package/storages/key_value_store_codec.d.ts +0 -1
- package/storages/key_value_store_codec.js +0 -1
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +6 -1
- package/storages/request_list.js +16 -3
- package/storages/request_loader.d.ts +0 -1
- package/storages/request_loader.js +0 -1
- package/storages/request_manager.d.ts +0 -1
- package/storages/request_manager.js +0 -1
- package/storages/request_manager_tandem.d.ts +0 -1
- package/storages/request_manager_tandem.js +0 -1
- package/storages/request_queue.d.ts +24 -1
- package/storages/request_queue.js +80 -27
- package/storages/sitemap_request_loader.d.ts +0 -1
- package/storages/sitemap_request_loader.js +0 -1
- package/storages/storage_instance_manager.d.ts +0 -1
- package/storages/storage_instance_manager.js +0 -1
- package/storages/storage_stats.d.ts +0 -1
- package/storages/storage_stats.js +0 -1
- package/storages/utils.d.ts +0 -1
- package/storages/utils.js +0 -1
- package/typedefs.d.ts +0 -1
- package/typedefs.js +0 -1
- package/validators.d.ts +0 -1
- package/validators.js +0 -1
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/memory-storage/consts.d.ts.map +0 -1
- package/memory-storage/consts.js.map +0 -1
- package/memory-storage/index.d.ts.map +0 -1
- package/memory-storage/index.js.map +0 -1
- package/memory-storage/memory-storage.d.ts.map +0 -1
- package/memory-storage/memory-storage.js.map +0 -1
- package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
- package/memory-storage/resource-clients/common/base-client.js.map +0 -1
- package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
- package/memory-storage/resource-clients/dataset.js.map +0 -1
- package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
- package/memory-storage/resource-clients/key-value-store.js.map +0 -1
- package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
- package/memory-storage/resource-clients/request-queue.js.map +0 -1
- package/memory-storage/utils.d.ts.map +0 -1
- package/memory-storage/utils.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/service_locator.d.ts.map +0 -1
- package/service_locator.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/fingerprint.d.ts.map +0 -1
- package/session_pool/fingerprint.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/key_value_store_codec.d.ts.map +0 -1
- package/storages/key_value_store_codec.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_loader.d.ts.map +0 -1
- package/storages/request_loader.js.map +0 -1
- package/storages/request_manager.d.ts.map +0 -1
- package/storages/request_manager.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/sitemap_request_loader.d.ts.map +0 -1
- package/storages/sitemap_request_loader.js.map +0 -1
- package/storages/storage_instance_manager.d.ts.map +0 -1
- package/storages/storage_instance_manager.js.map +0 -1
- package/storages/storage_stats.d.ts.map +0 -1
- package/storages/storage_stats.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
|
@@ -170,6 +170,8 @@ export declare class AutoscaledPool {
|
|
|
170
170
|
private resolve;
|
|
171
171
|
private reject;
|
|
172
172
|
private snapshotter;
|
|
173
|
+
/** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
|
|
174
|
+
private loadSignals;
|
|
173
175
|
private systemStatus;
|
|
174
176
|
private autoscaleInterval;
|
|
175
177
|
private maybeRunInterval;
|
|
@@ -295,4 +297,3 @@ export declare class AutoscaledPool {
|
|
|
295
297
|
protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
|
|
296
298
|
protected get _isOverMaxRequestLimit(): boolean;
|
|
297
299
|
}
|
|
298
|
-
//# sourceMappingURL=autoscaled_pool.d.ts.map
|
|
@@ -77,6 +77,8 @@ export class AutoscaledPool {
|
|
|
77
77
|
resolve = null;
|
|
78
78
|
reject = null;
|
|
79
79
|
snapshotter;
|
|
80
|
+
/** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
|
|
81
|
+
loadSignals;
|
|
80
82
|
systemStatus;
|
|
81
83
|
autoscaleInterval;
|
|
82
84
|
maybeRunInterval;
|
|
@@ -136,6 +138,7 @@ export class AutoscaledPool {
|
|
|
136
138
|
log: this.log,
|
|
137
139
|
});
|
|
138
140
|
this.snapshotter = ssoCopy.snapshotter;
|
|
141
|
+
this.loadSignals = ssoCopy.loadSignals ?? [];
|
|
139
142
|
this.systemStatus = new SystemStatus(ssoCopy);
|
|
140
143
|
}
|
|
141
144
|
/**
|
|
@@ -198,6 +201,7 @@ export class AutoscaledPool {
|
|
|
198
201
|
this.reject = reject;
|
|
199
202
|
});
|
|
200
203
|
await this.snapshotter.start();
|
|
204
|
+
await Promise.all(this.loadSignals.map((s) => s.start()));
|
|
201
205
|
// This interval checks the system status and updates the desired concurrency accordingly.
|
|
202
206
|
this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
|
|
203
207
|
// This is here because if we scale down to let's say 1, then after each promise is finished
|
|
@@ -252,14 +256,17 @@ export class AutoscaledPool {
|
|
|
252
256
|
this.isStopped = true;
|
|
253
257
|
await new Promise((resolve, reject) => {
|
|
254
258
|
let timeout;
|
|
259
|
+
let interval;
|
|
255
260
|
if (timeoutSecs) {
|
|
256
261
|
timeout = setTimeout(() => {
|
|
262
|
+
// Clean up the polling interval to prevent it from leaking on timeout.
|
|
263
|
+
clearInterval(interval);
|
|
257
264
|
const err = new Error("The pool's running tasks did not finish" +
|
|
258
265
|
`in ${timeoutSecs} secs after pool.pause() invocation.`);
|
|
259
266
|
reject(err);
|
|
260
267
|
}, timeoutSecs);
|
|
261
268
|
}
|
|
262
|
-
|
|
269
|
+
interval = setInterval(() => {
|
|
263
270
|
if (this._currentConcurrency <= 0) {
|
|
264
271
|
// Clean up timeout and interval to prevent process hanging.
|
|
265
272
|
if (timeout)
|
|
@@ -511,6 +518,7 @@ export class AutoscaledPool {
|
|
|
511
518
|
betterClearInterval(this.tasksDonePerSecondInterval);
|
|
512
519
|
if (this.snapshotter)
|
|
513
520
|
await this.snapshotter.stop();
|
|
521
|
+
await Promise.all(this.loadSignals.map((s) => s.stop()));
|
|
514
522
|
}
|
|
515
523
|
_incrementTasksDonePerSecond(intervalCallback) {
|
|
516
524
|
this._tasksPerMinute.unshift(0);
|
|
@@ -524,4 +532,3 @@ export class AutoscaledPool {
|
|
|
524
532
|
return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
|
|
525
533
|
}
|
|
526
534
|
}
|
|
527
|
-
//# sourceMappingURL=autoscaled_pool.js.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { StorageBackend } from '@crawlee/types';
|
|
2
|
+
import type { LoadSnapshot } from './load_signal.js';
|
|
3
|
+
import { SnapshotStore } from './load_signal.js';
|
|
4
|
+
export interface ClientSnapshot extends LoadSnapshot {
|
|
5
|
+
rateLimitErrorCount: number;
|
|
6
|
+
}
|
|
7
|
+
export interface ClientLoadSignalOptions {
|
|
8
|
+
client: StorageBackend;
|
|
9
|
+
clientSnapshotIntervalSecs?: number;
|
|
10
|
+
maxClientErrors?: number;
|
|
11
|
+
overloadedRatio?: number;
|
|
12
|
+
snapshotHistoryMillis?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Periodically checks the storage client for rate-limit errors (HTTP 429)
|
|
16
|
+
* and reports overload when the error delta exceeds a threshold.
|
|
17
|
+
*/
|
|
18
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
19
|
+
export declare function createClientLoadSignal(options: ClientLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
|
|
20
|
+
store: SnapshotStore<ClientSnapshot>;
|
|
21
|
+
handle: (cb: () => unknown) => void;
|
|
22
|
+
getSample(sampleDurationMillis?: number): ClientSnapshot[];
|
|
23
|
+
};
|
|
24
|
+
/** @internal Return type for backward compat in Snapshotter facade */
|
|
25
|
+
export type ClientLoadSignal = ReturnType<typeof createClientLoadSignal>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { SnapshotStore } from './load_signal.js';
|
|
2
|
+
const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
|
|
3
|
+
/**
|
|
4
|
+
* Periodically checks the storage client for rate-limit errors (HTTP 429)
|
|
5
|
+
* and reports overload when the error delta exceeds a threshold.
|
|
6
|
+
*/
|
|
7
|
+
export function createClientLoadSignal(options) {
|
|
8
|
+
const maxClientErrors = options.maxClientErrors ?? 3;
|
|
9
|
+
const signal = SnapshotStore.fromInterval({
|
|
10
|
+
name: 'clientInfo',
|
|
11
|
+
overloadedRatio: options.overloadedRatio ?? 0.3,
|
|
12
|
+
intervalMillis: (options.clientSnapshotIntervalSecs ?? 1) * 1000,
|
|
13
|
+
snapshotHistoryMillis: options.snapshotHistoryMillis,
|
|
14
|
+
handler(store, intervalCallback) {
|
|
15
|
+
const now = new Date();
|
|
16
|
+
const allErrorCounts = options.client.stats?.rateLimitErrors ?? [];
|
|
17
|
+
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
18
|
+
const snapshot = {
|
|
19
|
+
createdAt: now,
|
|
20
|
+
isOverloaded: false,
|
|
21
|
+
rateLimitErrorCount: currentErrCount,
|
|
22
|
+
};
|
|
23
|
+
const all = store.getAll();
|
|
24
|
+
const previousSnapshot = all[all.length - 1];
|
|
25
|
+
if (previousSnapshot) {
|
|
26
|
+
const { rateLimitErrorCount } = previousSnapshot;
|
|
27
|
+
const delta = currentErrCount - rateLimitErrorCount;
|
|
28
|
+
if (delta > maxClientErrors)
|
|
29
|
+
snapshot.isOverloaded = true;
|
|
30
|
+
}
|
|
31
|
+
store.push(snapshot, now);
|
|
32
|
+
intervalCallback();
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
return signal;
|
|
36
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import type { LoadSnapshot } from './load_signal.js';
|
|
3
|
+
import { SnapshotStore } from './load_signal.js';
|
|
4
|
+
import type { SystemInfo } from './system_status.js';
|
|
5
|
+
export interface CpuSnapshot extends LoadSnapshot {
|
|
6
|
+
usedRatio: number;
|
|
7
|
+
ticks?: {
|
|
8
|
+
idle: number;
|
|
9
|
+
total: number;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export interface CpuLoadSignalOptions {
|
|
13
|
+
overloadedRatio?: number;
|
|
14
|
+
snapshotHistoryMillis?: number;
|
|
15
|
+
config: Configuration;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
|
|
19
|
+
* the platform or local OS metrics indicate the CPU is overloaded.
|
|
20
|
+
*/
|
|
21
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
22
|
+
export declare function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
|
|
23
|
+
store: SnapshotStore<CpuSnapshot>;
|
|
24
|
+
handle: (payload: SystemInfo) => void;
|
|
25
|
+
getSample(sampleDurationMillis?: number): CpuSnapshot[];
|
|
26
|
+
};
|
|
27
|
+
/** @internal Return type for backward compat in Snapshotter facade */
|
|
28
|
+
export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { serviceLocator } from '../service_locator.js';
|
|
2
|
+
import { SnapshotStore } from './load_signal.js';
|
|
3
|
+
/**
|
|
4
|
+
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
|
|
5
|
+
* the platform or local OS metrics indicate the CPU is overloaded.
|
|
6
|
+
*/
|
|
7
|
+
export function createCpuLoadSignal(options) {
|
|
8
|
+
return SnapshotStore.fromEvent({
|
|
9
|
+
name: 'cpuInfo',
|
|
10
|
+
overloadedRatio: options.overloadedRatio ?? 0.4,
|
|
11
|
+
events: serviceLocator.getEventManager(),
|
|
12
|
+
event: "systemInfo" /* EventType.SYSTEM_INFO */,
|
|
13
|
+
snapshotHistoryMillis: options.snapshotHistoryMillis,
|
|
14
|
+
handler(store, systemInfo) {
|
|
15
|
+
const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
|
|
16
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
17
|
+
store.push({
|
|
18
|
+
createdAt,
|
|
19
|
+
isOverloaded: isCpuOverloaded,
|
|
20
|
+
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
21
|
+
}, createdAt);
|
|
22
|
+
},
|
|
23
|
+
});
|
|
24
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { LoadSnapshot } from './load_signal.js';
|
|
2
|
+
import { SnapshotStore } from './load_signal.js';
|
|
3
|
+
export interface EventLoopSnapshot extends LoadSnapshot {
|
|
4
|
+
exceededMillis: number;
|
|
5
|
+
}
|
|
6
|
+
export interface EventLoopLoadSignalOptions {
|
|
7
|
+
eventLoopSnapshotIntervalSecs?: number;
|
|
8
|
+
maxBlockedMillis?: number;
|
|
9
|
+
overloadedRatio?: number;
|
|
10
|
+
snapshotHistoryMillis?: number;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Periodically measures event loop delay and reports overload when the
|
|
14
|
+
* delay exceeds a configured threshold.
|
|
15
|
+
*/
|
|
16
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
17
|
+
export declare function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
|
|
18
|
+
store: SnapshotStore<EventLoopSnapshot>;
|
|
19
|
+
handle: (cb: () => unknown) => void;
|
|
20
|
+
getSample(sampleDurationMillis?: number): EventLoopSnapshot[];
|
|
21
|
+
};
|
|
22
|
+
/** @internal Return type for backward compat in Snapshotter facade */
|
|
23
|
+
export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { SnapshotStore } from './load_signal.js';
|
|
2
|
+
/**
|
|
3
|
+
* Periodically measures event loop delay and reports overload when the
|
|
4
|
+
* delay exceeds a configured threshold.
|
|
5
|
+
*/
|
|
6
|
+
export function createEventLoopLoadSignal(options = {}) {
|
|
7
|
+
const intervalMillis = (options.eventLoopSnapshotIntervalSecs ?? 0.5) * 1000;
|
|
8
|
+
const maxBlockedMillis = options.maxBlockedMillis ?? 50;
|
|
9
|
+
const signal = SnapshotStore.fromInterval({
|
|
10
|
+
name: 'eventLoopInfo',
|
|
11
|
+
overloadedRatio: options.overloadedRatio ?? 0.6,
|
|
12
|
+
intervalMillis,
|
|
13
|
+
snapshotHistoryMillis: options.snapshotHistoryMillis,
|
|
14
|
+
handler(store, intervalCallback) {
|
|
15
|
+
const now = new Date();
|
|
16
|
+
const snapshot = {
|
|
17
|
+
createdAt: now,
|
|
18
|
+
isOverloaded: false,
|
|
19
|
+
exceededMillis: 0,
|
|
20
|
+
};
|
|
21
|
+
const all = store.getAll();
|
|
22
|
+
const previousSnapshot = all[all.length - 1];
|
|
23
|
+
if (previousSnapshot) {
|
|
24
|
+
const { createdAt } = previousSnapshot;
|
|
25
|
+
const delta = now.getTime() - +createdAt - intervalMillis;
|
|
26
|
+
if (delta > maxBlockedMillis)
|
|
27
|
+
snapshot.isOverloaded = true;
|
|
28
|
+
snapshot.exceededMillis = Math.max(delta - maxBlockedMillis, 0);
|
|
29
|
+
}
|
|
30
|
+
store.push(snapshot, now);
|
|
31
|
+
intervalCallback();
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
return signal;
|
|
35
|
+
}
|
package/autoscaling/index.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
export * from './autoscaled_pool.js';
|
|
2
|
+
export * from './client_load_signal.js';
|
|
3
|
+
export * from './cpu_load_signal.js';
|
|
4
|
+
export * from './event_loop_load_signal.js';
|
|
5
|
+
export * from './load_signal.js';
|
|
6
|
+
export * from './memory_load_signal.js';
|
|
2
7
|
export * from './snapshotter.js';
|
|
3
8
|
export * from './system_status.js';
|
|
4
|
-
//# sourceMappingURL=index.d.ts.map
|
package/autoscaling/index.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
export * from './autoscaled_pool.js';
|
|
2
|
+
export * from './client_load_signal.js';
|
|
3
|
+
export * from './cpu_load_signal.js';
|
|
4
|
+
export * from './event_loop_load_signal.js';
|
|
5
|
+
export * from './load_signal.js';
|
|
6
|
+
export * from './memory_load_signal.js';
|
|
2
7
|
export * from './snapshotter.js';
|
|
3
8
|
export * from './system_status.js';
|
|
4
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import type { EventManager, EventTypeName } from '../events/event_manager.js';
|
|
2
|
+
import type { ClientInfo } from './system_status.js';
|
|
3
|
+
/**
|
|
4
|
+
* A snapshot of a resource's overload state at a point in time.
|
|
5
|
+
*/
|
|
6
|
+
export interface LoadSnapshot {
|
|
7
|
+
createdAt: Date;
|
|
8
|
+
isOverloaded: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* A signal that reports whether a particular resource is overloaded.
|
|
12
|
+
*
|
|
13
|
+
* `SystemStatus` aggregates multiple `LoadSignal` instances to determine
|
|
14
|
+
* overall system health. The built-in signals cover memory, CPU, event loop,
|
|
15
|
+
* and API client rate limits. You can implement this interface to add
|
|
16
|
+
* custom overload signals (e.g. navigation timeouts, proxy health).
|
|
17
|
+
*/
|
|
18
|
+
export interface LoadSignal {
|
|
19
|
+
/** Human-readable name used in logging and `SystemInfo` keys. */
|
|
20
|
+
readonly name: string;
|
|
21
|
+
/**
|
|
22
|
+
* Maximum ratio of overloaded snapshots in a sample before the signal
|
|
23
|
+
* is considered overloaded. For example, `0.2` means the signal fires
|
|
24
|
+
* when more than 20% of the sample window is overloaded.
|
|
25
|
+
*/
|
|
26
|
+
readonly overloadedRatio: number;
|
|
27
|
+
/** Start collecting snapshots. Called when the pool starts. */
|
|
28
|
+
start(): Promise<void>;
|
|
29
|
+
/** Stop collecting snapshots. Called when the pool shuts down. */
|
|
30
|
+
stop(): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Return snapshots for a recent time window (used for "current" status).
|
|
33
|
+
* @param sampleDurationMillis How far back to look, in milliseconds.
|
|
34
|
+
*/
|
|
35
|
+
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* A time-pruning, time-windowed store for `LoadSnapshot` values.
|
|
39
|
+
* Signals compose with this instead of inheriting from a base class.
|
|
40
|
+
*/
|
|
41
|
+
export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
|
|
42
|
+
private snapshots;
|
|
43
|
+
private readonly historyMillis;
|
|
44
|
+
constructor(historyMillis?: number);
|
|
45
|
+
/**
|
|
46
|
+
* Add a snapshot and prune entries older than the history window.
|
|
47
|
+
*/
|
|
48
|
+
push(snapshot: T, now?: Date): void;
|
|
49
|
+
/**
|
|
50
|
+
* Return all snapshots, or only those within the given time window.
|
|
51
|
+
*/
|
|
52
|
+
getSample(sampleDurationMillis?: number): T[];
|
|
53
|
+
/**
|
|
54
|
+
* Direct access to the underlying array (for backward-compat getters).
|
|
55
|
+
*/
|
|
56
|
+
getAll(): T[];
|
|
57
|
+
/**
|
|
58
|
+
* Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
|
|
59
|
+
*
|
|
60
|
+
* The `handler` receives the store (to read previous snapshots) and the
|
|
61
|
+
* interval callback (which it **must** call when done). It should call
|
|
62
|
+
* `store.push()` to record a snapshot.
|
|
63
|
+
*/
|
|
64
|
+
static fromInterval<T extends LoadSnapshot>(options: {
|
|
65
|
+
name: string;
|
|
66
|
+
overloadedRatio: number;
|
|
67
|
+
intervalMillis: number;
|
|
68
|
+
snapshotHistoryMillis?: number;
|
|
69
|
+
handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
|
|
70
|
+
}): Omit<LoadSignal, 'getSample'> & {
|
|
71
|
+
store: SnapshotStore<T>;
|
|
72
|
+
handle: (cb: () => unknown) => void;
|
|
73
|
+
getSample(sampleDurationMillis?: number): T[];
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Create a `LoadSignal` that snapshots in response to an `EventManager` event.
|
|
77
|
+
*
|
|
78
|
+
* The `handler` receives the event payload and the store. It should call
|
|
79
|
+
* `store.push()` to record a snapshot.
|
|
80
|
+
*/
|
|
81
|
+
static fromEvent<T extends LoadSnapshot, E>(options: {
|
|
82
|
+
name: string;
|
|
83
|
+
overloadedRatio: number;
|
|
84
|
+
events: EventManager;
|
|
85
|
+
event: EventTypeName;
|
|
86
|
+
snapshotHistoryMillis?: number;
|
|
87
|
+
handler: (store: SnapshotStore<T>, payload: E) => void;
|
|
88
|
+
}): Omit<LoadSignal, 'getSample'> & {
|
|
89
|
+
store: SnapshotStore<T>;
|
|
90
|
+
handle: (payload: E) => void;
|
|
91
|
+
getSample(sampleDurationMillis?: number): T[];
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
|
|
96
|
+
* overloaded ratio, using a time-weighted average. This is the shared
|
|
97
|
+
* evaluation logic used by `SystemStatus` for all signal types.
|
|
98
|
+
*/
|
|
99
|
+
export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { weightedAvg } from '@crawlee/utils';
|
|
2
|
+
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
+
/**
|
|
4
|
+
* A time-pruning, time-windowed store for `LoadSnapshot` values.
|
|
5
|
+
* Signals compose with this instead of inheriting from a base class.
|
|
6
|
+
*/
|
|
7
|
+
export class SnapshotStore {
|
|
8
|
+
snapshots = [];
|
|
9
|
+
historyMillis;
|
|
10
|
+
constructor(historyMillis = 30_000) {
|
|
11
|
+
this.historyMillis = historyMillis;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Add a snapshot and prune entries older than the history window.
|
|
15
|
+
*/
|
|
16
|
+
push(snapshot, now = snapshot.createdAt) {
|
|
17
|
+
// Inline pruning to avoid private-method transpilation issues
|
|
18
|
+
let oldCount = 0;
|
|
19
|
+
for (let i = 0; i < this.snapshots.length; i++) {
|
|
20
|
+
const { createdAt } = this.snapshots[i];
|
|
21
|
+
if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
|
|
22
|
+
oldCount++;
|
|
23
|
+
else
|
|
24
|
+
break;
|
|
25
|
+
}
|
|
26
|
+
if (oldCount)
|
|
27
|
+
this.snapshots.splice(0, oldCount);
|
|
28
|
+
this.snapshots.push(snapshot);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Return all snapshots, or only those within the given time window.
|
|
32
|
+
*/
|
|
33
|
+
getSample(sampleDurationMillis) {
|
|
34
|
+
if (!sampleDurationMillis)
|
|
35
|
+
return this.snapshots;
|
|
36
|
+
const sample = [];
|
|
37
|
+
let idx = this.snapshots.length;
|
|
38
|
+
if (!idx)
|
|
39
|
+
return sample;
|
|
40
|
+
const latestTime = this.snapshots[idx - 1].createdAt;
|
|
41
|
+
while (idx--) {
|
|
42
|
+
const snapshot = this.snapshots[idx];
|
|
43
|
+
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
44
|
+
sample.unshift(snapshot);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return sample;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Direct access to the underlying array (for backward-compat getters).
|
|
54
|
+
*/
|
|
55
|
+
getAll() {
|
|
56
|
+
return this.snapshots;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
|
|
60
|
+
*
|
|
61
|
+
* The `handler` receives the store (to read previous snapshots) and the
|
|
62
|
+
* interval callback (which it **must** call when done). It should call
|
|
63
|
+
* `store.push()` to record a snapshot.
|
|
64
|
+
*/
|
|
65
|
+
static fromInterval(options) {
|
|
66
|
+
const store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
67
|
+
let interval = null;
|
|
68
|
+
const handle = (cb) => options.handler(store, cb);
|
|
69
|
+
return {
|
|
70
|
+
name: options.name,
|
|
71
|
+
overloadedRatio: options.overloadedRatio,
|
|
72
|
+
store,
|
|
73
|
+
handle,
|
|
74
|
+
getSample: (ms) => store.getSample(ms),
|
|
75
|
+
async start() {
|
|
76
|
+
interval = betterSetInterval(handle, options.intervalMillis);
|
|
77
|
+
},
|
|
78
|
+
async stop() {
|
|
79
|
+
betterClearInterval(interval);
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Create a `LoadSignal` that snapshots in response to an `EventManager` event.
|
|
85
|
+
*
|
|
86
|
+
* The `handler` receives the event payload and the store. It should call
|
|
87
|
+
* `store.push()` to record a snapshot.
|
|
88
|
+
*/
|
|
89
|
+
static fromEvent(options) {
|
|
90
|
+
const store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
91
|
+
const handle = (payload) => options.handler(store, payload);
|
|
92
|
+
return {
|
|
93
|
+
name: options.name,
|
|
94
|
+
overloadedRatio: options.overloadedRatio,
|
|
95
|
+
store,
|
|
96
|
+
handle,
|
|
97
|
+
getSample: (ms) => store.getSample(ms),
|
|
98
|
+
async start() {
|
|
99
|
+
options.events.on(options.event, handle);
|
|
100
|
+
},
|
|
101
|
+
async stop() {
|
|
102
|
+
options.events.off(options.event, handle);
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
|
|
109
|
+
* overloaded ratio, using a time-weighted average. This is the shared
|
|
110
|
+
* evaluation logic used by `SystemStatus` for all signal types.
|
|
111
|
+
*/
|
|
112
|
+
export function evaluateLoadSignalSample(sample, overloadedRatio) {
|
|
113
|
+
if (sample.length === 0) {
|
|
114
|
+
return {
|
|
115
|
+
isOverloaded: false,
|
|
116
|
+
limitRatio: overloadedRatio,
|
|
117
|
+
actualRatio: 0,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const weights = [];
|
|
121
|
+
const values = [];
|
|
122
|
+
for (let i = 1; i < sample.length; i++) {
|
|
123
|
+
const previous = sample[i - 1];
|
|
124
|
+
const current = sample[i];
|
|
125
|
+
const weight = +current.createdAt - +previous.createdAt;
|
|
126
|
+
weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
|
|
127
|
+
values.push(+current.isOverloaded);
|
|
128
|
+
}
|
|
129
|
+
const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
|
|
130
|
+
return {
|
|
131
|
+
isOverloaded: wAvg > overloadedRatio,
|
|
132
|
+
limitRatio: overloadedRatio,
|
|
133
|
+
actualRatio: Math.round(wAvg * 1000) / 1000,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import type { CrawleeLogger } from '../log.js';
|
|
3
|
+
import type { LoadSignal, LoadSnapshot } from './load_signal.js';
|
|
4
|
+
import type { SystemInfo } from './system_status.js';
|
|
5
|
+
export interface MemorySnapshot extends LoadSnapshot {
|
|
6
|
+
usedBytes?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface MemoryLoadSignalOptions {
|
|
9
|
+
maxUsedMemoryRatio?: number;
|
|
10
|
+
overloadedRatio?: number;
|
|
11
|
+
snapshotHistoryMillis?: number;
|
|
12
|
+
config: Configuration;
|
|
13
|
+
log?: CrawleeLogger;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
17
|
+
* the used-to-available memory ratio exceeds a threshold.
|
|
18
|
+
*/
|
|
19
|
+
export declare class MemoryLoadSignal implements LoadSignal {
|
|
20
|
+
readonly name = "memInfo";
|
|
21
|
+
readonly overloadedRatio: number;
|
|
22
|
+
private readonly store;
|
|
23
|
+
private readonly config;
|
|
24
|
+
private readonly events;
|
|
25
|
+
private readonly log;
|
|
26
|
+
private readonly maxUsedMemoryRatio;
|
|
27
|
+
private maxMemoryRatio;
|
|
28
|
+
private maxMemoryBytes;
|
|
29
|
+
private lastLoggedCriticalMemoryOverloadAt;
|
|
30
|
+
constructor(options: MemoryLoadSignalOptions);
|
|
31
|
+
start(): Promise<void>;
|
|
32
|
+
stop(): Promise<void>;
|
|
33
|
+
getSample(sampleDurationMillis?: number): MemorySnapshot[];
|
|
34
|
+
/**
|
|
35
|
+
* Returns typed memory snapshots for backward compatibility with `Snapshotter`.
|
|
36
|
+
*/
|
|
37
|
+
getMemorySnapshots(): MemorySnapshot[];
|
|
38
|
+
/** @internal */
|
|
39
|
+
_onSystemInfo(systemInfo: SystemInfo): void;
|
|
40
|
+
/** @internal */
|
|
41
|
+
_memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
|
|
42
|
+
private _getTotalMemoryBytes;
|
|
43
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { getMemoryInfo, isContainerized } from '@crawlee/utils';
|
|
2
|
+
import { serviceLocator } from '../service_locator.js';
|
|
3
|
+
import { SnapshotStore } from './load_signal.js';
|
|
4
|
+
const RESERVE_MEMORY_RATIO = 0.5;
|
|
5
|
+
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
6
|
+
/**
|
|
7
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
8
|
+
* the used-to-available memory ratio exceeds a threshold.
|
|
9
|
+
*/
|
|
10
|
+
export class MemoryLoadSignal {
|
|
11
|
+
name = 'memInfo';
|
|
12
|
+
overloadedRatio;
|
|
13
|
+
store;
|
|
14
|
+
config;
|
|
15
|
+
events;
|
|
16
|
+
log;
|
|
17
|
+
maxUsedMemoryRatio;
|
|
18
|
+
maxMemoryRatio;
|
|
19
|
+
maxMemoryBytes;
|
|
20
|
+
lastLoggedCriticalMemoryOverloadAt = null;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
this.store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
23
|
+
this.config = options.config;
|
|
24
|
+
this.events = serviceLocator.getEventManager();
|
|
25
|
+
this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
26
|
+
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
|
|
27
|
+
this.overloadedRatio = options.overloadedRatio ?? 0.2;
|
|
28
|
+
this._onSystemInfo = this._onSystemInfo.bind(this);
|
|
29
|
+
}
|
|
30
|
+
async start() {
|
|
31
|
+
const memoryMbytes = this.config.memoryMbytes ?? 0;
|
|
32
|
+
if (memoryMbytes > 0) {
|
|
33
|
+
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
this.maxMemoryRatio = this.config.availableMemoryRatio;
|
|
37
|
+
if (!this.maxMemoryRatio) {
|
|
38
|
+
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
|
|
42
|
+
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
43
|
+
}
|
|
44
|
+
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
45
|
+
this.maxMemoryBytes = await this._getTotalMemoryBytes();
|
|
46
|
+
}
|
|
47
|
+
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
|
|
48
|
+
}
|
|
49
|
+
async stop() {
|
|
50
|
+
this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
|
|
51
|
+
}
|
|
52
|
+
getSample(sampleDurationMillis) {
|
|
53
|
+
return this.store.getSample(sampleDurationMillis);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Returns typed memory snapshots for backward compatibility with `Snapshotter`.
|
|
57
|
+
*/
|
|
58
|
+
getMemorySnapshots() {
|
|
59
|
+
return this.store.getAll();
|
|
60
|
+
}
|
|
61
|
+
/** @internal */
|
|
62
|
+
_onSystemInfo(systemInfo) {
|
|
63
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
64
|
+
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
65
|
+
let maxMemoryBytes = this.maxMemoryBytes;
|
|
66
|
+
if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
|
|
67
|
+
maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
|
|
68
|
+
}
|
|
69
|
+
const snapshot = {
|
|
70
|
+
createdAt,
|
|
71
|
+
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
72
|
+
usedBytes: memCurrentBytes,
|
|
73
|
+
};
|
|
74
|
+
this.store.push(snapshot, createdAt);
|
|
75
|
+
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
76
|
+
}
|
|
77
|
+
/** @internal */
|
|
78
|
+
_memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
79
|
+
const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
|
|
80
|
+
const { memCurrentBytes } = systemInfo;
|
|
81
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
82
|
+
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
83
|
+
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
84
|
+
return;
|
|
85
|
+
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
|
|
86
|
+
const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
87
|
+
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
88
|
+
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
89
|
+
if (isCriticalOverload) {
|
|
90
|
+
const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
|
|
91
|
+
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
92
|
+
this.log.warning('Memory is critically overloaded. ' +
|
|
93
|
+
`Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
94
|
+
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async _getTotalMemoryBytes() {
|
|
98
|
+
const containerized = this.config.containerized ?? (await isContainerized());
|
|
99
|
+
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
100
|
+
}
|
|
101
|
+
}
|