@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +5 -6
- package/autoscaling/autoscaled_pool.js +12 -11
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +102 -0
- package/autoscaling/snapshotter.d.ts +35 -72
- package/autoscaling/snapshotter.js +78 -181
- package/autoscaling/system_status.d.ts +21 -36
- package/autoscaling/system_status.js +46 -84
- package/configuration.d.ts +88 -228
- package/configuration.js +162 -224
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +20 -13
- package/crawlers/context_pipeline.d.ts +70 -0
- package/crawlers/context_pipeline.js +122 -0
- package/crawlers/crawler_commons.d.ts +20 -30
- package/crawlers/crawler_commons.js +12 -21
- package/crawlers/crawler_utils.d.ts +2 -3
- package/crawlers/crawler_utils.js +1 -2
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.js +2 -3
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -2
- package/crawlers/index.js +1 -2
- package/crawlers/internals/types.d.ts +7 -0
- package/crawlers/internals/types.js +1 -0
- package/crawlers/statistics.d.ts +22 -17
- package/crawlers/statistics.js +30 -26
- package/enqueue_links/enqueue_links.d.ts +44 -22
- package/enqueue_links/enqueue_links.js +51 -30
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +25 -9
- package/enqueue_links/shared.js +69 -38
- package/errors.d.ts +53 -4
- package/errors.js +70 -5
- package/events/event_manager.d.ts +34 -8
- package/events/event_manager.js +8 -10
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -5
- package/events/local_event_manager.js +37 -40
- package/index.d.ts +4 -3
- package/index.js +3 -2
- package/log.d.ts +82 -3
- package/log.js +102 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +46 -0
- package/memory-storage/memory-storage.js +136 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +113 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +203 -0
- package/memory-storage/resource-clients/request-queue.d.ts +96 -0
- package/memory-storage/resource-clients/request-queue.js +421 -0
- package/memory-storage/utils.d.ts +17 -0
- package/memory-storage/utils.js +42 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +14 -149
- package/proxy_configuration.js +19 -168
- package/recoverable_state.d.ts +120 -0
- package/recoverable_state.js +141 -0
- package/request.d.ts +84 -13
- package/request.js +107 -28
- package/router.d.ts +117 -15
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +1 -2
- package/service_locator.d.ts +156 -0
- package/service_locator.js +238 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +35 -73
- package/session_pool/session.js +46 -99
- package/session_pool/session_pool.d.ts +71 -72
- package/session_pool/session_pool.js +111 -100
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +5 -2
- package/storages/dataset.d.ts +101 -51
- package/storages/dataset.js +168 -126
- package/storages/index.d.ts +8 -7
- package/storages/index.js +6 -7
- package/storages/key_value_store.d.ts +165 -36
- package/storages/key_value_store.js +269 -122
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +38 -82
- package/storages/request_list.js +65 -41
- package/storages/request_loader.d.ts +96 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +106 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +303 -46
- package/storages/request_queue.js +629 -214
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.js +258 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/utils.d.ts +51 -6
- package/storages/utils.js +60 -9
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +1 -2
- package/typedefs.js +0 -1
- package/validators.d.ts +8 -1
- package/validators.js +8 -1
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import type { CrawleeLogger } from '../log.js';
|
|
3
|
+
import type { LoadSignal, LoadSnapshot } from './load_signal.js';
|
|
4
|
+
import type { SystemInfo } from './system_status.js';
|
|
5
|
+
export interface MemorySnapshot extends LoadSnapshot {
|
|
6
|
+
usedBytes?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface MemoryLoadSignalOptions {
|
|
9
|
+
maxUsedMemoryRatio?: number;
|
|
10
|
+
overloadedRatio?: number;
|
|
11
|
+
snapshotHistoryMillis?: number;
|
|
12
|
+
config: Configuration;
|
|
13
|
+
log?: CrawleeLogger;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
17
|
+
* the used-to-available memory ratio exceeds a threshold.
|
|
18
|
+
*/
|
|
19
|
+
export declare class MemoryLoadSignal implements LoadSignal {
|
|
20
|
+
readonly name = "memInfo";
|
|
21
|
+
readonly overloadedRatio: number;
|
|
22
|
+
private readonly store;
|
|
23
|
+
private readonly config;
|
|
24
|
+
private readonly events;
|
|
25
|
+
private readonly log;
|
|
26
|
+
private readonly maxUsedMemoryRatio;
|
|
27
|
+
private maxMemoryRatio;
|
|
28
|
+
private maxMemoryBytes;
|
|
29
|
+
private lastLoggedCriticalMemoryOverloadAt;
|
|
30
|
+
constructor(options: MemoryLoadSignalOptions);
|
|
31
|
+
start(): Promise<void>;
|
|
32
|
+
stop(): Promise<void>;
|
|
33
|
+
getSample(sampleDurationMillis?: number): MemorySnapshot[];
|
|
34
|
+
/**
|
|
35
|
+
* Returns typed memory snapshots for backward compatibility with `Snapshotter`.
|
|
36
|
+
*/
|
|
37
|
+
getMemorySnapshots(): MemorySnapshot[];
|
|
38
|
+
/** @internal */
|
|
39
|
+
_onSystemInfo(systemInfo: SystemInfo): void;
|
|
40
|
+
/** @internal */
|
|
41
|
+
_memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
|
|
42
|
+
private _getTotalMemoryBytes;
|
|
43
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { serviceLocator } from '../service_locator.js';
|
|
2
|
+
import { getMemoryInfo } from '../system-info/memory-info.js';
|
|
3
|
+
import { isContainerized } from '../system-info/runtime.js';
|
|
4
|
+
import { SnapshotStore } from './load_signal.js';
|
|
5
|
+
const RESERVE_MEMORY_RATIO = 0.5;
|
|
6
|
+
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
7
|
+
/**
|
|
8
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
9
|
+
* the used-to-available memory ratio exceeds a threshold.
|
|
10
|
+
*/
|
|
11
|
+
export class MemoryLoadSignal {
|
|
12
|
+
name = 'memInfo';
|
|
13
|
+
overloadedRatio;
|
|
14
|
+
store;
|
|
15
|
+
config;
|
|
16
|
+
events;
|
|
17
|
+
log;
|
|
18
|
+
maxUsedMemoryRatio;
|
|
19
|
+
maxMemoryRatio;
|
|
20
|
+
maxMemoryBytes;
|
|
21
|
+
lastLoggedCriticalMemoryOverloadAt = null;
|
|
22
|
+
constructor(options) {
|
|
23
|
+
this.store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
24
|
+
this.config = options.config;
|
|
25
|
+
this.events = serviceLocator.getEventManager();
|
|
26
|
+
this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
27
|
+
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
|
|
28
|
+
this.overloadedRatio = options.overloadedRatio ?? 0.2;
|
|
29
|
+
this._onSystemInfo = this._onSystemInfo.bind(this);
|
|
30
|
+
}
|
|
31
|
+
async start() {
|
|
32
|
+
const memoryMbytes = this.config.memoryMbytes ?? 0;
|
|
33
|
+
if (memoryMbytes > 0) {
|
|
34
|
+
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
this.maxMemoryRatio = this.config.availableMemoryRatio;
|
|
38
|
+
if (!this.maxMemoryRatio) {
|
|
39
|
+
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
|
|
43
|
+
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
44
|
+
}
|
|
45
|
+
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
46
|
+
this.maxMemoryBytes = await this._getTotalMemoryBytes();
|
|
47
|
+
}
|
|
48
|
+
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
|
|
49
|
+
}
|
|
50
|
+
async stop() {
|
|
51
|
+
this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
|
|
52
|
+
}
|
|
53
|
+
getSample(sampleDurationMillis) {
|
|
54
|
+
return this.store.getSample(sampleDurationMillis);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Returns typed memory snapshots for backward compatibility with `Snapshotter`.
|
|
58
|
+
*/
|
|
59
|
+
getMemorySnapshots() {
|
|
60
|
+
return this.store.getAll();
|
|
61
|
+
}
|
|
62
|
+
/** @internal */
|
|
63
|
+
_onSystemInfo(systemInfo) {
|
|
64
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
65
|
+
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
66
|
+
let maxMemoryBytes = this.maxMemoryBytes;
|
|
67
|
+
if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
|
|
68
|
+
maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
|
|
69
|
+
}
|
|
70
|
+
const snapshot = {
|
|
71
|
+
createdAt,
|
|
72
|
+
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
73
|
+
usedBytes: memCurrentBytes,
|
|
74
|
+
};
|
|
75
|
+
this.store.push(snapshot, createdAt);
|
|
76
|
+
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
77
|
+
}
|
|
78
|
+
/** @internal */
|
|
79
|
+
_memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
80
|
+
const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
|
|
81
|
+
const { memCurrentBytes } = systemInfo;
|
|
82
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
83
|
+
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
84
|
+
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
85
|
+
return;
|
|
86
|
+
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
|
|
87
|
+
const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
88
|
+
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
89
|
+
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
90
|
+
if (isCriticalOverload) {
|
|
91
|
+
const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
|
|
92
|
+
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
93
|
+
this.log.warning('Memory is critically overloaded. ' +
|
|
94
|
+
`Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
95
|
+
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async _getTotalMemoryBytes() {
|
|
99
|
+
const containerized = this.config.containerized ?? (await isContainerized());
|
|
100
|
+
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
4
|
-
import {
|
|
5
|
-
import type {
|
|
1
|
+
import type { StorageBackend } from '@crawlee/types';
|
|
2
|
+
import type { Configuration } from '../configuration.js';
|
|
3
|
+
import type { CrawleeLogger } from '../log.js';
|
|
4
|
+
import type { ClientSnapshot } from './client_load_signal.js';
|
|
5
|
+
import type { CpuSnapshot } from './cpu_load_signal.js';
|
|
6
|
+
import type { EventLoopSnapshot } from './event_loop_load_signal.js';
|
|
7
|
+
import type { LoadSignal } from './load_signal.js';
|
|
8
|
+
import type { MemorySnapshot } from './memory_load_signal.js';
|
|
6
9
|
import type { SystemInfo } from './system_status.js';
|
|
7
10
|
export interface SnapshotterOptions {
|
|
8
11
|
/**
|
|
@@ -31,46 +34,22 @@ export interface SnapshotterOptions {
|
|
|
31
34
|
/**
|
|
32
35
|
* Defines the maximum number of new rate limit errors within
|
|
33
36
|
* the given interval.
|
|
34
|
-
* @default
|
|
37
|
+
* @default 3
|
|
35
38
|
*/
|
|
36
39
|
maxClientErrors?: number;
|
|
37
40
|
/**
|
|
38
41
|
* Sets the interval in seconds for which a history of resource snapshots
|
|
39
42
|
* will be kept. Increasing this to very high numbers will affect performance.
|
|
40
|
-
* @default
|
|
43
|
+
* @default 30
|
|
41
44
|
*/
|
|
42
45
|
snapshotHistorySecs?: number;
|
|
43
46
|
/** @internal */
|
|
44
|
-
log?:
|
|
47
|
+
log?: CrawleeLogger;
|
|
45
48
|
/** @internal */
|
|
46
|
-
client?:
|
|
49
|
+
client?: StorageBackend;
|
|
47
50
|
/** @internal */
|
|
48
51
|
config?: Configuration;
|
|
49
52
|
}
|
|
50
|
-
interface MemorySnapshot {
|
|
51
|
-
createdAt: Date;
|
|
52
|
-
isOverloaded: boolean;
|
|
53
|
-
usedBytes?: number;
|
|
54
|
-
}
|
|
55
|
-
interface CpuSnapshot {
|
|
56
|
-
createdAt: Date;
|
|
57
|
-
isOverloaded: boolean;
|
|
58
|
-
usedRatio: number;
|
|
59
|
-
ticks?: {
|
|
60
|
-
idle: number;
|
|
61
|
-
total: number;
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
interface EventLoopSnapshot {
|
|
65
|
-
createdAt: Date;
|
|
66
|
-
isOverloaded: boolean;
|
|
67
|
-
exceededMillis: number;
|
|
68
|
-
}
|
|
69
|
-
interface ClientSnapshot {
|
|
70
|
-
createdAt: Date;
|
|
71
|
-
isOverloaded: boolean;
|
|
72
|
-
rateLimitErrorCount: number;
|
|
73
|
-
}
|
|
74
53
|
/**
|
|
75
54
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
76
55
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -94,27 +73,26 @@ interface ClientSnapshot {
|
|
|
94
73
|
*
|
|
95
74
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
96
75
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
76
|
+
*
|
|
97
77
|
* @category Scaling
|
|
98
78
|
*/
|
|
99
79
|
export declare class Snapshotter {
|
|
100
|
-
log:
|
|
101
|
-
client:
|
|
80
|
+
log: CrawleeLogger;
|
|
81
|
+
client: StorageBackend;
|
|
102
82
|
config: Configuration;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
clientInterval: BetterIntervalID;
|
|
117
|
-
lastLoggedCriticalMemoryOverloadAt: Date | null;
|
|
83
|
+
private readonly memorySignal;
|
|
84
|
+
private readonly eventLoopSignal;
|
|
85
|
+
private readonly cpuSignal;
|
|
86
|
+
private readonly clientSignal;
|
|
87
|
+
/**
|
|
88
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
89
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
90
|
+
*/
|
|
91
|
+
getLoadSignals(): LoadSignal[];
|
|
92
|
+
get cpuSnapshots(): CpuSnapshot[];
|
|
93
|
+
get eventLoopSnapshots(): EventLoopSnapshot[];
|
|
94
|
+
get memorySnapshots(): MemorySnapshot[];
|
|
95
|
+
get clientSnapshots(): ClientSnapshot[];
|
|
118
96
|
/**
|
|
119
97
|
* @param [options] All `Snapshotter` configuration options.
|
|
120
98
|
*/
|
|
@@ -148,42 +126,27 @@ export declare class Snapshotter {
|
|
|
148
126
|
*/
|
|
149
127
|
getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
|
|
150
128
|
/**
|
|
151
|
-
*
|
|
152
|
-
*/
|
|
153
|
-
protected _getSample<T extends {
|
|
154
|
-
createdAt: Date;
|
|
155
|
-
}>(snapshots: T[], sampleDurationMillis?: number): T[];
|
|
156
|
-
/**
|
|
157
|
-
* Creates a snapshot of current memory usage
|
|
158
|
-
* using the Apify platform `systemInfo` event.
|
|
129
|
+
* @deprecated Kept for backward compatibility.
|
|
159
130
|
*/
|
|
160
131
|
protected _snapshotMemory(systemInfo: SystemInfo): void;
|
|
161
132
|
/**
|
|
162
|
-
*
|
|
133
|
+
* @deprecated Kept for backward compatibility.
|
|
163
134
|
*/
|
|
164
135
|
protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
|
|
165
136
|
/**
|
|
166
|
-
*
|
|
137
|
+
* @deprecated Kept for backward compatibility.
|
|
167
138
|
*/
|
|
168
139
|
protected _snapshotEventLoop(intervalCallback: () => unknown): void;
|
|
169
140
|
/**
|
|
170
|
-
*
|
|
141
|
+
* @deprecated Kept for backward compatibility.
|
|
171
142
|
*/
|
|
172
143
|
protected _snapshotCpu(systemInfo: SystemInfo): void;
|
|
173
144
|
/**
|
|
174
|
-
*
|
|
175
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
176
|
-
* of the API call are considered for snapshotting since
|
|
177
|
-
* earlier errors may just be caused by a random spike in
|
|
178
|
-
* number of requests and do not necessarily signify API
|
|
179
|
-
* overloading.
|
|
145
|
+
* @deprecated Kept for backward compatibility.
|
|
180
146
|
*/
|
|
181
147
|
protected _snapshotClient(intervalCallback: () => unknown): void;
|
|
182
148
|
/**
|
|
183
|
-
*
|
|
184
|
-
* from the array (destructively - in place).
|
|
149
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
185
150
|
*/
|
|
186
|
-
protected _pruneSnapshots(
|
|
151
|
+
protected _pruneSnapshots(_snapshots: any[], _now: Date): void;
|
|
187
152
|
}
|
|
188
|
-
export {};
|
|
189
|
-
//# sourceMappingURL=snapshotter.d.ts.map
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
|
|
2
1
|
import ow from 'ow';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
2
|
+
import { serviceLocator } from '../service_locator.js';
|
|
3
|
+
import { createClientLoadSignal } from './client_load_signal.js';
|
|
4
|
+
import { createCpuLoadSignal } from './cpu_load_signal.js';
|
|
5
|
+
import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
6
|
+
import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
9
7
|
/**
|
|
10
8
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
11
9
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -29,27 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
|
|
|
29
27
|
*
|
|
30
28
|
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
31
29
|
* typically received from the request queue, exceed the set limit within the set interval.
|
|
30
|
+
*
|
|
32
31
|
* @category Scaling
|
|
33
32
|
*/
|
|
34
33
|
export class Snapshotter {
|
|
35
34
|
log;
|
|
36
35
|
client;
|
|
37
36
|
config;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
37
|
+
memorySignal;
|
|
38
|
+
eventLoopSignal;
|
|
39
|
+
cpuSignal;
|
|
40
|
+
clientSignal;
|
|
41
|
+
/**
|
|
42
|
+
* Returns the four built-in signals as an array, so `SystemStatus` can
|
|
43
|
+
* iterate them alongside any custom `LoadSignal` instances.
|
|
44
|
+
*/
|
|
45
|
+
getLoadSignals() {
|
|
46
|
+
return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
|
|
47
|
+
}
|
|
48
|
+
// Legacy public properties kept for backward compat (tests read these directly)
|
|
49
|
+
get cpuSnapshots() {
|
|
50
|
+
return this.cpuSignal.store.getAll();
|
|
51
|
+
}
|
|
52
|
+
get eventLoopSnapshots() {
|
|
53
|
+
return this.eventLoopSignal.store.getAll();
|
|
54
|
+
}
|
|
55
|
+
get memorySnapshots() {
|
|
56
|
+
return this.memorySignal.getMemorySnapshots();
|
|
57
|
+
}
|
|
58
|
+
get clientSnapshots() {
|
|
59
|
+
return this.clientSignal.store.getAll();
|
|
60
|
+
}
|
|
53
61
|
/**
|
|
54
62
|
* @param [options] All `Snapshotter` configuration options.
|
|
55
63
|
*/
|
|
@@ -65,58 +73,50 @@ export class Snapshotter {
|
|
|
65
73
|
client: ow.optional.object,
|
|
66
74
|
config: ow.optional.object,
|
|
67
75
|
}));
|
|
68
|
-
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log =
|
|
76
|
+
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
|
|
69
77
|
this.log = log.child({ prefix: 'Snapshotter' });
|
|
70
78
|
this.client = client;
|
|
71
79
|
this.config = config;
|
|
72
|
-
|
|
73
|
-
this.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
81
|
+
this.memorySignal = new MemoryLoadSignal({
|
|
82
|
+
maxUsedMemoryRatio,
|
|
83
|
+
snapshotHistoryMillis,
|
|
84
|
+
config: this.config,
|
|
85
|
+
log: this.log,
|
|
86
|
+
});
|
|
87
|
+
this.eventLoopSignal = createEventLoopLoadSignal({
|
|
88
|
+
eventLoopSnapshotIntervalSecs,
|
|
89
|
+
maxBlockedMillis,
|
|
90
|
+
snapshotHistoryMillis,
|
|
91
|
+
});
|
|
92
|
+
this.cpuSignal = createCpuLoadSignal({
|
|
93
|
+
snapshotHistoryMillis,
|
|
94
|
+
config: this.config,
|
|
95
|
+
});
|
|
96
|
+
this.clientSignal = createClientLoadSignal({
|
|
97
|
+
client: this.client,
|
|
98
|
+
clientSnapshotIntervalSecs,
|
|
99
|
+
maxClientErrors,
|
|
100
|
+
snapshotHistoryMillis,
|
|
101
|
+
});
|
|
82
102
|
}
|
|
83
103
|
/**
|
|
84
104
|
* Starts capturing snapshots at configured intervals.
|
|
85
105
|
*/
|
|
86
106
|
async start() {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
else {
|
|
92
|
-
let totalBytes;
|
|
93
|
-
if (this.config.get('systemInfoV2')) {
|
|
94
|
-
const containerized = this.config.get('containerized', await isContainerized());
|
|
95
|
-
const memInfo = await getMemoryInfoV2(containerized);
|
|
96
|
-
totalBytes = memInfo.totalBytes;
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
const memInfo = await getMemoryInfo();
|
|
100
|
-
totalBytes = memInfo.totalBytes;
|
|
101
|
-
}
|
|
102
|
-
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
|
|
103
|
-
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
104
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
105
|
-
}
|
|
106
|
-
// Start snapshotting.
|
|
107
|
-
this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
|
|
108
|
-
this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
|
|
109
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
|
|
110
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
|
|
107
|
+
await this.memorySignal.start();
|
|
108
|
+
await this.eventLoopSignal.start();
|
|
109
|
+
await this.cpuSignal.start();
|
|
110
|
+
await this.clientSignal.start();
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
113
|
* Stops all resource capturing.
|
|
114
114
|
*/
|
|
115
115
|
async stop() {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
this.
|
|
119
|
-
this.
|
|
116
|
+
await this.memorySignal.stop();
|
|
117
|
+
await this.eventLoopSignal.stop();
|
|
118
|
+
await this.cpuSignal.stop();
|
|
119
|
+
await this.clientSignal.stop();
|
|
120
120
|
// Allow microtask queue to unwind before stop returns.
|
|
121
121
|
await new Promise((resolve) => {
|
|
122
122
|
setImmediate(resolve);
|
|
@@ -127,166 +127,63 @@ export class Snapshotter {
|
|
|
127
127
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
128
128
|
*/
|
|
129
129
|
getMemorySample(sampleDurationMillis) {
|
|
130
|
-
return this.
|
|
130
|
+
return this.memorySignal.getSample(sampleDurationMillis);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
133
|
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
134
134
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
135
135
|
*/
|
|
136
136
|
getEventLoopSample(sampleDurationMillis) {
|
|
137
|
-
return this.
|
|
137
|
+
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
138
138
|
}
|
|
139
139
|
/**
|
|
140
140
|
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
141
141
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
142
142
|
*/
|
|
143
143
|
getCpuSample(sampleDurationMillis) {
|
|
144
|
-
return this.
|
|
144
|
+
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
145
145
|
}
|
|
146
146
|
/**
|
|
147
147
|
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
148
148
|
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
149
149
|
*/
|
|
150
150
|
getClientSample(sampleDurationMillis) {
|
|
151
|
-
return this.
|
|
151
|
+
return this.clientSignal.getSample(sampleDurationMillis);
|
|
152
152
|
}
|
|
153
153
|
/**
|
|
154
|
-
*
|
|
155
|
-
*/
|
|
156
|
-
_getSample(snapshots, sampleDurationMillis) {
|
|
157
|
-
if (!sampleDurationMillis)
|
|
158
|
-
return snapshots;
|
|
159
|
-
const sample = [];
|
|
160
|
-
let idx = snapshots.length;
|
|
161
|
-
if (!idx)
|
|
162
|
-
return sample;
|
|
163
|
-
const latestTime = snapshots[idx - 1].createdAt;
|
|
164
|
-
while (idx--) {
|
|
165
|
-
const snapshot = snapshots[idx];
|
|
166
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
167
|
-
sample.unshift(snapshot);
|
|
168
|
-
}
|
|
169
|
-
else {
|
|
170
|
-
break;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
return sample;
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Creates a snapshot of current memory usage
|
|
177
|
-
* using the Apify platform `systemInfo` event.
|
|
154
|
+
* @deprecated Kept for backward compatibility.
|
|
178
155
|
*/
|
|
179
156
|
_snapshotMemory(systemInfo) {
|
|
180
|
-
|
|
181
|
-
this._pruneSnapshots(this.memorySnapshots, createdAt);
|
|
182
|
-
const { memCurrentBytes } = systemInfo;
|
|
183
|
-
const snapshot = {
|
|
184
|
-
createdAt,
|
|
185
|
-
isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
|
|
186
|
-
usedBytes: memCurrentBytes,
|
|
187
|
-
};
|
|
188
|
-
this.memorySnapshots.push(snapshot);
|
|
189
|
-
this._memoryOverloadWarning(systemInfo);
|
|
157
|
+
this.memorySignal._onSystemInfo(systemInfo);
|
|
190
158
|
}
|
|
191
159
|
/**
|
|
192
|
-
*
|
|
160
|
+
* @deprecated Kept for backward compatibility.
|
|
193
161
|
*/
|
|
194
162
|
_memoryOverloadWarning(systemInfo) {
|
|
195
|
-
|
|
196
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
197
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
198
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
199
|
-
return;
|
|
200
|
-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
|
|
201
|
-
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
|
|
202
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
203
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
204
|
-
if (isCriticalOverload) {
|
|
205
|
-
const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
|
|
206
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
207
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
208
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
209
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
210
|
-
}
|
|
163
|
+
this.memorySignal._memoryOverloadWarning(systemInfo);
|
|
211
164
|
}
|
|
212
165
|
/**
|
|
213
|
-
*
|
|
166
|
+
* @deprecated Kept for backward compatibility.
|
|
214
167
|
*/
|
|
215
168
|
_snapshotEventLoop(intervalCallback) {
|
|
216
|
-
|
|
217
|
-
this._pruneSnapshots(this.eventLoopSnapshots, now);
|
|
218
|
-
const snapshot = {
|
|
219
|
-
createdAt: now,
|
|
220
|
-
isOverloaded: false,
|
|
221
|
-
exceededMillis: 0,
|
|
222
|
-
};
|
|
223
|
-
const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
|
|
224
|
-
if (previousSnapshot) {
|
|
225
|
-
const { createdAt } = previousSnapshot;
|
|
226
|
-
const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
|
|
227
|
-
if (delta > this.maxBlockedMillis)
|
|
228
|
-
snapshot.isOverloaded = true;
|
|
229
|
-
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
230
|
-
}
|
|
231
|
-
this.eventLoopSnapshots.push(snapshot);
|
|
232
|
-
intervalCallback();
|
|
169
|
+
this.eventLoopSignal.handle(intervalCallback);
|
|
233
170
|
}
|
|
234
171
|
/**
|
|
235
|
-
*
|
|
172
|
+
* @deprecated Kept for backward compatibility.
|
|
236
173
|
*/
|
|
237
174
|
_snapshotCpu(systemInfo) {
|
|
238
|
-
|
|
239
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
240
|
-
this._pruneSnapshots(this.cpuSnapshots, createdAt);
|
|
241
|
-
this.cpuSnapshots.push({
|
|
242
|
-
createdAt,
|
|
243
|
-
isOverloaded: isCpuOverloaded,
|
|
244
|
-
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
245
|
-
});
|
|
175
|
+
this.cpuSignal.handle(systemInfo);
|
|
246
176
|
}
|
|
247
177
|
/**
|
|
248
|
-
*
|
|
249
|
-
* rate limit errors. Only errors produced by a 2nd retry
|
|
250
|
-
* of the API call are considered for snapshotting since
|
|
251
|
-
* earlier errors may just be caused by a random spike in
|
|
252
|
-
* number of requests and do not necessarily signify API
|
|
253
|
-
* overloading.
|
|
178
|
+
* @deprecated Kept for backward compatibility.
|
|
254
179
|
*/
|
|
255
180
|
_snapshotClient(intervalCallback) {
|
|
256
|
-
|
|
257
|
-
this._pruneSnapshots(this.clientSnapshots, now);
|
|
258
|
-
const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
|
|
259
|
-
const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
|
|
260
|
-
// Handle empty snapshots array
|
|
261
|
-
const snapshot = {
|
|
262
|
-
createdAt: now,
|
|
263
|
-
isOverloaded: false,
|
|
264
|
-
rateLimitErrorCount: currentErrCount,
|
|
265
|
-
};
|
|
266
|
-
const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
|
|
267
|
-
if (previousSnapshot) {
|
|
268
|
-
const { rateLimitErrorCount } = previousSnapshot;
|
|
269
|
-
const delta = currentErrCount - rateLimitErrorCount;
|
|
270
|
-
if (delta > this.maxClientErrors)
|
|
271
|
-
snapshot.isOverloaded = true;
|
|
272
|
-
}
|
|
273
|
-
this.clientSnapshots.push(snapshot);
|
|
274
|
-
intervalCallback();
|
|
181
|
+
this.clientSignal.handle(intervalCallback);
|
|
275
182
|
}
|
|
276
183
|
/**
|
|
277
|
-
*
|
|
278
|
-
* from the array (destructively - in place).
|
|
184
|
+
* @deprecated Pruning is now handled by individual signals.
|
|
279
185
|
*/
|
|
280
|
-
_pruneSnapshots(
|
|
281
|
-
|
|
282
|
-
for (let i = 0; i < snapshots.length; i++) {
|
|
283
|
-
const { createdAt } = snapshots[i];
|
|
284
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
|
|
285
|
-
oldCount++;
|
|
286
|
-
else
|
|
287
|
-
break;
|
|
288
|
-
}
|
|
289
|
-
snapshots.splice(0, oldCount);
|
|
186
|
+
_pruneSnapshots(_snapshots, _now) {
|
|
187
|
+
// no-op — signals prune themselves
|
|
290
188
|
}
|
|
291
189
|
}
|
|
292
|
-
//# sourceMappingURL=snapshotter.js.map
|