@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/configuration.d.ts +16 -47
- package/configuration.js +13 -25
- package/debug.js +4 -4
- package/errors.d.ts +28 -38
- package/errors.js +33 -47
- package/events/event_manager.d.ts +2 -2
- package/events/event_manager.js +7 -6
- package/events/index.d.ts +1 -0
- package/events/local_event_manager.d.ts +1 -8
- package/events/local_event_manager.js +13 -13
- package/events/system_info.d.ts +38 -0
- package/index.d.ts +2 -8
- package/index.js +4 -8
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/log.d.ts +10 -11
- package/log.js +52 -20
- package/memory-storage/memory-storage.d.ts +15 -18
- package/memory-storage/memory-storage.js +80 -58
- package/memory-storage/resource-clients/dataset.d.ts +1 -6
- package/memory-storage/resource-clients/dataset.js +23 -31
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -10
- package/memory-storage/resource-clients/key-value-store.js +43 -67
- package/memory-storage/resource-clients/request-queue.d.ts +1 -42
- package/memory-storage/resource-clients/request-queue.js +109 -117
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +17 -20
- package/proxy_configuration.d.ts +21 -26
- package/proxy_configuration.js +35 -25
- package/recoverable_state.d.ts +104 -47
- package/recoverable_state.js +199 -74
- package/request.d.ts +20 -107
- package/request.js +78 -244
- package/serialization.js +17 -16
- package/service_locator.d.ts +22 -10
- package/service_locator.js +59 -48
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +13 -8
- package/storages/dataset.js +149 -40
- package/storages/index.d.ts +4 -4
- package/storages/index.js +2 -4
- package/storages/key_value_store.d.ts +16 -35
- package/storages/key_value_store.js +223 -110
- package/storages/key_value_store_codec.js +6 -11
- package/storages/request_dedup_cache.d.ts +1 -4
- package/storages/request_dedup_cache.js +15 -15
- package/storages/request_list.d.ts +9 -104
- package/storages/request_list.js +236 -233
- package/storages/request_loader.d.ts +49 -18
- package/storages/request_loader.js +36 -1
- package/storages/request_manager.d.ts +86 -0
- package/storages/request_manager_tandem.d.ts +14 -38
- package/storages/request_manager_tandem.js +67 -64
- package/storages/request_queue.d.ts +23 -50
- package/storages/request_queue.js +371 -226
- package/storages/storage_instance_manager.d.ts +2 -4
- package/storages/storage_instance_manager.js +21 -21
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/transaction.d.ts +270 -0
- package/storages/transaction.js +296 -0
- package/storages/utils.d.ts +6 -3
- package/storages/utils.js +11 -2
- package/system-info/runtime.js +7 -7
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -25
- package/validators.js +14 -25
- package/autoscaling/autoscaled_pool.d.ts +0 -213
- package/autoscaling/autoscaled_pool.js +0 -378
- package/autoscaling/client_load_signal.d.ts +0 -59
- package/autoscaling/client_load_signal.js +0 -73
- package/autoscaling/concurrency_system.d.ts +0 -283
- package/autoscaling/concurrency_system.js +0 -350
- package/autoscaling/cpu_load_signal.d.ts +0 -44
- package/autoscaling/cpu_load_signal.js +0 -46
- package/autoscaling/event_loop_load_signal.d.ts +0 -54
- package/autoscaling/event_loop_load_signal.js +0 -60
- package/autoscaling/index.d.ts +0 -9
- package/autoscaling/index.js +0 -9
- package/autoscaling/load_signal.d.ts +0 -99
- package/autoscaling/load_signal.js +0 -103
- package/autoscaling/memory_load_signal.d.ts +0 -56
- package/autoscaling/memory_load_signal.js +0 -106
- package/autoscaling/snapshotter.d.ts +0 -87
- package/autoscaling/snapshotter.js +0 -67
- package/autoscaling/system_status.d.ts +0 -161
- package/autoscaling/system_status.js +0 -139
- package/autoscaling/weighted_avg.d.ts +0 -5
- package/autoscaling/weighted_avg.js +0 -14
- package/cookie_utils.d.ts +0 -44
- package/cookie_utils.js +0 -122
- package/crawlers/context_pipeline.d.ts +0 -70
- package/crawlers/context_pipeline.js +0 -122
- package/crawlers/crawler_commons.d.ts +0 -257
- package/crawlers/crawler_commons.js +0 -107
- package/crawlers/error_snapshotter.d.ts +0 -59
- package/crawlers/error_snapshotter.js +0 -117
- package/crawlers/error_tracker.d.ts +0 -54
- package/crawlers/error_tracker.js +0 -308
- package/crawlers/index.d.ts +0 -5
- package/crawlers/index.js +0 -5
- package/crawlers/internals/types.d.ts +0 -7
- package/crawlers/statistics.d.ts +0 -209
- package/crawlers/statistics.js +0 -350
- package/enqueue_links/enqueue_links.d.ts +0 -264
- package/enqueue_links/enqueue_links.js +0 -271
- package/enqueue_links/index.d.ts +0 -2
- package/enqueue_links/index.js +0 -2
- package/enqueue_links/shared.d.ts +0 -83
- package/enqueue_links/shared.js +0 -221
- package/router.d.ts +0 -309
- package/router.js +0 -309
- package/session_pool/consts.d.ts +0 -3
- package/session_pool/consts.js +0 -3
- package/session_pool/errors.d.ts +0 -7
- package/session_pool/errors.js +0 -11
- package/session_pool/fingerprint.d.ts +0 -9
- package/session_pool/fingerprint.js +0 -30
- package/session_pool/index.d.ts +0 -4
- package/session_pool/index.js +0 -4
- package/session_pool/session.d.ts +0 -161
- package/session_pool/session.js +0 -218
- package/session_pool/session_pool.d.ts +0 -246
- package/session_pool/session_pool.js +0 -386
- package/storages/access_checking.d.ts +0 -12
- package/storages/access_checking.js +0 -17
- package/storages/sitemap_request_loader.d.ts +0 -249
- package/storages/sitemap_request_loader.js +0 -432
- /package/{crawlers/internals/types.js → events/system_info.js} +0 -0
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
import { weightedAvg } from './weighted_avg.js';
|
|
2
|
-
/**
|
|
3
|
-
* A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
|
|
4
|
-
* and so can yours — it is the only part of their machinery worth reusing.
|
|
5
|
-
*/
|
|
6
|
-
export class SnapshotStore {
|
|
7
|
-
snapshots = [];
|
|
8
|
-
/** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
|
|
9
|
-
historyMillis = Infinity;
|
|
10
|
-
/**
|
|
11
|
-
* Sizes retention to the window the signal will be sampled over, as handed to it in
|
|
12
|
-
* {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
|
|
13
|
-
* its start context grows unboundedly.
|
|
14
|
-
*/
|
|
15
|
-
useSampleWindow(maxSampleWindowMillis) {
|
|
16
|
-
this.historyMillis = maxSampleWindowMillis;
|
|
17
|
-
}
|
|
18
|
-
/**
|
|
19
|
-
* Add a snapshot and prune entries older than the history window.
|
|
20
|
-
*/
|
|
21
|
-
push(snapshot, now = snapshot.createdAt) {
|
|
22
|
-
// Inline pruning to avoid private-method transpilation issues
|
|
23
|
-
let oldCount = 0;
|
|
24
|
-
for (let i = 0; i < this.snapshots.length; i++) {
|
|
25
|
-
const { createdAt } = this.snapshots[i];
|
|
26
|
-
if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
|
|
27
|
-
oldCount++;
|
|
28
|
-
else
|
|
29
|
-
break;
|
|
30
|
-
}
|
|
31
|
-
if (oldCount)
|
|
32
|
-
this.snapshots.splice(0, oldCount);
|
|
33
|
-
this.snapshots.push(snapshot);
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Return all snapshots, or only those within the given time window.
|
|
37
|
-
*/
|
|
38
|
-
getSample(sampleDurationMillis) {
|
|
39
|
-
if (!sampleDurationMillis)
|
|
40
|
-
return this.snapshots;
|
|
41
|
-
const sample = [];
|
|
42
|
-
let idx = this.snapshots.length;
|
|
43
|
-
if (!idx)
|
|
44
|
-
return sample;
|
|
45
|
-
const latestTime = this.snapshots[idx - 1].createdAt;
|
|
46
|
-
while (idx--) {
|
|
47
|
-
const snapshot = this.snapshots[idx];
|
|
48
|
-
if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
|
|
49
|
-
sample.unshift(snapshot);
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
break;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return sample;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
|
|
59
|
-
* to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
|
|
60
|
-
*/
|
|
61
|
-
getAll() {
|
|
62
|
-
return this.snapshots;
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
|
|
66
|
-
* samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
|
|
67
|
-
* snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
|
|
68
|
-
* rather than on stop leaves a finished session readable.
|
|
69
|
-
*/
|
|
70
|
-
clear() {
|
|
71
|
-
this.snapshots = [];
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
|
|
76
|
-
* overloaded ratio, using a time-weighted average. This is the shared
|
|
77
|
-
* evaluation logic used by `SystemStatus` for all signal types.
|
|
78
|
-
* @internal
|
|
79
|
-
*/
|
|
80
|
-
export function evaluateLoadSignalSample(sample, overloadedRatio) {
|
|
81
|
-
if (sample.length === 0) {
|
|
82
|
-
return {
|
|
83
|
-
isOverloaded: false,
|
|
84
|
-
limitRatio: overloadedRatio,
|
|
85
|
-
actualRatio: 0,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
const weights = [];
|
|
89
|
-
const values = [];
|
|
90
|
-
for (let i = 1; i < sample.length; i++) {
|
|
91
|
-
const previous = sample[i - 1];
|
|
92
|
-
const current = sample[i];
|
|
93
|
-
const weight = +current.createdAt - +previous.createdAt;
|
|
94
|
-
weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
|
|
95
|
-
values.push(+current.isOverloaded);
|
|
96
|
-
}
|
|
97
|
-
const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
|
|
98
|
-
return {
|
|
99
|
-
isOverloaded: wAvg > overloadedRatio,
|
|
100
|
-
limitRatio: overloadedRatio,
|
|
101
|
-
actualRatio: Math.round(wAvg * 1000) / 1000,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
2
|
-
import type { SystemInfo } from './system_status.js';
|
|
3
|
-
/**
|
|
4
|
-
* A snapshot produced by the built-in memory signal.
|
|
5
|
-
* @internal
|
|
6
|
-
*/
|
|
7
|
-
export interface MemorySnapshot extends LoadSnapshot {
|
|
8
|
-
usedBytes?: number;
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
|
|
12
|
-
* {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
|
|
13
|
-
*/
|
|
14
|
-
export interface MemoryLoadSignalOptions {
|
|
15
|
-
/**
|
|
16
|
-
* Defines the maximum ratio of total memory that can be used.
|
|
17
|
-
* Exceeding this limit overloads the memory.
|
|
18
|
-
* @default 0.9
|
|
19
|
-
*/
|
|
20
|
-
maxUsedRatio?: number;
|
|
21
|
-
/**
|
|
22
|
-
* Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
|
|
23
|
-
* @default 0.2
|
|
24
|
-
*/
|
|
25
|
-
overloadedRatio?: number;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
|
|
29
|
-
* threshold. Also warns when memory use becomes critical.
|
|
30
|
-
*
|
|
31
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
32
|
-
*
|
|
33
|
-
* @category Scaling
|
|
34
|
-
*/
|
|
35
|
-
export declare class MemoryLoadSignal implements LoadSignal {
|
|
36
|
-
readonly name = "memInfo";
|
|
37
|
-
readonly overloadedRatio: number;
|
|
38
|
-
private readonly store;
|
|
39
|
-
private readonly maxUsedRatio;
|
|
40
|
-
/** All resolved in `start()`, before anything that reads them can fire. */
|
|
41
|
-
private config;
|
|
42
|
-
private log;
|
|
43
|
-
private maxMemoryBytes;
|
|
44
|
-
private events?;
|
|
45
|
-
private maxMemoryRatio;
|
|
46
|
-
private lastLoggedCriticalMemoryOverloadAt;
|
|
47
|
-
constructor(options?: MemoryLoadSignalOptions);
|
|
48
|
-
start(context: LoadSignalStartContext): Promise<void>;
|
|
49
|
-
stop(): Promise<void>;
|
|
50
|
-
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
51
|
-
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
52
|
-
handle(systemInfo: SystemInfo): void;
|
|
53
|
-
/** @internal */
|
|
54
|
-
_memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
|
|
55
|
-
private _getTotalMemoryBytes;
|
|
56
|
-
}
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import { serviceLocator } from '../service_locator.js';
|
|
2
|
-
import { getMemoryInfo } from '../system-info/memory-info.js';
|
|
3
|
-
import { isContainerized } from '../system-info/runtime.js';
|
|
4
|
-
import { SnapshotStore } from './load_signal.js';
|
|
5
|
-
const RESERVE_MEMORY_RATIO = 0.5;
|
|
6
|
-
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
7
|
-
/**
|
|
8
|
-
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
|
|
9
|
-
* threshold. Also warns when memory use becomes critical.
|
|
10
|
-
*
|
|
11
|
-
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
12
|
-
*
|
|
13
|
-
* @category Scaling
|
|
14
|
-
*/
|
|
15
|
-
export class MemoryLoadSignal {
|
|
16
|
-
name = 'memInfo';
|
|
17
|
-
overloadedRatio;
|
|
18
|
-
store = new SnapshotStore();
|
|
19
|
-
maxUsedRatio;
|
|
20
|
-
/** All resolved in `start()`, before anything that reads them can fire. */
|
|
21
|
-
config;
|
|
22
|
-
log;
|
|
23
|
-
maxMemoryBytes;
|
|
24
|
-
events;
|
|
25
|
-
maxMemoryRatio;
|
|
26
|
-
lastLoggedCriticalMemoryOverloadAt = null;
|
|
27
|
-
constructor(options = {}) {
|
|
28
|
-
this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
|
|
29
|
-
this.overloadedRatio = options.overloadedRatio ?? 0.2;
|
|
30
|
-
this.handle = this.handle.bind(this);
|
|
31
|
-
}
|
|
32
|
-
async start(context) {
|
|
33
|
-
this.store.useSampleWindow(context.maxSampleWindowMillis);
|
|
34
|
-
// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
|
|
35
|
-
this.store.clear();
|
|
36
|
-
// Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
|
|
37
|
-
// between systems) must not capture whichever services happened to be registered at that moment.
|
|
38
|
-
this.config = serviceLocator.getConfiguration();
|
|
39
|
-
this.events = serviceLocator.getEventManager();
|
|
40
|
-
this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
41
|
-
const memoryMbytes = this.config.memoryMbytes ?? 0;
|
|
42
|
-
if (memoryMbytes > 0) {
|
|
43
|
-
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
this.maxMemoryRatio = this.config.availableMemoryRatio;
|
|
47
|
-
if (!this.maxMemoryRatio) {
|
|
48
|
-
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
49
|
-
}
|
|
50
|
-
else {
|
|
51
|
-
this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
|
|
52
|
-
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
53
|
-
}
|
|
54
|
-
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
55
|
-
this.maxMemoryBytes = await this._getTotalMemoryBytes();
|
|
56
|
-
}
|
|
57
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
58
|
-
}
|
|
59
|
-
async stop() {
|
|
60
|
-
this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
61
|
-
this.events = undefined;
|
|
62
|
-
}
|
|
63
|
-
getSample(sampleDurationMillis) {
|
|
64
|
-
return this.store.getSample(sampleDurationMillis);
|
|
65
|
-
}
|
|
66
|
-
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
67
|
-
handle(systemInfo) {
|
|
68
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
69
|
-
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
70
|
-
let maxMemoryBytes = this.maxMemoryBytes;
|
|
71
|
-
if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
|
|
72
|
-
maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
|
|
73
|
-
}
|
|
74
|
-
const snapshot = {
|
|
75
|
-
createdAt,
|
|
76
|
-
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
|
|
77
|
-
usedBytes: memCurrentBytes,
|
|
78
|
-
};
|
|
79
|
-
this.store.push(snapshot, createdAt);
|
|
80
|
-
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
81
|
-
}
|
|
82
|
-
/** @internal */
|
|
83
|
-
_memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
84
|
-
const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
|
|
85
|
-
const { memCurrentBytes } = systemInfo;
|
|
86
|
-
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
87
|
-
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
88
|
-
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
89
|
-
return;
|
|
90
|
-
const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
|
|
91
|
-
const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
|
|
92
|
-
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
93
|
-
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
94
|
-
if (isCriticalOverload) {
|
|
95
|
-
const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
|
|
96
|
-
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
97
|
-
this.log.warning('Memory is critically overloaded. ' +
|
|
98
|
-
`Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
99
|
-
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
async _getTotalMemoryBytes() {
|
|
103
|
-
const containerized = this.config.containerized ?? (await isContainerized());
|
|
104
|
-
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
105
|
-
}
|
|
106
|
-
}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
import type { ClientLoadSignalOptions } from './client_load_signal.js';
|
|
2
|
-
import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
|
|
3
|
-
import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
|
|
4
|
-
import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
|
|
5
|
-
import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
|
|
6
|
-
/**
|
|
7
|
-
* The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
|
|
8
|
-
*
|
|
9
|
-
* Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
|
|
10
|
-
* corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
|
|
11
|
-
* `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
|
|
12
|
-
* unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
|
|
13
|
-
*
|
|
14
|
-
* How far back the signals are evaluated is *not* set here, but by
|
|
15
|
-
* {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
|
|
16
|
-
* {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
|
|
17
|
-
*/
|
|
18
|
-
export interface LoadSignalsOptions {
|
|
19
|
-
/**
|
|
20
|
-
* Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
|
|
21
|
-
* it off.
|
|
22
|
-
*/
|
|
23
|
-
memory?: MemoryLoadSignalOptions | false;
|
|
24
|
-
/**
|
|
25
|
-
* Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
|
|
26
|
-
* ratio), or `false` to switch it off — which also stops its measuring interval.
|
|
27
|
-
*/
|
|
28
|
-
eventLoop?: EventLoopLoadSignalOptions | false;
|
|
29
|
-
/**
|
|
30
|
-
* Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
|
|
31
|
-
*/
|
|
32
|
-
cpu?: CpuLoadSignalOptions | false;
|
|
33
|
-
/**
|
|
34
|
-
* Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
|
|
35
|
-
* `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
|
|
36
|
-
* signal otherwise polls it every second to no purpose.
|
|
37
|
-
*/
|
|
38
|
-
client?: ClientLoadSignalOptions | false;
|
|
39
|
-
/**
|
|
40
|
-
* Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
|
|
41
|
-
* alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
|
|
42
|
-
* is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
|
|
43
|
-
* not collide with an enabled built-in's.
|
|
44
|
-
*/
|
|
45
|
-
custom?: LoadSignal[];
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
|
|
49
|
-
* {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
|
|
50
|
-
* them here.
|
|
51
|
-
* @internal
|
|
52
|
-
*/
|
|
53
|
-
export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
|
|
54
|
-
/**
|
|
55
|
-
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
56
|
-
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
|
|
57
|
-
* that were not switched off and driving their shared lifecycle.
|
|
58
|
-
*
|
|
59
|
-
* Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
|
|
60
|
-
* are simply forwarded to the corresponding constructor.
|
|
61
|
-
* @internal
|
|
62
|
-
*/
|
|
63
|
-
export declare class Snapshotter {
|
|
64
|
-
private readonly memorySignal?;
|
|
65
|
-
private readonly eventLoopSignal?;
|
|
66
|
-
private readonly cpuSignal?;
|
|
67
|
-
private readonly clientSignal?;
|
|
68
|
-
/**
|
|
69
|
-
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
70
|
-
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
71
|
-
* not overloaded.
|
|
72
|
-
*/
|
|
73
|
-
getLoadSignals(): LoadSignal[];
|
|
74
|
-
/**
|
|
75
|
-
* @param [options] All `Snapshotter` configuration options.
|
|
76
|
-
*/
|
|
77
|
-
constructor(options?: SnapshotterOptions);
|
|
78
|
-
/**
|
|
79
|
-
* Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
|
|
80
|
-
* be queried with, which is also how much history they retain.
|
|
81
|
-
*/
|
|
82
|
-
start(context: LoadSignalStartContext): Promise<void>;
|
|
83
|
-
/**
|
|
84
|
-
* Stops all resource capturing.
|
|
85
|
-
*/
|
|
86
|
-
stop(): Promise<void>;
|
|
87
|
-
}
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
import { ClientLoadSignal } from './client_load_signal.js';
|
|
2
|
-
import { CpuLoadSignal } from './cpu_load_signal.js';
|
|
3
|
-
import { EventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
4
|
-
import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
5
|
-
/**
|
|
6
|
-
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
7
|
-
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
|
|
8
|
-
* that were not switched off and driving their shared lifecycle.
|
|
9
|
-
*
|
|
10
|
-
* Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
|
|
11
|
-
* are simply forwarded to the corresponding constructor.
|
|
12
|
-
* @internal
|
|
13
|
-
*/
|
|
14
|
-
export class Snapshotter {
|
|
15
|
-
// Absent when switched off through the corresponding option (e.g. `client: false`).
|
|
16
|
-
memorySignal;
|
|
17
|
-
eventLoopSignal;
|
|
18
|
-
cpuSignal;
|
|
19
|
-
clientSignal;
|
|
20
|
-
/**
|
|
21
|
-
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
22
|
-
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
23
|
-
* not overloaded.
|
|
24
|
-
*/
|
|
25
|
-
getLoadSignals() {
|
|
26
|
-
const builtin = [
|
|
27
|
-
this.memorySignal,
|
|
28
|
-
this.eventLoopSignal,
|
|
29
|
-
this.cpuSignal,
|
|
30
|
-
this.clientSignal,
|
|
31
|
-
];
|
|
32
|
-
return builtin.filter((signal) => signal !== undefined);
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* @param [options] All `Snapshotter` configuration options.
|
|
36
|
-
*/
|
|
37
|
-
constructor(options = {}) {
|
|
38
|
-
const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
|
|
39
|
-
// Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
|
|
40
|
-
// over then too - so there is nothing to thread in here beyond the caller's tuning.
|
|
41
|
-
if (memory !== false)
|
|
42
|
-
this.memorySignal = new MemoryLoadSignal(memory);
|
|
43
|
-
if (eventLoop !== false)
|
|
44
|
-
this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
|
|
45
|
-
if (cpu !== false)
|
|
46
|
-
this.cpuSignal = new CpuLoadSignal(cpu);
|
|
47
|
-
if (client !== false)
|
|
48
|
-
this.clientSignal = new ClientLoadSignal(client);
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
|
|
52
|
-
* be queried with, which is also how much history they retain.
|
|
53
|
-
*/
|
|
54
|
-
async start(context) {
|
|
55
|
-
await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Stops all resource capturing.
|
|
59
|
-
*/
|
|
60
|
-
async stop() {
|
|
61
|
-
await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
|
|
62
|
-
// Allow microtask queue to unwind before stop returns.
|
|
63
|
-
await new Promise((resolve) => {
|
|
64
|
-
setImmediate(resolve);
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
}
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import type { LoadSignal } from './load_signal.js';
|
|
2
|
-
import type { Snapshotter } from './snapshotter.js';
|
|
3
|
-
/**
|
|
4
|
-
* Represents the current status of the system.
|
|
5
|
-
*/
|
|
6
|
-
export interface SystemInfo {
|
|
7
|
-
/** If false, system is being overloaded. */
|
|
8
|
-
isSystemIdle: boolean;
|
|
9
|
-
memInfo: ClientInfo;
|
|
10
|
-
eventLoopInfo: ClientInfo;
|
|
11
|
-
cpuInfo: ClientInfo;
|
|
12
|
-
clientInfo: ClientInfo;
|
|
13
|
-
memTotalBytes?: number;
|
|
14
|
-
memCurrentBytes?: number;
|
|
15
|
-
/**
|
|
16
|
-
* Platform only property
|
|
17
|
-
* @internal
|
|
18
|
-
*/
|
|
19
|
-
cpuCurrentUsage?: number;
|
|
20
|
-
/**
|
|
21
|
-
* Platform only property
|
|
22
|
-
* @internal
|
|
23
|
-
*/
|
|
24
|
-
isCpuOverloaded?: boolean;
|
|
25
|
-
/**
|
|
26
|
-
* Platform only property
|
|
27
|
-
* @internal
|
|
28
|
-
*/
|
|
29
|
-
createdAt?: Date;
|
|
30
|
-
/**
|
|
31
|
-
* Status of additional load signals beyond the built-in four.
|
|
32
|
-
* Keys are `LoadSignal.name` values, values are overload info.
|
|
33
|
-
*/
|
|
34
|
-
loadSignalInfo?: Record<string, ClientInfo>;
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* How far back the *current* system status looks by default — the window that gates task dispatch.
|
|
38
|
-
* @internal
|
|
39
|
-
*/
|
|
40
|
-
export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
|
|
41
|
-
/**
|
|
42
|
-
* How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
|
|
43
|
-
* therefore how much history the signals are asked to retain.
|
|
44
|
-
* @internal
|
|
45
|
-
*/
|
|
46
|
-
export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
|
|
47
|
-
/**
|
|
48
|
-
* An implementation detail of the {@link ConcurrencySystem} — configure it through
|
|
49
|
-
* {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
|
|
50
|
-
* @internal
|
|
51
|
-
*/
|
|
52
|
-
export interface SystemStatusOptions {
|
|
53
|
-
/**
|
|
54
|
-
* Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
|
|
55
|
-
* @default 5
|
|
56
|
-
*/
|
|
57
|
-
currentHistorySecs?: number;
|
|
58
|
-
/**
|
|
59
|
-
* Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
|
|
60
|
-
* autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
|
|
61
|
-
* private retention cannot silently widen the window.
|
|
62
|
-
* @default 30
|
|
63
|
-
*/
|
|
64
|
-
historySecs?: number;
|
|
65
|
-
/**
|
|
66
|
-
* The `Snapshotter` whose built-in signals are evaluated.
|
|
67
|
-
*/
|
|
68
|
-
snapshotter: Snapshotter;
|
|
69
|
-
/**
|
|
70
|
-
* Additional load signals to include in the system status evaluation.
|
|
71
|
-
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
72
|
-
* and client signals. If any signal reports overload, the system is
|
|
73
|
-
* considered overloaded. Each signal carries its own overload ratio.
|
|
74
|
-
*/
|
|
75
|
-
loadSignals?: LoadSignal[];
|
|
76
|
-
}
|
|
77
|
-
export interface ClientInfo {
|
|
78
|
-
isOverloaded: boolean;
|
|
79
|
-
limitRatio: number;
|
|
80
|
-
actualRatio: number;
|
|
81
|
-
}
|
|
82
|
-
export interface FinalStatistics {
|
|
83
|
-
requestsFinished: number;
|
|
84
|
-
requestsFailed: number;
|
|
85
|
-
retryHistogram: number[];
|
|
86
|
-
requestAvgFailedDurationMillis: number;
|
|
87
|
-
requestAvgFinishedDurationMillis: number;
|
|
88
|
-
requestsFinishedPerMinute: number;
|
|
89
|
-
requestsFailedPerMinute: number;
|
|
90
|
-
requestTotalDurationMillis: number;
|
|
91
|
-
requestsTotal: number;
|
|
92
|
-
crawlerRuntimeMillis: number;
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
|
|
96
|
-
* combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
|
|
97
|
-
* is overloaded whenever at least one of them is.
|
|
98
|
-
*
|
|
99
|
-
* Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
|
|
100
|
-
* widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
|
|
101
|
-
* dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
|
|
102
|
-
*
|
|
103
|
-
* An implementation detail of the {@link ConcurrencySystem}, configured through
|
|
104
|
-
* {@link ConcurrencySystemOptions}.
|
|
105
|
-
* @internal
|
|
106
|
-
*/
|
|
107
|
-
export declare class SystemStatus {
|
|
108
|
-
private readonly currentHistoryMillis;
|
|
109
|
-
private readonly historyMillis;
|
|
110
|
-
private readonly signals;
|
|
111
|
-
constructor(options: SystemStatusOptions);
|
|
112
|
-
/**
|
|
113
|
-
* The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
|
|
114
|
-
* to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
|
|
115
|
-
* defaults.
|
|
116
|
-
*/
|
|
117
|
-
get maxSampleWindowMillis(): number;
|
|
118
|
-
/**
|
|
119
|
-
* Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
|
|
120
|
-
* contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
|
|
121
|
-
* only the last is reported.
|
|
122
|
-
*/
|
|
123
|
-
private assertUniqueSignalNames;
|
|
124
|
-
/**
|
|
125
|
-
* Returns an {@link SystemInfo} object with the following structure:
|
|
126
|
-
*
|
|
127
|
-
* ```javascript
|
|
128
|
-
* {
|
|
129
|
-
* isSystemIdle: Boolean,
|
|
130
|
-
* memInfo: Object,
|
|
131
|
-
* eventLoopInfo: Object,
|
|
132
|
-
* cpuInfo: Object
|
|
133
|
-
* }
|
|
134
|
-
* ```
|
|
135
|
-
*
|
|
136
|
-
* Where the `isSystemIdle` property is set to `false` if the system
|
|
137
|
-
* has been overloaded in the last `options.currentHistorySecs` seconds,
|
|
138
|
-
* and `true` otherwise.
|
|
139
|
-
*/
|
|
140
|
-
getCurrentStatus(): SystemInfo;
|
|
141
|
-
/**
|
|
142
|
-
* Returns an {@link SystemInfo} object with the following structure:
|
|
143
|
-
*
|
|
144
|
-
* ```javascript
|
|
145
|
-
* {
|
|
146
|
-
* isSystemIdle: Boolean,
|
|
147
|
-
* memInfo: Object,
|
|
148
|
-
* eventLoopInfo: Object,
|
|
149
|
-
* cpuInfo: Object
|
|
150
|
-
* }
|
|
151
|
-
* ```
|
|
152
|
-
*
|
|
153
|
-
* Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
|
|
154
|
-
* `historySecs` seconds and `true` otherwise.
|
|
155
|
-
*/
|
|
156
|
-
getHistoricalStatus(): SystemInfo;
|
|
157
|
-
/**
|
|
158
|
-
* Returns a system status object.
|
|
159
|
-
*/
|
|
160
|
-
private isSystemIdle;
|
|
161
|
-
}
|