@crawlee/core 4.0.0-beta.87 → 4.0.0-beta.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +61 -147
- package/autoscaling/autoscaled_pool.js +77 -234
- package/autoscaling/client_load_signal.d.ts +51 -17
- package/autoscaling/client_load_signal.js +68 -31
- package/autoscaling/concurrency_system.d.ts +283 -0
- package/autoscaling/concurrency_system.js +350 -0
- package/autoscaling/cpu_load_signal.d.ts +31 -15
- package/autoscaling/cpu_load_signal.js +41 -19
- package/autoscaling/event_loop_load_signal.d.ts +45 -14
- package/autoscaling/event_loop_load_signal.js +56 -31
- package/autoscaling/index.d.ts +1 -0
- package/autoscaling/index.js +1 -0
- package/autoscaling/load_signal.d.ts +49 -49
- package/autoscaling/load_signal.js +20 -52
- package/autoscaling/memory_load_signal.d.ts +36 -23
- package/autoscaling/memory_load_signal.js +35 -31
- package/autoscaling/snapshotter.d.ts +58 -98
- package/autoscaling/snapshotter.js +36 -122
- package/autoscaling/system_status.d.ts +48 -53
- package/autoscaling/system_status.js +63 -55
- package/package.json +5 -5
|
@@ -5,36 +5,45 @@ import { SnapshotStore } from './load_signal.js';
|
|
|
5
5
|
const RESERVE_MEMORY_RATIO = 0.5;
|
|
6
6
|
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
7
7
|
/**
|
|
8
|
-
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
9
|
-
*
|
|
8
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
|
|
9
|
+
* threshold. Also warns when memory use becomes critical.
|
|
10
|
+
*
|
|
11
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
12
|
+
*
|
|
13
|
+
* @category Scaling
|
|
10
14
|
*/
|
|
11
15
|
export class MemoryLoadSignal {
|
|
12
16
|
name = 'memInfo';
|
|
13
17
|
overloadedRatio;
|
|
14
|
-
store;
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
store = new SnapshotStore();
|
|
19
|
+
maxUsedRatio;
|
|
20
|
+
/** All resolved in `start()`, before anything that reads them can fire. */
|
|
21
|
+
config;
|
|
17
22
|
log;
|
|
18
|
-
maxUsedMemoryRatio;
|
|
19
|
-
maxMemoryRatio;
|
|
20
23
|
maxMemoryBytes;
|
|
24
|
+
events;
|
|
25
|
+
maxMemoryRatio;
|
|
21
26
|
lastLoggedCriticalMemoryOverloadAt = null;
|
|
22
|
-
constructor(options) {
|
|
23
|
-
this.
|
|
24
|
-
this.configuration = options.configuration;
|
|
25
|
-
this.events = serviceLocator.getEventManager();
|
|
26
|
-
this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
27
|
-
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
|
|
27
|
+
constructor(options = {}) {
|
|
28
|
+
this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
|
|
28
29
|
this.overloadedRatio = options.overloadedRatio ?? 0.2;
|
|
29
|
-
this.
|
|
30
|
+
this.handle = this.handle.bind(this);
|
|
30
31
|
}
|
|
31
|
-
async start() {
|
|
32
|
-
|
|
32
|
+
async start(context) {
|
|
33
|
+
this.store.useSampleWindow(context.maxSampleWindowMillis);
|
|
34
|
+
// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
|
|
35
|
+
this.store.clear();
|
|
36
|
+
// Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
|
|
37
|
+
// between systems) must not capture whichever services happened to be registered at that moment.
|
|
38
|
+
this.config = serviceLocator.getConfiguration();
|
|
39
|
+
this.events = serviceLocator.getEventManager();
|
|
40
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
41
|
+
const memoryMbytes = this.config.memoryMbytes ?? 0;
|
|
33
42
|
if (memoryMbytes > 0) {
|
|
34
43
|
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
35
44
|
}
|
|
36
45
|
else {
|
|
37
|
-
this.maxMemoryRatio = this.
|
|
46
|
+
this.maxMemoryRatio = this.config.availableMemoryRatio;
|
|
38
47
|
if (!this.maxMemoryRatio) {
|
|
39
48
|
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
40
49
|
}
|
|
@@ -45,22 +54,17 @@ export class MemoryLoadSignal {
|
|
|
45
54
|
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
46
55
|
this.maxMemoryBytes = await this._getTotalMemoryBytes();
|
|
47
56
|
}
|
|
48
|
-
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.
|
|
57
|
+
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
49
58
|
}
|
|
50
59
|
async stop() {
|
|
51
|
-
this.events
|
|
60
|
+
this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
61
|
+
this.events = undefined;
|
|
52
62
|
}
|
|
53
63
|
getSample(sampleDurationMillis) {
|
|
54
64
|
return this.store.getSample(sampleDurationMillis);
|
|
55
65
|
}
|
|
56
|
-
/**
|
|
57
|
-
|
|
58
|
-
*/
|
|
59
|
-
getMemorySnapshots() {
|
|
60
|
-
return this.store.getAll();
|
|
61
|
-
}
|
|
62
|
-
/** @internal */
|
|
63
|
-
_onSystemInfo(systemInfo) {
|
|
66
|
+
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
67
|
+
handle(systemInfo) {
|
|
64
68
|
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
65
69
|
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
66
70
|
let maxMemoryBytes = this.maxMemoryBytes;
|
|
@@ -69,7 +73,7 @@ export class MemoryLoadSignal {
|
|
|
69
73
|
}
|
|
70
74
|
const snapshot = {
|
|
71
75
|
createdAt,
|
|
72
|
-
isOverloaded: memCurrentBytes / maxMemoryBytes > this.
|
|
76
|
+
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
|
|
73
77
|
usedBytes: memCurrentBytes,
|
|
74
78
|
};
|
|
75
79
|
this.store.push(snapshot, createdAt);
|
|
@@ -83,8 +87,8 @@ export class MemoryLoadSignal {
|
|
|
83
87
|
if (this.lastLoggedCriticalMemoryOverloadAt &&
|
|
84
88
|
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
85
89
|
return;
|
|
86
|
-
const maxDesiredMemoryBytes = this.
|
|
87
|
-
const reserveMemory = effectiveMax * (1 - this.
|
|
90
|
+
const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
|
|
91
|
+
const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
|
|
88
92
|
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
89
93
|
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
90
94
|
if (isCriticalOverload) {
|
|
@@ -96,7 +100,7 @@ export class MemoryLoadSignal {
|
|
|
96
100
|
}
|
|
97
101
|
}
|
|
98
102
|
async _getTotalMemoryBytes() {
|
|
99
|
-
const containerized = this.
|
|
103
|
+
const containerized = this.config.containerized ?? (await isContainerized());
|
|
100
104
|
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
101
105
|
}
|
|
102
106
|
}
|
|
@@ -1,127 +1,87 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
4
|
-
import type {
|
|
5
|
-
import type {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
1
|
+
import type { ClientLoadSignalOptions } from './client_load_signal.js';
|
|
2
|
+
import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
|
|
3
|
+
import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
|
|
4
|
+
import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
|
|
5
|
+
import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
|
|
6
|
+
/**
|
|
7
|
+
* The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
|
|
8
|
+
*
|
|
9
|
+
* Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
|
|
10
|
+
* corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
|
|
11
|
+
* `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
|
|
12
|
+
* unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
|
|
13
|
+
*
|
|
14
|
+
* How far back the signals are evaluated is *not* set here, but by
|
|
15
|
+
* {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
|
|
16
|
+
* {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
|
|
17
|
+
*/
|
|
18
|
+
export interface LoadSignalsOptions {
|
|
15
19
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* @default 1
|
|
20
|
+
* Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
|
|
21
|
+
* it off.
|
|
19
22
|
*/
|
|
20
|
-
|
|
23
|
+
memory?: MemoryLoadSignalOptions | false;
|
|
21
24
|
/**
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* @default 50
|
|
25
|
+
* Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
|
|
26
|
+
* ratio), or `false` to switch it off — which also stops its measuring interval.
|
|
25
27
|
*/
|
|
26
|
-
|
|
28
|
+
eventLoop?: EventLoopLoadSignalOptions | false;
|
|
27
29
|
/**
|
|
28
|
-
*
|
|
29
|
-
* Exceeding this limit overloads the memory.
|
|
30
|
-
* @default 0.9
|
|
30
|
+
* Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
|
|
31
31
|
*/
|
|
32
|
-
|
|
32
|
+
cpu?: CpuLoadSignalOptions | false;
|
|
33
33
|
/**
|
|
34
|
-
*
|
|
35
|
-
* the
|
|
36
|
-
*
|
|
34
|
+
* Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
|
|
35
|
+
* `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
|
|
36
|
+
* signal otherwise polls it every second to no purpose.
|
|
37
37
|
*/
|
|
38
|
-
|
|
38
|
+
client?: ClientLoadSignalOptions | false;
|
|
39
39
|
/**
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* @
|
|
40
|
+
* Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
|
|
41
|
+
* alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
|
|
42
|
+
* is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
|
|
43
|
+
* not collide with an enabled built-in's.
|
|
43
44
|
*/
|
|
44
|
-
|
|
45
|
-
/** @internal */
|
|
46
|
-
log?: CrawleeLogger;
|
|
47
|
-
/** @internal */
|
|
48
|
-
client?: StorageBackend;
|
|
49
|
-
/** @internal */
|
|
50
|
-
configuration?: Configuration;
|
|
45
|
+
custom?: LoadSignal[];
|
|
51
46
|
}
|
|
52
47
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* when Apify platform marks it as overloaded.
|
|
64
|
-
*
|
|
65
|
-
* Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
|
|
66
|
-
* It's computed using the total memory available to the container when running on
|
|
67
|
-
* the Apify platform and a quarter of total system memory when running locally.
|
|
68
|
-
* Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
|
|
69
|
-
* environment variable.
|
|
70
|
-
*
|
|
71
|
-
* Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
|
|
72
|
-
*
|
|
73
|
-
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
74
|
-
* typically received from the request queue, exceed the set limit within the set interval.
|
|
48
|
+
* An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
|
|
49
|
+
* {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
|
|
50
|
+
* them here.
|
|
51
|
+
* @internal
|
|
52
|
+
*/
|
|
53
|
+
export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
|
|
54
|
+
/**
|
|
55
|
+
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
56
|
+
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
|
|
57
|
+
* that were not switched off and driving their shared lifecycle.
|
|
75
58
|
*
|
|
76
|
-
* @
|
|
59
|
+
* Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
|
|
60
|
+
* are simply forwarded to the corresponding constructor.
|
|
61
|
+
* @internal
|
|
77
62
|
*/
|
|
78
63
|
export declare class Snapshotter {
|
|
79
|
-
readonly
|
|
80
|
-
readonly
|
|
81
|
-
readonly
|
|
82
|
-
private readonly
|
|
83
|
-
private readonly eventLoopSignal;
|
|
84
|
-
private readonly cpuSignal;
|
|
85
|
-
private readonly clientSignal;
|
|
64
|
+
private readonly memorySignal?;
|
|
65
|
+
private readonly eventLoopSignal?;
|
|
66
|
+
private readonly cpuSignal?;
|
|
67
|
+
private readonly clientSignal?;
|
|
86
68
|
/**
|
|
87
|
-
* Returns the
|
|
88
|
-
*
|
|
69
|
+
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
70
|
+
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
71
|
+
* not overloaded.
|
|
89
72
|
*/
|
|
90
73
|
getLoadSignals(): LoadSignal[];
|
|
91
|
-
get cpuSnapshots(): CpuSnapshot[];
|
|
92
|
-
get eventLoopSnapshots(): EventLoopSnapshot[];
|
|
93
|
-
get memorySnapshots(): MemorySnapshot[];
|
|
94
|
-
get clientSnapshots(): ClientSnapshot[];
|
|
95
74
|
/**
|
|
96
75
|
* @param [options] All `Snapshotter` configuration options.
|
|
97
76
|
*/
|
|
98
77
|
constructor(options?: SnapshotterOptions);
|
|
99
78
|
/**
|
|
100
|
-
* Starts capturing snapshots at configured intervals.
|
|
79
|
+
* Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
|
|
80
|
+
* be queried with, which is also how much history they retain.
|
|
101
81
|
*/
|
|
102
|
-
start(): Promise<void>;
|
|
82
|
+
start(context: LoadSignalStartContext): Promise<void>;
|
|
103
83
|
/**
|
|
104
84
|
* Stops all resource capturing.
|
|
105
85
|
*/
|
|
106
86
|
stop(): Promise<void>;
|
|
107
|
-
/**
|
|
108
|
-
* Returns a sample of latest memory snapshots, with the size of the sample defined
|
|
109
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
110
|
-
*/
|
|
111
|
-
getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];
|
|
112
|
-
/**
|
|
113
|
-
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
114
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
115
|
-
*/
|
|
116
|
-
getEventLoopSample(sampleDurationMillis?: number): EventLoopSnapshot[];
|
|
117
|
-
/**
|
|
118
|
-
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
119
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
120
|
-
*/
|
|
121
|
-
getCpuSample(sampleDurationMillis?: number): CpuSnapshot[];
|
|
122
|
-
/**
|
|
123
|
-
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
124
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
125
|
-
*/
|
|
126
|
-
getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
|
|
127
87
|
}
|
|
@@ -1,153 +1,67 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { createCpuLoadSignal } from './cpu_load_signal.js';
|
|
5
|
-
import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
1
|
+
import { ClientLoadSignal } from './client_load_signal.js';
|
|
2
|
+
import { CpuLoadSignal } from './cpu_load_signal.js';
|
|
3
|
+
import { EventLoopLoadSignal } from './event_loop_load_signal.js';
|
|
6
4
|
import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
7
5
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* The class is used by the {@link AutoscaledPool} class.
|
|
6
|
+
* Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
|
|
7
|
+
* {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
|
|
8
|
+
* that were not switched off and driving their shared lifecycle.
|
|
12
9
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
|
|
18
|
-
* when Apify platform marks it as overloaded.
|
|
19
|
-
*
|
|
20
|
-
* Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
|
|
21
|
-
* It's computed using the total memory available to the container when running on
|
|
22
|
-
* the Apify platform and a quarter of total system memory when running locally.
|
|
23
|
-
* Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
|
|
24
|
-
* environment variable.
|
|
25
|
-
*
|
|
26
|
-
* Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
|
|
27
|
-
*
|
|
28
|
-
* Client becomes overloaded when rate limit errors (429 - Too Many Requests),
|
|
29
|
-
* typically received from the request queue, exceed the set limit within the set interval.
|
|
30
|
-
*
|
|
31
|
-
* @category Scaling
|
|
10
|
+
* Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
|
|
11
|
+
* are simply forwarded to the corresponding constructor.
|
|
12
|
+
* @internal
|
|
32
13
|
*/
|
|
33
14
|
export class Snapshotter {
|
|
34
|
-
|
|
35
|
-
client;
|
|
36
|
-
configuration;
|
|
15
|
+
// Absent when switched off through the corresponding option (e.g. `client: false`).
|
|
37
16
|
memorySignal;
|
|
38
17
|
eventLoopSignal;
|
|
39
18
|
cpuSignal;
|
|
40
19
|
clientSignal;
|
|
41
20
|
/**
|
|
42
|
-
* Returns the
|
|
43
|
-
*
|
|
21
|
+
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
22
|
+
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
23
|
+
* not overloaded.
|
|
44
24
|
*/
|
|
45
25
|
getLoadSignals() {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
return this.eventLoopSignal.store.getAll();
|
|
54
|
-
}
|
|
55
|
-
get memorySnapshots() {
|
|
56
|
-
return this.memorySignal.getMemorySnapshots();
|
|
57
|
-
}
|
|
58
|
-
get clientSnapshots() {
|
|
59
|
-
return this.clientSignal.store.getAll();
|
|
26
|
+
const builtin = [
|
|
27
|
+
this.memorySignal,
|
|
28
|
+
this.eventLoopSignal,
|
|
29
|
+
this.cpuSignal,
|
|
30
|
+
this.clientSignal,
|
|
31
|
+
];
|
|
32
|
+
return builtin.filter((signal) => signal !== undefined);
|
|
60
33
|
}
|
|
61
34
|
/**
|
|
62
35
|
* @param [options] All `Snapshotter` configuration options.
|
|
63
36
|
*/
|
|
64
37
|
constructor(options = {}) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), configuration = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
|
|
77
|
-
this.log = log.child({ prefix: 'Snapshotter' });
|
|
78
|
-
this.client = client;
|
|
79
|
-
this.configuration = configuration;
|
|
80
|
-
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
81
|
-
this.memorySignal = new MemoryLoadSignal({
|
|
82
|
-
maxUsedMemoryRatio,
|
|
83
|
-
snapshotHistoryMillis,
|
|
84
|
-
configuration: this.configuration,
|
|
85
|
-
log: this.log,
|
|
86
|
-
});
|
|
87
|
-
this.eventLoopSignal = createEventLoopLoadSignal({
|
|
88
|
-
eventLoopSnapshotIntervalSecs,
|
|
89
|
-
maxBlockedMillis,
|
|
90
|
-
snapshotHistoryMillis,
|
|
91
|
-
});
|
|
92
|
-
this.cpuSignal = createCpuLoadSignal({
|
|
93
|
-
snapshotHistoryMillis,
|
|
94
|
-
configuration: this.configuration,
|
|
95
|
-
});
|
|
96
|
-
this.clientSignal = createClientLoadSignal({
|
|
97
|
-
client: this.client,
|
|
98
|
-
clientSnapshotIntervalSecs,
|
|
99
|
-
maxClientErrors,
|
|
100
|
-
snapshotHistoryMillis,
|
|
101
|
-
});
|
|
38
|
+
const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
|
|
39
|
+
// Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
|
|
40
|
+
// over then too - so there is nothing to thread in here beyond the caller's tuning.
|
|
41
|
+
if (memory !== false)
|
|
42
|
+
this.memorySignal = new MemoryLoadSignal(memory);
|
|
43
|
+
if (eventLoop !== false)
|
|
44
|
+
this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
|
|
45
|
+
if (cpu !== false)
|
|
46
|
+
this.cpuSignal = new CpuLoadSignal(cpu);
|
|
47
|
+
if (client !== false)
|
|
48
|
+
this.clientSignal = new ClientLoadSignal(client);
|
|
102
49
|
}
|
|
103
50
|
/**
|
|
104
|
-
* Starts capturing snapshots at configured intervals.
|
|
51
|
+
* Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
|
|
52
|
+
* be queried with, which is also how much history they retain.
|
|
105
53
|
*/
|
|
106
|
-
async start() {
|
|
107
|
-
await this.
|
|
108
|
-
await this.eventLoopSignal.start();
|
|
109
|
-
await this.cpuSignal.start();
|
|
110
|
-
await this.clientSignal.start();
|
|
54
|
+
async start(context) {
|
|
55
|
+
await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
|
|
111
56
|
}
|
|
112
57
|
/**
|
|
113
58
|
* Stops all resource capturing.
|
|
114
59
|
*/
|
|
115
60
|
async stop() {
|
|
116
|
-
await this.
|
|
117
|
-
await this.eventLoopSignal.stop();
|
|
118
|
-
await this.cpuSignal.stop();
|
|
119
|
-
await this.clientSignal.stop();
|
|
61
|
+
await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
|
|
120
62
|
// Allow microtask queue to unwind before stop returns.
|
|
121
63
|
await new Promise((resolve) => {
|
|
122
64
|
setImmediate(resolve);
|
|
123
65
|
});
|
|
124
66
|
}
|
|
125
|
-
/**
|
|
126
|
-
* Returns a sample of latest memory snapshots, with the size of the sample defined
|
|
127
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
128
|
-
*/
|
|
129
|
-
getMemorySample(sampleDurationMillis) {
|
|
130
|
-
return this.memorySignal.getSample(sampleDurationMillis);
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* Returns a sample of latest event loop snapshots, with the size of the sample defined
|
|
134
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
135
|
-
*/
|
|
136
|
-
getEventLoopSample(sampleDurationMillis) {
|
|
137
|
-
return this.eventLoopSignal.getSample(sampleDurationMillis);
|
|
138
|
-
}
|
|
139
|
-
/**
|
|
140
|
-
* Returns a sample of latest CPU snapshots, with the size of the sample defined
|
|
141
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
142
|
-
*/
|
|
143
|
-
getCpuSample(sampleDurationMillis) {
|
|
144
|
-
return this.cpuSignal.getSample(sampleDurationMillis);
|
|
145
|
-
}
|
|
146
|
-
/**
|
|
147
|
-
* Returns a sample of latest Client snapshots, with the size of the sample defined
|
|
148
|
-
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
|
|
149
|
-
*/
|
|
150
|
-
getClientSample(sampleDurationMillis) {
|
|
151
|
-
return this.clientSignal.getSample(sampleDurationMillis);
|
|
152
|
-
}
|
|
153
67
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LoadSignal } from './load_signal.js';
|
|
2
|
-
import { Snapshotter } from './snapshotter.js';
|
|
2
|
+
import type { Snapshotter } from './snapshotter.js';
|
|
3
3
|
/**
|
|
4
4
|
* Represents the current status of the system.
|
|
5
5
|
*/
|
|
@@ -33,6 +33,22 @@ export interface SystemInfo {
|
|
|
33
33
|
*/
|
|
34
34
|
loadSignalInfo?: Record<string, ClientInfo>;
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* How far back the *current* system status looks by default — the window that gates task dispatch.
|
|
38
|
+
* @internal
|
|
39
|
+
*/
|
|
40
|
+
export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
|
|
41
|
+
/**
|
|
42
|
+
* How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
|
|
43
|
+
* therefore how much history the signals are asked to retain.
|
|
44
|
+
* @internal
|
|
45
|
+
*/
|
|
46
|
+
export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
|
|
47
|
+
/**
|
|
48
|
+
* An implementation detail of the {@link ConcurrencySystem} — configure it through
|
|
49
|
+
* {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
|
|
50
|
+
* @internal
|
|
51
|
+
*/
|
|
36
52
|
export interface SystemStatusOptions {
|
|
37
53
|
/**
|
|
38
54
|
* Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
|
|
@@ -40,38 +56,21 @@ export interface SystemStatusOptions {
|
|
|
40
56
|
*/
|
|
41
57
|
currentHistorySecs?: number;
|
|
42
58
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
|
|
47
|
-
maxMemoryOverloadedRatio?: number;
|
|
48
|
-
/**
|
|
49
|
-
* Sets the maximum ratio of overloaded snapshots in an event loop sample.
|
|
50
|
-
* If the sample exceeds this ratio, the system will be overloaded.
|
|
51
|
-
* @default 0.6
|
|
59
|
+
* Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
|
|
60
|
+
* autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
|
|
61
|
+
* private retention cannot silently widen the window.
|
|
62
|
+
* @default 30
|
|
52
63
|
*/
|
|
53
|
-
|
|
64
|
+
historySecs?: number;
|
|
54
65
|
/**
|
|
55
|
-
*
|
|
56
|
-
* If the sample exceeds this ratio, the system will be overloaded.
|
|
57
|
-
* @default 0.4
|
|
66
|
+
* The `Snapshotter` whose built-in signals are evaluated.
|
|
58
67
|
*/
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* Sets the maximum ratio of overloaded snapshots in a Client sample.
|
|
62
|
-
* If the sample exceeds this ratio, the system will be overloaded.
|
|
63
|
-
* @default 0.3
|
|
64
|
-
*/
|
|
65
|
-
maxClientOverloadedRatio?: number;
|
|
66
|
-
/**
|
|
67
|
-
* The `Snapshotter` instance to be queried for `SystemStatus`.
|
|
68
|
-
*/
|
|
69
|
-
snapshotter?: Snapshotter;
|
|
68
|
+
snapshotter: Snapshotter;
|
|
70
69
|
/**
|
|
71
70
|
* Additional load signals to include in the system status evaluation.
|
|
72
71
|
* These are evaluated alongside the built-in memory, CPU, event loop,
|
|
73
72
|
* and client signals. If any signal reports overload, the system is
|
|
74
|
-
* considered overloaded.
|
|
73
|
+
* considered overloaded. Each signal carries its own overload ratio.
|
|
75
74
|
*/
|
|
76
75
|
loadSignals?: LoadSignal[];
|
|
77
76
|
}
|
|
@@ -93,38 +92,35 @@ export interface FinalStatistics {
|
|
|
93
92
|
crawlerRuntimeMillis: number;
|
|
94
93
|
}
|
|
95
94
|
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
* The system status is calculated using a weighted average of overloaded
|
|
100
|
-
* messages in the snapshots, with the weights being the time intervals
|
|
101
|
-
* between the snapshots. Each resource is calculated separately
|
|
102
|
-
* and the system is overloaded whenever at least one resource is overloaded.
|
|
103
|
-
* The class is used by the {@link AutoscaledPool} class.
|
|
95
|
+
* Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
|
|
96
|
+
* combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
|
|
97
|
+
* is overloaded whenever at least one of them is.
|
|
104
98
|
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
* by the `currentHistorySecs` option and represents the max age
|
|
109
|
-
* of snapshots to be considered for the calculation.
|
|
99
|
+
* Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
|
|
100
|
+
* widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
|
|
101
|
+
* dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
|
|
110
102
|
*
|
|
111
|
-
* {@link
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
* in the {@link Snapshotter} instance.
|
|
115
|
-
* @category Scaling
|
|
103
|
+
* An implementation detail of the {@link ConcurrencySystem}, configured through
|
|
104
|
+
* {@link ConcurrencySystemOptions}.
|
|
105
|
+
* @internal
|
|
116
106
|
*/
|
|
117
107
|
export declare class SystemStatus {
|
|
118
108
|
private readonly currentHistoryMillis;
|
|
119
|
-
private readonly
|
|
109
|
+
private readonly historyMillis;
|
|
120
110
|
private readonly signals;
|
|
111
|
+
constructor(options: SystemStatusOptions);
|
|
121
112
|
/**
|
|
122
|
-
*
|
|
123
|
-
* the
|
|
124
|
-
*
|
|
113
|
+
* The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
|
|
114
|
+
* to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
|
|
115
|
+
* defaults.
|
|
125
116
|
*/
|
|
126
|
-
|
|
127
|
-
|
|
117
|
+
get maxSampleWindowMillis(): number;
|
|
118
|
+
/**
|
|
119
|
+
* Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
|
|
120
|
+
* contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
|
|
121
|
+
* only the last is reported.
|
|
122
|
+
*/
|
|
123
|
+
private assertUniqueSignalNames;
|
|
128
124
|
/**
|
|
129
125
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
130
126
|
*
|
|
@@ -154,9 +150,8 @@ export declare class SystemStatus {
|
|
|
154
150
|
* }
|
|
155
151
|
* ```
|
|
156
152
|
*
|
|
157
|
-
* Where the `isSystemIdle` property is set to `false` if the system
|
|
158
|
-
*
|
|
159
|
-
* (which is configurable in the {@link Snapshotter}) and `true` otherwise.
|
|
153
|
+
* Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
|
|
154
|
+
* `historySecs` seconds and `true` otherwise.
|
|
160
155
|
*/
|
|
161
156
|
getHistoricalStatus(): SystemInfo;
|
|
162
157
|
/**
|