@crawlee/core 4.0.0-beta.87 → 4.0.0-beta.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +61 -147
- package/autoscaling/autoscaled_pool.js +77 -234
- package/autoscaling/client_load_signal.d.ts +51 -17
- package/autoscaling/client_load_signal.js +68 -31
- package/autoscaling/concurrency_system.d.ts +283 -0
- package/autoscaling/concurrency_system.js +350 -0
- package/autoscaling/cpu_load_signal.d.ts +31 -15
- package/autoscaling/cpu_load_signal.js +41 -19
- package/autoscaling/event_loop_load_signal.d.ts +45 -14
- package/autoscaling/event_loop_load_signal.js +56 -31
- package/autoscaling/index.d.ts +1 -0
- package/autoscaling/index.js +1 -0
- package/autoscaling/load_signal.d.ts +49 -49
- package/autoscaling/load_signal.js +20 -52
- package/autoscaling/memory_load_signal.d.ts +36 -23
- package/autoscaling/memory_load_signal.js +35 -31
- package/autoscaling/snapshotter.d.ts +58 -98
- package/autoscaling/snapshotter.js +36 -122
- package/autoscaling/system_status.d.ts +48 -53
- package/autoscaling/system_status.js +63 -55
- package/package.json +5 -5
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { LoadSnapshot } from './load_signal.js';
|
|
3
|
-
import { SnapshotStore } from './load_signal.js';
|
|
1
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
4
2
|
import type { SystemInfo } from './system_status.js';
|
|
3
|
+
/**
|
|
4
|
+
* A snapshot produced by the built-in CPU signal.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
5
7
|
export interface CpuSnapshot extends LoadSnapshot {
|
|
6
8
|
usedRatio: number;
|
|
7
9
|
ticks?: {
|
|
@@ -9,20 +11,34 @@ export interface CpuSnapshot extends LoadSnapshot {
|
|
|
9
11
|
total: number;
|
|
10
12
|
};
|
|
11
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Tuning for the built-in **CPU** load signal, as accepted both by {@link CpuLoadSignal} and by the
|
|
16
|
+
* {@link LoadSignalsOptions.cpu|`cpu`} shorthand on {@link LoadSignalsOptions}.
|
|
17
|
+
*/
|
|
12
18
|
export interface CpuLoadSignalOptions {
|
|
19
|
+
/**
|
|
20
|
+
* Maximum ratio of overloaded snapshots in a sample before the CPU counts as overloaded.
|
|
21
|
+
* @default 0.4
|
|
22
|
+
*/
|
|
13
23
|
overloadedRatio?: number;
|
|
14
|
-
snapshotHistoryMillis?: number;
|
|
15
|
-
configuration: Configuration;
|
|
16
24
|
}
|
|
17
25
|
/**
|
|
18
|
-
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
|
|
19
|
-
*
|
|
26
|
+
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
27
|
+
* CPU is overloaded.
|
|
28
|
+
*
|
|
29
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
30
|
+
*
|
|
31
|
+
* @category Scaling
|
|
20
32
|
*/
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
33
|
+
export declare class CpuLoadSignal implements LoadSignal {
|
|
34
|
+
readonly name = "cpuInfo";
|
|
35
|
+
readonly overloadedRatio: number;
|
|
36
|
+
private readonly store;
|
|
37
|
+
private events?;
|
|
38
|
+
constructor(options?: CpuLoadSignalOptions);
|
|
39
|
+
start(context: LoadSignalStartContext): Promise<void>;
|
|
40
|
+
stop(): Promise<void>;
|
|
41
|
+
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
42
|
+
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
43
|
+
handle(systemInfo: SystemInfo): void;
|
|
44
|
+
}
|
|
@@ -1,24 +1,46 @@
|
|
|
1
1
|
import { serviceLocator } from '../service_locator.js';
|
|
2
2
|
import { SnapshotStore } from './load_signal.js';
|
|
3
3
|
/**
|
|
4
|
-
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
|
|
5
|
-
*
|
|
4
|
+
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
|
|
5
|
+
* CPU is overloaded.
|
|
6
|
+
*
|
|
7
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
8
|
+
*
|
|
9
|
+
* @category Scaling
|
|
6
10
|
*/
|
|
7
|
-
export
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
11
|
+
export class CpuLoadSignal {
|
|
12
|
+
name = 'cpuInfo';
|
|
13
|
+
overloadedRatio;
|
|
14
|
+
store = new SnapshotStore();
|
|
15
|
+
events;
|
|
16
|
+
constructor(options = {}) {
|
|
17
|
+
this.overloadedRatio = options.overloadedRatio ?? 0.4;
|
|
18
|
+
this.handle = this.handle.bind(this);
|
|
19
|
+
}
|
|
20
|
+
async start(context) {
|
|
21
|
+
this.store.useSampleWindow(context.maxSampleWindowMillis);
|
|
22
|
+
// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
|
|
23
|
+
this.store.clear();
|
|
24
|
+
// Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
|
|
25
|
+
// between systems) cannot capture whichever event manager happened to be registered at that moment.
|
|
26
|
+
this.events = serviceLocator.getEventManager();
|
|
27
|
+
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
28
|
+
}
|
|
29
|
+
async stop() {
|
|
30
|
+
this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
31
|
+
this.events = undefined;
|
|
32
|
+
}
|
|
33
|
+
getSample(sampleDurationMillis) {
|
|
34
|
+
return this.store.getSample(sampleDurationMillis);
|
|
35
|
+
}
|
|
36
|
+
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
37
|
+
handle(systemInfo) {
|
|
38
|
+
const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
|
|
39
|
+
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
40
|
+
this.store.push({
|
|
41
|
+
createdAt,
|
|
42
|
+
isOverloaded: isCpuOverloaded,
|
|
43
|
+
usedRatio: Math.ceil(cpuCurrentUsage / 100),
|
|
44
|
+
}, createdAt);
|
|
45
|
+
}
|
|
24
46
|
}
|
|
@@ -1,23 +1,54 @@
|
|
|
1
|
-
import type { LoadSnapshot } from './load_signal.js';
|
|
2
|
-
|
|
1
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
2
|
+
/**
|
|
3
|
+
* A snapshot produced by the built-in event loop signal.
|
|
4
|
+
* @internal
|
|
5
|
+
*/
|
|
3
6
|
export interface EventLoopSnapshot extends LoadSnapshot {
|
|
4
7
|
exceededMillis: number;
|
|
5
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Tuning for the built-in **event loop** load signal, as accepted both by {@link EventLoopLoadSignal} and by the
|
|
11
|
+
* {@link LoadSignalsOptions.eventLoop|`eventLoop`} shorthand on {@link LoadSignalsOptions}.
|
|
12
|
+
*/
|
|
6
13
|
export interface EventLoopLoadSignalOptions {
|
|
7
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Defines the interval of measuring the event loop response time, in seconds.
|
|
16
|
+
* @default 0.5
|
|
17
|
+
*/
|
|
18
|
+
snapshotIntervalSecs?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Maximum allowed delay of the event loop in milliseconds.
|
|
21
|
+
* Exceeding this limit overloads the event loop.
|
|
22
|
+
* @default 50
|
|
23
|
+
*/
|
|
8
24
|
maxBlockedMillis?: number;
|
|
25
|
+
/**
|
|
26
|
+
* Maximum ratio of overloaded snapshots in a sample before the event loop counts as overloaded.
|
|
27
|
+
* @default 0.6
|
|
28
|
+
*/
|
|
9
29
|
overloadedRatio?: number;
|
|
10
|
-
snapshotHistoryMillis?: number;
|
|
11
30
|
}
|
|
12
31
|
/**
|
|
13
|
-
* Periodically measures event loop delay and reports overload when the
|
|
14
|
-
*
|
|
32
|
+
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
33
|
+
*
|
|
34
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
35
|
+
*
|
|
36
|
+
* @category Scaling
|
|
15
37
|
*/
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
38
|
+
export declare class EventLoopLoadSignal implements LoadSignal {
|
|
39
|
+
readonly name = "eventLoopInfo";
|
|
40
|
+
readonly overloadedRatio: number;
|
|
41
|
+
private readonly store;
|
|
42
|
+
private readonly intervalMillis;
|
|
43
|
+
private readonly maxBlockedMillis;
|
|
44
|
+
private interval?;
|
|
45
|
+
constructor(options?: EventLoopLoadSignalOptions);
|
|
46
|
+
start(context: LoadSignalStartContext): Promise<void>;
|
|
47
|
+
stop(): Promise<void>;
|
|
48
|
+
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
49
|
+
/**
|
|
50
|
+
* Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
|
|
51
|
+
* @internal Also lets tests drive the measurement without waiting on a timer.
|
|
52
|
+
*/
|
|
53
|
+
handle(intervalCallback: () => unknown): void;
|
|
54
|
+
}
|
|
@@ -1,35 +1,60 @@
|
|
|
1
|
+
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
1
2
|
import { SnapshotStore } from './load_signal.js';
|
|
2
3
|
/**
|
|
3
|
-
* Periodically measures event loop delay and reports overload when the
|
|
4
|
-
*
|
|
4
|
+
* Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
|
|
5
|
+
*
|
|
6
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
7
|
+
*
|
|
8
|
+
* @category Scaling
|
|
5
9
|
*/
|
|
6
|
-
export
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
|
|
10
|
+
export class EventLoopLoadSignal {
|
|
11
|
+
name = 'eventLoopInfo';
|
|
12
|
+
overloadedRatio;
|
|
13
|
+
store = new SnapshotStore();
|
|
14
|
+
intervalMillis;
|
|
15
|
+
maxBlockedMillis;
|
|
16
|
+
interval;
|
|
17
|
+
constructor(options = {}) {
|
|
18
|
+
this.overloadedRatio = options.overloadedRatio ?? 0.6;
|
|
19
|
+
this.intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
|
|
20
|
+
this.maxBlockedMillis = options.maxBlockedMillis ?? 50;
|
|
21
|
+
this.handle = this.handle.bind(this);
|
|
22
|
+
}
|
|
23
|
+
async start(context) {
|
|
24
|
+
this.store.useSampleWindow(context.maxSampleWindowMillis);
|
|
25
|
+
// A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
|
|
26
|
+
// the gap since the previous snapshot, which across a restart is however long the system was stopped.
|
|
27
|
+
this.store.clear();
|
|
28
|
+
this.interval = betterSetInterval(this.handle, this.intervalMillis);
|
|
29
|
+
}
|
|
30
|
+
async stop() {
|
|
31
|
+
if (this.interval)
|
|
32
|
+
betterClearInterval(this.interval);
|
|
33
|
+
this.interval = undefined;
|
|
34
|
+
}
|
|
35
|
+
getSample(sampleDurationMillis) {
|
|
36
|
+
return this.store.getSample(sampleDurationMillis);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
|
|
40
|
+
* @internal Also lets tests drive the measurement without waiting on a timer.
|
|
41
|
+
*/
|
|
42
|
+
handle(intervalCallback) {
|
|
43
|
+
const now = new Date();
|
|
44
|
+
const snapshot = {
|
|
45
|
+
createdAt: now,
|
|
46
|
+
isOverloaded: false,
|
|
47
|
+
exceededMillis: 0,
|
|
48
|
+
};
|
|
49
|
+
const all = this.store.getAll();
|
|
50
|
+
const previousSnapshot = all[all.length - 1];
|
|
51
|
+
if (previousSnapshot) {
|
|
52
|
+
const delta = now.getTime() - +previousSnapshot.createdAt - this.intervalMillis;
|
|
53
|
+
if (delta > this.maxBlockedMillis)
|
|
54
|
+
snapshot.isOverloaded = true;
|
|
55
|
+
snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
|
|
56
|
+
}
|
|
57
|
+
this.store.push(snapshot, now);
|
|
58
|
+
intervalCallback();
|
|
59
|
+
}
|
|
35
60
|
}
|
package/autoscaling/index.d.ts
CHANGED
package/autoscaling/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { EventManager, EventTypeName } from '../events/event_manager.js';
|
|
2
1
|
import type { ClientInfo } from './system_status.js';
|
|
3
2
|
/**
|
|
4
3
|
* A snapshot of a resource's overload state at a point in time.
|
|
@@ -8,15 +7,34 @@ export interface LoadSnapshot {
|
|
|
8
7
|
isOverloaded: boolean;
|
|
9
8
|
}
|
|
10
9
|
/**
|
|
11
|
-
*
|
|
10
|
+
* Handed to a {@link LoadSignal} when it starts, so it can size its snapshot retention to what it will actually
|
|
11
|
+
* be asked for — without having to know how the {@link ConcurrencySystem} that drives it is configured.
|
|
12
|
+
*/
|
|
13
|
+
export interface LoadSignalStartContext {
|
|
14
|
+
/**
|
|
15
|
+
* The longest sample window the signal will be queried with (the wider of the task-gating and autoscaling
|
|
16
|
+
* windows). Keeping less history than this contributes a narrower view of the resource than the other signals;
|
|
17
|
+
* keeping more is wasted memory, as the extra snapshots are never sampled.
|
|
18
|
+
*/
|
|
19
|
+
maxSampleWindowMillis: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* A signal that reports whether a particular resource is overloaded. The {@link ConcurrencySystem} aggregates
|
|
23
|
+
* several of them — if any one reports overload, the system is overloaded.
|
|
12
24
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
25
|
+
* The built-in signals cover memory, CPU, event loop and storage-client rate limits. Implement this interface to add
|
|
26
|
+
* your own (navigation timeouts, proxy health, …) and pass them via
|
|
27
|
+
* {@link LoadSignalsOptions.custom|`loadSignals.custom`}; {@link SnapshotStore} does the time-windowed
|
|
28
|
+
* bookkeeping if you want it. Each built-in is also a public class, so one can be *wrapped* rather than reimplemented
|
|
29
|
+
* — construct it yourself and switch the default off with {@link LoadSignalsOptions.cpu|`cpu: false`} or friends.
|
|
17
30
|
*/
|
|
18
31
|
export interface LoadSignal {
|
|
19
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* This signal's key in the reported {@link SystemInfo}, also used in logging — so it must be unique among the
|
|
34
|
+
* signals of one {@link ConcurrencySystem}, which throws on a duplicate. The four built-in names (`memInfo`,
|
|
35
|
+
* `eventLoopInfo`, `cpuInfo`, `clientInfo`) land in the correspondingly named `SystemInfo` fields rather than the
|
|
36
|
+
* `loadSignalInfo` bag; taking one over means switching that built-in off.
|
|
37
|
+
*/
|
|
20
38
|
readonly name: string;
|
|
21
39
|
/**
|
|
22
40
|
* Maximum ratio of overloaded snapshots in a sample before the signal
|
|
@@ -24,9 +42,12 @@ export interface LoadSignal {
|
|
|
24
42
|
* when more than 20% of the sample window is overloaded.
|
|
25
43
|
*/
|
|
26
44
|
readonly overloadedRatio: number;
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
|
|
45
|
+
/**
|
|
46
|
+
* Start collecting snapshots, retaining at least the sample window named in the `context`. Called when the
|
|
47
|
+
* {@link ConcurrencySystem} starts — which may be a *restart*, so drop anything measured before it.
|
|
48
|
+
*/
|
|
49
|
+
start(context: LoadSignalStartContext): Promise<void>;
|
|
50
|
+
/** Stop collecting snapshots. Called when the {@link ConcurrencySystem} shuts down. */
|
|
30
51
|
stop(): Promise<void>;
|
|
31
52
|
/**
|
|
32
53
|
* Return snapshots for a recent time window (used for "current" status).
|
|
@@ -35,13 +56,19 @@ export interface LoadSignal {
|
|
|
35
56
|
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
36
57
|
}
|
|
37
58
|
/**
|
|
38
|
-
* A time-pruning, time-windowed store for `LoadSnapshot` values.
|
|
39
|
-
*
|
|
59
|
+
* A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
|
|
60
|
+
* and so can yours — it is the only part of their machinery worth reusing.
|
|
40
61
|
*/
|
|
41
62
|
export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
|
|
42
63
|
private snapshots;
|
|
43
|
-
|
|
44
|
-
|
|
64
|
+
/** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
|
|
65
|
+
private historyMillis;
|
|
66
|
+
/**
|
|
67
|
+
* Sizes retention to the window the signal will be sampled over, as handed to it in
|
|
68
|
+
* {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
|
|
69
|
+
* its start context grows unboundedly.
|
|
70
|
+
*/
|
|
71
|
+
useSampleWindow(maxSampleWindowMillis: number): void;
|
|
45
72
|
/**
|
|
46
73
|
* Add a snapshot and prune entries older than the history window.
|
|
47
74
|
*/
|
|
@@ -51,49 +78,22 @@ export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
|
|
|
51
78
|
*/
|
|
52
79
|
getSample(sampleDurationMillis?: number): T[];
|
|
53
80
|
/**
|
|
54
|
-
* Direct access to the underlying array
|
|
81
|
+
* Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
|
|
82
|
+
* to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
|
|
55
83
|
*/
|
|
56
84
|
getAll(): T[];
|
|
57
85
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* `store.push()` to record a snapshot.
|
|
63
|
-
*/
|
|
64
|
-
static fromInterval<T extends LoadSnapshot>(options: {
|
|
65
|
-
name: string;
|
|
66
|
-
overloadedRatio: number;
|
|
67
|
-
intervalMillis: number;
|
|
68
|
-
snapshotHistoryMillis?: number;
|
|
69
|
-
handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
|
|
70
|
-
}): Omit<LoadSignal, 'getSample'> & {
|
|
71
|
-
store: SnapshotStore<T>;
|
|
72
|
-
handle: (cb: () => unknown) => void;
|
|
73
|
-
getSample(sampleDurationMillis?: number): T[];
|
|
74
|
-
};
|
|
75
|
-
/**
|
|
76
|
-
* Create a `LoadSignal` that snapshots in response to an `EventManager` event.
|
|
77
|
-
*
|
|
78
|
-
* The `handler` receives the event payload and the store. It should call
|
|
79
|
-
* `store.push()` to record a snapshot.
|
|
86
|
+
* Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
|
|
87
|
+
* samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
|
|
88
|
+
* snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
|
|
89
|
+
* rather than on stop leaves a finished session readable.
|
|
80
90
|
*/
|
|
81
|
-
|
|
82
|
-
name: string;
|
|
83
|
-
overloadedRatio: number;
|
|
84
|
-
events: EventManager;
|
|
85
|
-
event: EventTypeName;
|
|
86
|
-
snapshotHistoryMillis?: number;
|
|
87
|
-
handler: (store: SnapshotStore<T>, payload: E) => void;
|
|
88
|
-
}): Omit<LoadSignal, 'getSample'> & {
|
|
89
|
-
store: SnapshotStore<T>;
|
|
90
|
-
handle: (payload: E) => void;
|
|
91
|
-
getSample(sampleDurationMillis?: number): T[];
|
|
92
|
-
};
|
|
91
|
+
clear(): void;
|
|
93
92
|
}
|
|
94
93
|
/**
|
|
95
94
|
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
|
|
96
95
|
* overloaded ratio, using a time-weighted average. This is the shared
|
|
97
96
|
* evaluation logic used by `SystemStatus` for all signal types.
|
|
97
|
+
* @internal
|
|
98
98
|
*/
|
|
99
99
|
export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
|
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { weightedAvg } from '@crawlee/utils';
|
|
2
|
-
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
2
|
/**
|
|
4
|
-
* A time-pruning, time-windowed store for `LoadSnapshot` values.
|
|
5
|
-
*
|
|
3
|
+
* A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
|
|
4
|
+
* and so can yours — it is the only part of their machinery worth reusing.
|
|
6
5
|
*/
|
|
7
6
|
export class SnapshotStore {
|
|
8
7
|
snapshots = [];
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
/** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
|
|
9
|
+
historyMillis = Infinity;
|
|
10
|
+
/**
|
|
11
|
+
* Sizes retention to the window the signal will be sampled over, as handed to it in
|
|
12
|
+
* {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
|
|
13
|
+
* its start context grows unboundedly.
|
|
14
|
+
*/
|
|
15
|
+
useSampleWindow(maxSampleWindowMillis) {
|
|
16
|
+
this.historyMillis = maxSampleWindowMillis;
|
|
12
17
|
}
|
|
13
18
|
/**
|
|
14
19
|
* Add a snapshot and prune entries older than the history window.
|
|
@@ -50,64 +55,27 @@ export class SnapshotStore {
|
|
|
50
55
|
return sample;
|
|
51
56
|
}
|
|
52
57
|
/**
|
|
53
|
-
* Direct access to the underlying array
|
|
58
|
+
* Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
|
|
59
|
+
* to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
|
|
54
60
|
*/
|
|
55
61
|
getAll() {
|
|
56
62
|
return this.snapshots;
|
|
57
63
|
}
|
|
58
64
|
/**
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* `store.push()` to record a snapshot.
|
|
65
|
+
* Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
|
|
66
|
+
* samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
|
|
67
|
+
* snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
|
|
68
|
+
* rather than on stop leaves a finished session readable.
|
|
64
69
|
*/
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
let interval = null;
|
|
68
|
-
const handle = (cb) => options.handler(store, cb);
|
|
69
|
-
return {
|
|
70
|
-
name: options.name,
|
|
71
|
-
overloadedRatio: options.overloadedRatio,
|
|
72
|
-
store,
|
|
73
|
-
handle,
|
|
74
|
-
getSample: (ms) => store.getSample(ms),
|
|
75
|
-
async start() {
|
|
76
|
-
interval = betterSetInterval(handle, options.intervalMillis);
|
|
77
|
-
},
|
|
78
|
-
async stop() {
|
|
79
|
-
betterClearInterval(interval);
|
|
80
|
-
},
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Create a `LoadSignal` that snapshots in response to an `EventManager` event.
|
|
85
|
-
*
|
|
86
|
-
* The `handler` receives the event payload and the store. It should call
|
|
87
|
-
* `store.push()` to record a snapshot.
|
|
88
|
-
*/
|
|
89
|
-
static fromEvent(options) {
|
|
90
|
-
const store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
91
|
-
const handle = (payload) => options.handler(store, payload);
|
|
92
|
-
return {
|
|
93
|
-
name: options.name,
|
|
94
|
-
overloadedRatio: options.overloadedRatio,
|
|
95
|
-
store,
|
|
96
|
-
handle,
|
|
97
|
-
getSample: (ms) => store.getSample(ms),
|
|
98
|
-
async start() {
|
|
99
|
-
options.events.on(options.event, handle);
|
|
100
|
-
},
|
|
101
|
-
async stop() {
|
|
102
|
-
options.events.off(options.event, handle);
|
|
103
|
-
},
|
|
104
|
-
};
|
|
70
|
+
clear() {
|
|
71
|
+
this.snapshots = [];
|
|
105
72
|
}
|
|
106
73
|
}
|
|
107
74
|
/**
|
|
108
75
|
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
|
|
109
76
|
* overloaded ratio, using a time-weighted average. This is the shared
|
|
110
77
|
* evaluation logic used by `SystemStatus` for all signal types.
|
|
78
|
+
* @internal
|
|
111
79
|
*/
|
|
112
80
|
export function evaluateLoadSignalSample(sample, overloadedRatio) {
|
|
113
81
|
if (sample.length === 0) {
|
|
@@ -1,42 +1,55 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { CrawleeLogger } from '../log.js';
|
|
3
|
-
import type { LoadSignal, LoadSnapshot } from './load_signal.js';
|
|
1
|
+
import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
|
|
4
2
|
import type { SystemInfo } from './system_status.js';
|
|
3
|
+
/**
|
|
4
|
+
* A snapshot produced by the built-in memory signal.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
5
7
|
export interface MemorySnapshot extends LoadSnapshot {
|
|
6
8
|
usedBytes?: number;
|
|
7
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
|
|
12
|
+
* {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
|
|
13
|
+
*/
|
|
8
14
|
export interface MemoryLoadSignalOptions {
|
|
9
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Defines the maximum ratio of total memory that can be used.
|
|
17
|
+
* Exceeding this limit overloads the memory.
|
|
18
|
+
* @default 0.9
|
|
19
|
+
*/
|
|
20
|
+
maxUsedRatio?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
|
|
23
|
+
* @default 0.2
|
|
24
|
+
*/
|
|
10
25
|
overloadedRatio?: number;
|
|
11
|
-
snapshotHistoryMillis?: number;
|
|
12
|
-
configuration: Configuration;
|
|
13
|
-
log?: CrawleeLogger;
|
|
14
26
|
}
|
|
15
27
|
/**
|
|
16
|
-
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
|
|
17
|
-
*
|
|
28
|
+
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
|
|
29
|
+
* threshold. Also warns when memory use becomes critical.
|
|
30
|
+
*
|
|
31
|
+
* Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
|
|
32
|
+
*
|
|
33
|
+
* @category Scaling
|
|
18
34
|
*/
|
|
19
35
|
export declare class MemoryLoadSignal implements LoadSignal {
|
|
20
36
|
readonly name = "memInfo";
|
|
21
37
|
readonly overloadedRatio: number;
|
|
22
38
|
private readonly store;
|
|
23
|
-
private readonly
|
|
24
|
-
|
|
25
|
-
private
|
|
26
|
-
private
|
|
27
|
-
private maxMemoryRatio;
|
|
39
|
+
private readonly maxUsedRatio;
|
|
40
|
+
/** All resolved in `start()`, before anything that reads them can fire. */
|
|
41
|
+
private config;
|
|
42
|
+
private log;
|
|
28
43
|
private maxMemoryBytes;
|
|
44
|
+
private events?;
|
|
45
|
+
private maxMemoryRatio;
|
|
29
46
|
private lastLoggedCriticalMemoryOverloadAt;
|
|
30
|
-
constructor(options
|
|
31
|
-
start(): Promise<void>;
|
|
47
|
+
constructor(options?: MemoryLoadSignalOptions);
|
|
48
|
+
start(context: LoadSignalStartContext): Promise<void>;
|
|
32
49
|
stop(): Promise<void>;
|
|
33
|
-
getSample(sampleDurationMillis?: number):
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
*/
|
|
37
|
-
getMemorySnapshots(): MemorySnapshot[];
|
|
38
|
-
/** @internal */
|
|
39
|
-
_onSystemInfo(systemInfo: SystemInfo): void;
|
|
50
|
+
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
51
|
+
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
52
|
+
handle(systemInfo: SystemInfo): void;
|
|
40
53
|
/** @internal */
|
|
41
54
|
_memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
|
|
42
55
|
private _getTotalMemoryBytes;
|