@crawlee/core 4.0.0-beta.87 → 4.0.0-beta.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,9 @@
1
- import type { Configuration } from '../configuration.js';
2
- import type { LoadSnapshot } from './load_signal.js';
3
- import { SnapshotStore } from './load_signal.js';
1
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
4
2
  import type { SystemInfo } from './system_status.js';
3
+ /**
4
+ * A snapshot produced by the built-in CPU signal.
5
+ * @internal
6
+ */
5
7
  export interface CpuSnapshot extends LoadSnapshot {
6
8
  usedRatio: number;
7
9
  ticks?: {
@@ -9,20 +11,34 @@ export interface CpuSnapshot extends LoadSnapshot {
9
11
  total: number;
10
12
  };
11
13
  }
14
+ /**
15
+ * Tuning for the built-in **CPU** load signal, as accepted both by {@link CpuLoadSignal} and by the
16
+ * {@link LoadSignalsOptions.cpu|`cpu`} shorthand on {@link LoadSignalsOptions}.
17
+ */
12
18
  export interface CpuLoadSignalOptions {
19
+ /**
20
+ * Maximum ratio of overloaded snapshots in a sample before the CPU counts as overloaded.
21
+ * @default 0.4
22
+ */
13
23
  overloadedRatio?: number;
14
- snapshotHistoryMillis?: number;
15
- configuration: Configuration;
16
24
  }
17
25
  /**
18
- * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
19
- * the platform or local OS metrics indicate the CPU is overloaded.
26
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
27
+ * CPU is overloaded.
28
+ *
29
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
30
+ *
31
+ * @category Scaling
20
32
  */
21
- // @ts-ignore optional peer dependency or compatibility with es2022
22
- export declare function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
23
- store: SnapshotStore<CpuSnapshot>;
24
- handle: (payload: SystemInfo) => void;
25
- getSample(sampleDurationMillis?: number): CpuSnapshot[];
26
- };
27
- /** @internal Return type for backward compat in Snapshotter facade */
28
- export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;
33
+ export declare class CpuLoadSignal implements LoadSignal {
34
+ readonly name = "cpuInfo";
35
+ readonly overloadedRatio: number;
36
+ private readonly store;
37
+ private events?;
38
+ constructor(options?: CpuLoadSignalOptions);
39
+ start(context: LoadSignalStartContext): Promise<void>;
40
+ stop(): Promise<void>;
41
+ getSample(sampleDurationMillis?: number): LoadSnapshot[];
42
+ /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
43
+ handle(systemInfo: SystemInfo): void;
44
+ }
@@ -1,24 +1,46 @@
1
1
  import { serviceLocator } from '../service_locator.js';
2
2
  import { SnapshotStore } from './load_signal.js';
3
3
  /**
4
- * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
5
- * the platform or local OS metrics indicate the CPU is overloaded.
4
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
5
+ * CPU is overloaded.
6
+ *
7
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
8
+ *
9
+ * @category Scaling
6
10
  */
7
- export function createCpuLoadSignal(options) {
8
- return SnapshotStore.fromEvent({
9
- name: 'cpuInfo',
10
- overloadedRatio: options.overloadedRatio ?? 0.4,
11
- events: serviceLocator.getEventManager(),
12
- event: "systemInfo" /* EventType.SYSTEM_INFO */,
13
- snapshotHistoryMillis: options.snapshotHistoryMillis,
14
- handler(store, systemInfo) {
15
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
16
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
17
- store.push({
18
- createdAt,
19
- isOverloaded: isCpuOverloaded,
20
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
21
- }, createdAt);
22
- },
23
- });
11
+ export class CpuLoadSignal {
12
+ name = 'cpuInfo';
13
+ overloadedRatio;
14
+ store = new SnapshotStore();
15
+ events;
16
+ constructor(options = {}) {
17
+ this.overloadedRatio = options.overloadedRatio ?? 0.4;
18
+ this.handle = this.handle.bind(this);
19
+ }
20
+ async start(context) {
21
+ this.store.useSampleWindow(context.maxSampleWindowMillis);
22
+ // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
23
+ this.store.clear();
24
+ // Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
25
+ // between systems) cannot capture whichever event manager happened to be registered at that moment.
26
+ this.events = serviceLocator.getEventManager();
27
+ this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
28
+ }
29
+ async stop() {
30
+ this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
31
+ this.events = undefined;
32
+ }
33
+ getSample(sampleDurationMillis) {
34
+ return this.store.getSample(sampleDurationMillis);
35
+ }
36
+ /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
37
+ handle(systemInfo) {
38
+ const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
39
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
40
+ this.store.push({
41
+ createdAt,
42
+ isOverloaded: isCpuOverloaded,
43
+ usedRatio: Math.ceil(cpuCurrentUsage / 100),
44
+ }, createdAt);
45
+ }
24
46
  }
@@ -1,23 +1,54 @@
1
- import type { LoadSnapshot } from './load_signal.js';
2
- import { SnapshotStore } from './load_signal.js';
1
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
+ /**
3
+ * A snapshot produced by the built-in event loop signal.
4
+ * @internal
5
+ */
3
6
  export interface EventLoopSnapshot extends LoadSnapshot {
4
7
  exceededMillis: number;
5
8
  }
9
+ /**
10
+ * Tuning for the built-in **event loop** load signal, as accepted both by {@link EventLoopLoadSignal} and by the
11
+ * {@link LoadSignalsOptions.eventLoop|`eventLoop`} shorthand on {@link LoadSignalsOptions}.
12
+ */
6
13
  export interface EventLoopLoadSignalOptions {
7
- eventLoopSnapshotIntervalSecs?: number;
14
+ /**
15
+ * Defines the interval of measuring the event loop response time, in seconds.
16
+ * @default 0.5
17
+ */
18
+ snapshotIntervalSecs?: number;
19
+ /**
20
+ * Maximum allowed delay of the event loop in milliseconds.
21
+ * Exceeding this limit overloads the event loop.
22
+ * @default 50
23
+ */
8
24
  maxBlockedMillis?: number;
25
+ /**
26
+ * Maximum ratio of overloaded snapshots in a sample before the event loop counts as overloaded.
27
+ * @default 0.6
28
+ */
9
29
  overloadedRatio?: number;
10
- snapshotHistoryMillis?: number;
11
30
  }
12
31
  /**
13
- * Periodically measures event loop delay and reports overload when the
14
- * delay exceeds a configured threshold.
32
+ * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
33
+ *
34
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
35
+ *
36
+ * @category Scaling
15
37
  */
16
- // @ts-ignore optional peer dependency or compatibility with es2022
17
- export declare function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
18
- store: SnapshotStore<EventLoopSnapshot>;
19
- handle: (cb: () => unknown) => void;
20
- getSample(sampleDurationMillis?: number): EventLoopSnapshot[];
21
- };
22
- /** @internal Return type for backward compat in Snapshotter facade */
23
- export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;
38
+ export declare class EventLoopLoadSignal implements LoadSignal {
39
+ readonly name = "eventLoopInfo";
40
+ readonly overloadedRatio: number;
41
+ private readonly store;
42
+ private readonly intervalMillis;
43
+ private readonly maxBlockedMillis;
44
+ private interval?;
45
+ constructor(options?: EventLoopLoadSignalOptions);
46
+ start(context: LoadSignalStartContext): Promise<void>;
47
+ stop(): Promise<void>;
48
+ getSample(sampleDurationMillis?: number): LoadSnapshot[];
49
+ /**
50
+ * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
51
+ * @internal Also lets tests drive the measurement without waiting on a timer.
52
+ */
53
+ handle(intervalCallback: () => unknown): void;
54
+ }
@@ -1,35 +1,60 @@
1
+ import { betterClearInterval, betterSetInterval } from '@apify/utilities';
1
2
  import { SnapshotStore } from './load_signal.js';
2
3
  /**
3
- * Periodically measures event loop delay and reports overload when the
4
- * delay exceeds a configured threshold.
4
+ * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
5
+ *
6
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
7
+ *
8
+ * @category Scaling
5
9
  */
6
- export function createEventLoopLoadSignal(options = {}) {
7
- const intervalMillis = (options.eventLoopSnapshotIntervalSecs ?? 0.5) * 1000;
8
- const maxBlockedMillis = options.maxBlockedMillis ?? 50;
9
- const signal = SnapshotStore.fromInterval({
10
- name: 'eventLoopInfo',
11
- overloadedRatio: options.overloadedRatio ?? 0.6,
12
- intervalMillis,
13
- snapshotHistoryMillis: options.snapshotHistoryMillis,
14
- handler(store, intervalCallback) {
15
- const now = new Date();
16
- const snapshot = {
17
- createdAt: now,
18
- isOverloaded: false,
19
- exceededMillis: 0,
20
- };
21
- const all = store.getAll();
22
- const previousSnapshot = all[all.length - 1];
23
- if (previousSnapshot) {
24
- const { createdAt } = previousSnapshot;
25
- const delta = now.getTime() - +createdAt - intervalMillis;
26
- if (delta > maxBlockedMillis)
27
- snapshot.isOverloaded = true;
28
- snapshot.exceededMillis = Math.max(delta - maxBlockedMillis, 0);
29
- }
30
- store.push(snapshot, now);
31
- intervalCallback();
32
- },
33
- });
34
- return signal;
10
+ export class EventLoopLoadSignal {
11
+ name = 'eventLoopInfo';
12
+ overloadedRatio;
13
+ store = new SnapshotStore();
14
+ intervalMillis;
15
+ maxBlockedMillis;
16
+ interval;
17
+ constructor(options = {}) {
18
+ this.overloadedRatio = options.overloadedRatio ?? 0.6;
19
+ this.intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
20
+ this.maxBlockedMillis = options.maxBlockedMillis ?? 50;
21
+ this.handle = this.handle.bind(this);
22
+ }
23
+ async start(context) {
24
+ this.store.useSampleWindow(context.maxSampleWindowMillis);
25
+ // A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
26
+ // the gap since the previous snapshot, which across a restart is however long the system was stopped.
27
+ this.store.clear();
28
+ this.interval = betterSetInterval(this.handle, this.intervalMillis);
29
+ }
30
+ async stop() {
31
+ if (this.interval)
32
+ betterClearInterval(this.interval);
33
+ this.interval = undefined;
34
+ }
35
+ getSample(sampleDurationMillis) {
36
+ return this.store.getSample(sampleDurationMillis);
37
+ }
38
+ /**
39
+ * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
40
+ * @internal Also lets tests drive the measurement without waiting on a timer.
41
+ */
42
+ handle(intervalCallback) {
43
+ const now = new Date();
44
+ const snapshot = {
45
+ createdAt: now,
46
+ isOverloaded: false,
47
+ exceededMillis: 0,
48
+ };
49
+ const all = this.store.getAll();
50
+ const previousSnapshot = all[all.length - 1];
51
+ if (previousSnapshot) {
52
+ const delta = now.getTime() - +previousSnapshot.createdAt - this.intervalMillis;
53
+ if (delta > this.maxBlockedMillis)
54
+ snapshot.isOverloaded = true;
55
+ snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
56
+ }
57
+ this.store.push(snapshot, now);
58
+ intervalCallback();
59
+ }
35
60
  }
@@ -1,4 +1,5 @@
1
1
  export * from './autoscaled_pool.js';
2
+ export * from './concurrency_system.js';
2
3
  export * from './client_load_signal.js';
3
4
  export * from './cpu_load_signal.js';
4
5
  export * from './event_loop_load_signal.js';
@@ -1,4 +1,5 @@
1
1
  export * from './autoscaled_pool.js';
2
+ export * from './concurrency_system.js';
2
3
  export * from './client_load_signal.js';
3
4
  export * from './cpu_load_signal.js';
4
5
  export * from './event_loop_load_signal.js';
@@ -1,4 +1,3 @@
1
- import type { EventManager, EventTypeName } from '../events/event_manager.js';
2
1
  import type { ClientInfo } from './system_status.js';
3
2
  /**
4
3
  * A snapshot of a resource's overload state at a point in time.
@@ -8,15 +7,34 @@ export interface LoadSnapshot {
8
7
  isOverloaded: boolean;
9
8
  }
10
9
  /**
11
- * A signal that reports whether a particular resource is overloaded.
10
+ * Handed to a {@link LoadSignal} when it starts, so it can size its snapshot retention to what it will actually
11
+ * be asked for — without having to know how the {@link ConcurrencySystem} that drives it is configured.
12
+ */
13
+ export interface LoadSignalStartContext {
14
+ /**
15
+ * The longest sample window the signal will be queried with (the wider of the task-gating and autoscaling
16
+ * windows). Keeping less history than this contributes a narrower view of the resource than the other signals;
17
+ * keeping more is wasted memory, as the extra snapshots are never sampled.
18
+ */
19
+ maxSampleWindowMillis: number;
20
+ }
21
+ /**
22
+ * A signal that reports whether a particular resource is overloaded. The {@link ConcurrencySystem} aggregates
23
+ * several of them — if any one reports overload, the system is overloaded.
12
24
  *
13
- * `SystemStatus` aggregates multiple `LoadSignal` instances to determine
14
- * overall system health. The built-in signals cover memory, CPU, event loop,
15
- * and API client rate limits. You can implement this interface to add
16
- * custom overload signals (e.g. navigation timeouts, proxy health).
25
+ * The built-in signals cover memory, CPU, event loop and storage-client rate limits. Implement this interface to add
26
+ * your own (navigation timeouts, proxy health, …) and pass them via
27
+ * {@link LoadSignalsOptions.custom|`loadSignals.custom`}; {@link SnapshotStore} does the time-windowed
28
+ * bookkeeping if you want it. Each built-in is also a public class, so one can be *wrapped* rather than reimplemented
29
+ * — construct it yourself and switch the default off with {@link LoadSignalsOptions.cpu|`cpu: false`} or friends.
17
30
  */
18
31
  export interface LoadSignal {
19
- /** Human-readable name used in logging and `SystemInfo` keys. */
32
+ /**
33
+ * This signal's key in the reported {@link SystemInfo}, also used in logging — so it must be unique among the
34
+ * signals of one {@link ConcurrencySystem}, which throws on a duplicate. The four built-in names (`memInfo`,
35
+ * `eventLoopInfo`, `cpuInfo`, `clientInfo`) land in the correspondingly named `SystemInfo` fields rather than the
36
+ * `loadSignalInfo` bag; taking one over means switching that built-in off.
37
+ */
20
38
  readonly name: string;
21
39
  /**
22
40
  * Maximum ratio of overloaded snapshots in a sample before the signal
@@ -24,9 +42,12 @@ export interface LoadSignal {
24
42
  * when more than 20% of the sample window is overloaded.
25
43
  */
26
44
  readonly overloadedRatio: number;
27
- /** Start collecting snapshots. Called when the pool starts. */
28
- start(): Promise<void>;
29
- /** Stop collecting snapshots. Called when the pool shuts down. */
45
+ /**
46
+ * Start collecting snapshots, retaining at least the sample window named in the `context`. Called when the
47
+ * {@link ConcurrencySystem} starts which may be a *restart*, so drop anything measured before it.
48
+ */
49
+ start(context: LoadSignalStartContext): Promise<void>;
50
+ /** Stop collecting snapshots. Called when the {@link ConcurrencySystem} shuts down. */
30
51
  stop(): Promise<void>;
31
52
  /**
32
53
  * Return snapshots for a recent time window (used for "current" status).
@@ -35,13 +56,19 @@ export interface LoadSignal {
35
56
  getSample(sampleDurationMillis?: number): LoadSnapshot[];
36
57
  }
37
58
  /**
38
- * A time-pruning, time-windowed store for `LoadSnapshot` values.
39
- * Signals compose with this instead of inheriting from a base class.
59
+ * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
60
+ * and so can yours it is the only part of their machinery worth reusing.
40
61
  */
41
62
  export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
42
63
  private snapshots;
43
- private readonly historyMillis;
44
- constructor(historyMillis?: number);
64
+ /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
65
+ private historyMillis;
66
+ /**
67
+ * Sizes retention to the window the signal will be sampled over, as handed to it in
68
+ * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
69
+ * its start context grows unboundedly.
70
+ */
71
+ useSampleWindow(maxSampleWindowMillis: number): void;
45
72
  /**
46
73
  * Add a snapshot and prune entries older than the history window.
47
74
  */
@@ -51,49 +78,22 @@ export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
51
78
  */
52
79
  getSample(sampleDurationMillis?: number): T[];
53
80
  /**
54
- * Direct access to the underlying array (for backward-compat getters).
81
+ * Direct, unwindowed access to the underlying array used by signals whose handler needs the previous snapshot
82
+ * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
55
83
  */
56
84
  getAll(): T[];
57
85
  /**
58
- * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
59
- *
60
- * The `handler` receives the store (to read previous snapshots) and the
61
- * interval callback (which it **must** call when done). It should call
62
- * `store.push()` to record a snapshot.
63
- */
64
- static fromInterval<T extends LoadSnapshot>(options: {
65
- name: string;
66
- overloadedRatio: number;
67
- intervalMillis: number;
68
- snapshotHistoryMillis?: number;
69
- handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
70
- }): Omit<LoadSignal, 'getSample'> & {
71
- store: SnapshotStore<T>;
72
- handle: (cb: () => unknown) => void;
73
- getSample(sampleDurationMillis?: number): T[];
74
- };
75
- /**
76
- * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
77
- *
78
- * The `handler` receives the event payload and the store. It should call
79
- * `store.push()` to record a snapshot.
86
+ * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
87
+ * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
88
+ * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
89
+ * rather than on stop leaves a finished session readable.
80
90
  */
81
- static fromEvent<T extends LoadSnapshot, E>(options: {
82
- name: string;
83
- overloadedRatio: number;
84
- events: EventManager;
85
- event: EventTypeName;
86
- snapshotHistoryMillis?: number;
87
- handler: (store: SnapshotStore<T>, payload: E) => void;
88
- }): Omit<LoadSignal, 'getSample'> & {
89
- store: SnapshotStore<T>;
90
- handle: (payload: E) => void;
91
- getSample(sampleDurationMillis?: number): T[];
92
- };
91
+ clear(): void;
93
92
  }
94
93
  /**
95
94
  * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
96
95
  * overloaded ratio, using a time-weighted average. This is the shared
97
96
  * evaluation logic used by `SystemStatus` for all signal types.
97
+ * @internal
98
98
  */
99
99
  export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;
@@ -1,14 +1,19 @@
1
1
  import { weightedAvg } from '@crawlee/utils';
2
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
3
2
  /**
4
- * A time-pruning, time-windowed store for `LoadSnapshot` values.
5
- * Signals compose with this instead of inheriting from a base class.
3
+ * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
4
+ * and so can yours it is the only part of their machinery worth reusing.
6
5
  */
7
6
  export class SnapshotStore {
8
7
  snapshots = [];
9
- historyMillis;
10
- constructor(historyMillis = 30_000) {
11
- this.historyMillis = historyMillis;
8
+ /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
9
+ historyMillis = Infinity;
10
+ /**
11
+ * Sizes retention to the window the signal will be sampled over, as handed to it in
12
+ * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
13
+ * its start context grows unboundedly.
14
+ */
15
+ useSampleWindow(maxSampleWindowMillis) {
16
+ this.historyMillis = maxSampleWindowMillis;
12
17
  }
13
18
  /**
14
19
  * Add a snapshot and prune entries older than the history window.
@@ -50,64 +55,27 @@ export class SnapshotStore {
50
55
  return sample;
51
56
  }
52
57
  /**
53
- * Direct access to the underlying array (for backward-compat getters).
58
+ * Direct, unwindowed access to the underlying array used by signals whose handler needs the previous snapshot
59
+ * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
54
60
  */
55
61
  getAll() {
56
62
  return this.snapshots;
57
63
  }
58
64
  /**
59
- * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
60
- *
61
- * The `handler` receives the store (to read previous snapshots) and the
62
- * interval callback (which it **must** call when done). It should call
63
- * `store.push()` to record a snapshot.
65
+ * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
66
+ * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
67
+ * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
68
+ * rather than on stop leaves a finished session readable.
64
69
  */
65
- static fromInterval(options) {
66
- const store = new SnapshotStore(options.snapshotHistoryMillis);
67
- let interval = null;
68
- const handle = (cb) => options.handler(store, cb);
69
- return {
70
- name: options.name,
71
- overloadedRatio: options.overloadedRatio,
72
- store,
73
- handle,
74
- getSample: (ms) => store.getSample(ms),
75
- async start() {
76
- interval = betterSetInterval(handle, options.intervalMillis);
77
- },
78
- async stop() {
79
- betterClearInterval(interval);
80
- },
81
- };
82
- }
83
- /**
84
- * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
85
- *
86
- * The `handler` receives the event payload and the store. It should call
87
- * `store.push()` to record a snapshot.
88
- */
89
- static fromEvent(options) {
90
- const store = new SnapshotStore(options.snapshotHistoryMillis);
91
- const handle = (payload) => options.handler(store, payload);
92
- return {
93
- name: options.name,
94
- overloadedRatio: options.overloadedRatio,
95
- store,
96
- handle,
97
- getSample: (ms) => store.getSample(ms),
98
- async start() {
99
- options.events.on(options.event, handle);
100
- },
101
- async stop() {
102
- options.events.off(options.event, handle);
103
- },
104
- };
70
+ clear() {
71
+ this.snapshots = [];
105
72
  }
106
73
  }
107
74
  /**
108
75
  * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
109
76
  * overloaded ratio, using a time-weighted average. This is the shared
110
77
  * evaluation logic used by `SystemStatus` for all signal types.
78
+ * @internal
111
79
  */
112
80
  export function evaluateLoadSignalSample(sample, overloadedRatio) {
113
81
  if (sample.length === 0) {
@@ -1,42 +1,55 @@
1
- import type { Configuration } from '../configuration.js';
2
- import type { CrawleeLogger } from '../log.js';
3
- import type { LoadSignal, LoadSnapshot } from './load_signal.js';
1
+ import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
4
2
  import type { SystemInfo } from './system_status.js';
3
+ /**
4
+ * A snapshot produced by the built-in memory signal.
5
+ * @internal
6
+ */
5
7
  export interface MemorySnapshot extends LoadSnapshot {
6
8
  usedBytes?: number;
7
9
  }
10
+ /**
11
+ * Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
12
+ * {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
13
+ */
8
14
  export interface MemoryLoadSignalOptions {
9
- maxUsedMemoryRatio?: number;
15
+ /**
16
+ * Defines the maximum ratio of total memory that can be used.
17
+ * Exceeding this limit overloads the memory.
18
+ * @default 0.9
19
+ */
20
+ maxUsedRatio?: number;
21
+ /**
22
+ * Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
23
+ * @default 0.2
24
+ */
10
25
  overloadedRatio?: number;
11
- snapshotHistoryMillis?: number;
12
- configuration: Configuration;
13
- log?: CrawleeLogger;
14
26
  }
15
27
  /**
16
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
17
- * the used-to-available memory ratio exceeds a threshold.
28
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
29
+ * threshold. Also warns when memory use becomes critical.
30
+ *
31
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
32
+ *
33
+ * @category Scaling
18
34
  */
19
35
  export declare class MemoryLoadSignal implements LoadSignal {
20
36
  readonly name = "memInfo";
21
37
  readonly overloadedRatio: number;
22
38
  private readonly store;
23
- private readonly configuration;
24
- private readonly events;
25
- private readonly log;
26
- private readonly maxUsedMemoryRatio;
27
- private maxMemoryRatio;
39
+ private readonly maxUsedRatio;
40
+ /** All resolved in `start()`, before anything that reads them can fire. */
41
+ private config;
42
+ private log;
28
43
  private maxMemoryBytes;
44
+ private events?;
45
+ private maxMemoryRatio;
29
46
  private lastLoggedCriticalMemoryOverloadAt;
30
- constructor(options: MemoryLoadSignalOptions);
31
- start(): Promise<void>;
47
+ constructor(options?: MemoryLoadSignalOptions);
48
+ start(context: LoadSignalStartContext): Promise<void>;
32
49
  stop(): Promise<void>;
33
- getSample(sampleDurationMillis?: number): MemorySnapshot[];
34
- /**
35
- * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
36
- */
37
- getMemorySnapshots(): MemorySnapshot[];
38
- /** @internal */
39
- _onSystemInfo(systemInfo: SystemInfo): void;
50
+ getSample(sampleDurationMillis?: number): LoadSnapshot[];
51
+ /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
52
+ handle(systemInfo: SystemInfo): void;
40
53
  /** @internal */
41
54
  _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
42
55
  private _getTotalMemoryBytes;