@crawlee/core 4.0.0-beta.87 → 4.0.0-beta.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,36 +5,45 @@ import { SnapshotStore } from './load_signal.js';
5
5
  const RESERVE_MEMORY_RATIO = 0.5;
6
6
  const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
7
7
  /**
8
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
9
- * the used-to-available memory ratio exceeds a threshold.
8
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
9
+ * threshold. Also warns when memory use becomes critical.
10
+ *
11
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
12
+ *
13
+ * @category Scaling
10
14
  */
11
15
  export class MemoryLoadSignal {
12
16
  name = 'memInfo';
13
17
  overloadedRatio;
14
- store;
15
- configuration;
16
- events;
18
+ store = new SnapshotStore();
19
+ maxUsedRatio;
20
+ /** All resolved in `start()`, before anything that reads them can fire. */
21
+ config;
17
22
  log;
18
- maxUsedMemoryRatio;
19
- maxMemoryRatio;
20
23
  maxMemoryBytes;
24
+ events;
25
+ maxMemoryRatio;
21
26
  lastLoggedCriticalMemoryOverloadAt = null;
22
- constructor(options) {
23
- this.store = new SnapshotStore(options.snapshotHistoryMillis);
24
- this.configuration = options.configuration;
25
- this.events = serviceLocator.getEventManager();
26
- this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
27
- this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
27
+ constructor(options = {}) {
28
+ this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
28
29
  this.overloadedRatio = options.overloadedRatio ?? 0.2;
29
- this._onSystemInfo = this._onSystemInfo.bind(this);
30
+ this.handle = this.handle.bind(this);
30
31
  }
31
- async start() {
32
- const memoryMbytes = this.configuration.memoryMbytes ?? 0;
32
+ async start(context) {
33
+ this.store.useSampleWindow(context.maxSampleWindowMillis);
34
+ // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
35
+ this.store.clear();
36
+ // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
37
+ // between systems) must not capture whichever services happened to be registered at that moment.
38
+ this.config = serviceLocator.getConfiguration();
39
+ this.events = serviceLocator.getEventManager();
40
+ this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
41
+ const memoryMbytes = this.config.memoryMbytes ?? 0;
33
42
  if (memoryMbytes > 0) {
34
43
  this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
35
44
  }
36
45
  else {
37
- this.maxMemoryRatio = this.configuration.availableMemoryRatio;
46
+ this.maxMemoryRatio = this.config.availableMemoryRatio;
38
47
  if (!this.maxMemoryRatio) {
39
48
  throw new Error('availableMemoryRatio is not set in configuration.');
40
49
  }
@@ -45,22 +54,17 @@ export class MemoryLoadSignal {
45
54
  // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
46
55
  this.maxMemoryBytes = await this._getTotalMemoryBytes();
47
56
  }
48
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
57
+ this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
49
58
  }
50
59
  async stop() {
51
- this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
60
+ this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
61
+ this.events = undefined;
52
62
  }
53
63
  getSample(sampleDurationMillis) {
54
64
  return this.store.getSample(sampleDurationMillis);
55
65
  }
56
- /**
57
- * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
58
- */
59
- getMemorySnapshots() {
60
- return this.store.getAll();
61
- }
62
- /** @internal */
63
- _onSystemInfo(systemInfo) {
66
+ /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
67
+ handle(systemInfo) {
64
68
  const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
65
69
  const { memCurrentBytes, memTotalBytes } = systemInfo;
66
70
  let maxMemoryBytes = this.maxMemoryBytes;
@@ -69,7 +73,7 @@ export class MemoryLoadSignal {
69
73
  }
70
74
  const snapshot = {
71
75
  createdAt,
72
- isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
76
+ isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
73
77
  usedBytes: memCurrentBytes,
74
78
  };
75
79
  this.store.push(snapshot, createdAt);
@@ -83,8 +87,8 @@ export class MemoryLoadSignal {
83
87
  if (this.lastLoggedCriticalMemoryOverloadAt &&
84
88
  +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
85
89
  return;
86
- const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
87
- const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
90
+ const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
91
+ const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
88
92
  const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
89
93
  const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
90
94
  if (isCriticalOverload) {
@@ -96,7 +100,7 @@ export class MemoryLoadSignal {
96
100
  }
97
101
  }
98
102
  async _getTotalMemoryBytes() {
99
- const containerized = this.configuration.containerized ?? (await isContainerized());
103
+ const containerized = this.config.containerized ?? (await isContainerized());
100
104
  return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
101
105
  }
102
106
  }
@@ -1,127 +1,87 @@
1
- import type { StorageBackend } from '@crawlee/types';
2
- import type { Configuration } from '../configuration.js';
3
- import type { CrawleeLogger } from '../log.js';
4
- import type { ClientSnapshot } from './client_load_signal.js';
5
- import type { CpuSnapshot } from './cpu_load_signal.js';
6
- import type { EventLoopSnapshot } from './event_loop_load_signal.js';
7
- import type { LoadSignal } from './load_signal.js';
8
- import type { MemorySnapshot } from './memory_load_signal.js';
9
- export interface SnapshotterOptions {
10
- /**
11
- * Defines the interval of measuring the event loop response time.
12
- * @default 0.5
13
- */
14
- eventLoopSnapshotIntervalSecs?: number;
1
+ import type { ClientLoadSignalOptions } from './client_load_signal.js';
2
+ import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
3
+ import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
4
+ import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
5
+ import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
6
+ /**
7
+ * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
8
+ *
9
+ * Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
10
+ * corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
11
+ * `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
12
+ * unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
13
+ *
14
+ * How far back the signals are evaluated is *not* set here, but by
15
+ * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
16
+ * {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
17
+ */
18
+ export interface LoadSignalsOptions {
15
19
  /**
16
- * Defines the interval of checking the current state
17
- * of the remote API client.
18
- * @default 1
20
+ * Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
21
+ * it off.
19
22
  */
20
- clientSnapshotIntervalSecs?: number;
23
+ memory?: MemoryLoadSignalOptions | false;
21
24
  /**
22
- * Maximum allowed delay of the event loop in milliseconds.
23
- * Exceeding this limit overloads the event loop.
24
- * @default 50
25
+ * Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
26
+ * ratio), or `false` to switch it off — which also stops its measuring interval.
25
27
  */
26
- maxBlockedMillis?: number;
28
+ eventLoop?: EventLoopLoadSignalOptions | false;
27
29
  /**
28
- * Defines the maximum ratio of total memory that can be used.
29
- * Exceeding this limit overloads the memory.
30
- * @default 0.9
30
+ * Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
31
31
  */
32
- maxUsedMemoryRatio?: number;
32
+ cpu?: CpuLoadSignalOptions | false;
33
33
  /**
34
- * Defines the maximum number of new rate limit errors within
35
- * the given interval.
36
- * @default 3
34
+ * Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
35
+ * `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
36
+ * signal otherwise polls it every second to no purpose.
37
37
  */
38
- maxClientErrors?: number;
38
+ client?: ClientLoadSignalOptions | false;
39
39
  /**
40
- * Sets the interval in seconds for which a history of resource snapshots
41
- * will be kept. Increasing this to very high numbers will affect performance.
42
- * @default 30
40
+ * Additional {@link LoadSignal} implementations e.g. navigation timeouts or proxy health — evaluated
41
+ * alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
42
+ * is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
43
+ * not collide with an enabled built-in's.
43
44
  */
44
- snapshotHistorySecs?: number;
45
- /** @internal */
46
- log?: CrawleeLogger;
47
- /** @internal */
48
- client?: StorageBackend;
49
- /** @internal */
50
- configuration?: Configuration;
45
+ custom?: LoadSignal[];
51
46
  }
52
47
  /**
53
- * Creates snapshots of system resources at given intervals and marks the resource
54
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
55
- * It tracks the following resources: Memory, EventLoop, API and CPU.
56
- * The class is used by the {@link AutoscaledPool} class.
57
- *
58
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
59
- * as collected from the running Docker container. When running locally, `Snapshotter`
60
- * makes its own statistics by querying the OS.
61
- *
62
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
63
- * when Apify platform marks it as overloaded.
64
- *
65
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
66
- * It's computed using the total memory available to the container when running on
67
- * the Apify platform and a quarter of total system memory when running locally.
68
- * Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
69
- * environment variable.
70
- *
71
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
72
- *
73
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
74
- * typically received from the request queue, exceed the set limit within the set interval.
48
+ * An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
49
+ * {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
50
+ * them here.
51
+ * @internal
52
+ */
53
+ export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
54
+ /**
55
+ * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
56
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
57
+ * that were not switched off and driving their shared lifecycle.
75
58
  *
76
- * @category Scaling
59
+ * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
60
+ * are simply forwarded to the corresponding constructor.
61
+ * @internal
77
62
  */
78
63
  export declare class Snapshotter {
79
- readonly log: CrawleeLogger;
80
- readonly client: StorageBackend;
81
- readonly configuration: Configuration;
82
- private readonly memorySignal;
83
- private readonly eventLoopSignal;
84
- private readonly cpuSignal;
85
- private readonly clientSignal;
64
+ private readonly memorySignal?;
65
+ private readonly eventLoopSignal?;
66
+ private readonly cpuSignal?;
67
+ private readonly clientSignal?;
86
68
  /**
87
- * Returns the four built-in signals as an array, so `SystemStatus` can
88
- * iterate them alongside any custom `LoadSignal` instances.
69
+ * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
70
+ * instances. Signals switched off through the options are simply absent — the system status reports them as
71
+ * not overloaded.
89
72
  */
90
73
  getLoadSignals(): LoadSignal[];
91
- get cpuSnapshots(): CpuSnapshot[];
92
- get eventLoopSnapshots(): EventLoopSnapshot[];
93
- get memorySnapshots(): MemorySnapshot[];
94
- get clientSnapshots(): ClientSnapshot[];
95
74
  /**
96
75
  * @param [options] All `Snapshotter` configuration options.
97
76
  */
98
77
  constructor(options?: SnapshotterOptions);
99
78
  /**
100
- * Starts capturing snapshots at configured intervals.
79
+ * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
80
+ * be queried with, which is also how much history they retain.
101
81
  */
102
- start(): Promise<void>;
82
+ start(context: LoadSignalStartContext): Promise<void>;
103
83
  /**
104
84
  * Stops all resource capturing.
105
85
  */
106
86
  stop(): Promise<void>;
107
- /**
108
- * Returns a sample of latest memory snapshots, with the size of the sample defined
109
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
110
- */
111
- getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];
112
- /**
113
- * Returns a sample of latest event loop snapshots, with the size of the sample defined
114
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
115
- */
116
- getEventLoopSample(sampleDurationMillis?: number): EventLoopSnapshot[];
117
- /**
118
- * Returns a sample of latest CPU snapshots, with the size of the sample defined
119
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
120
- */
121
- getCpuSample(sampleDurationMillis?: number): CpuSnapshot[];
122
- /**
123
- * Returns a sample of latest Client snapshots, with the size of the sample defined
124
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
125
- */
126
- getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
127
87
  }
@@ -1,153 +1,67 @@
1
- import ow from 'ow';
2
- import { serviceLocator } from '../service_locator.js';
3
- import { createClientLoadSignal } from './client_load_signal.js';
4
- import { createCpuLoadSignal } from './cpu_load_signal.js';
5
- import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
1
+ import { ClientLoadSignal } from './client_load_signal.js';
2
+ import { CpuLoadSignal } from './cpu_load_signal.js';
3
+ import { EventLoopLoadSignal } from './event_loop_load_signal.js';
6
4
  import { MemoryLoadSignal } from './memory_load_signal.js';
7
5
  /**
8
- * Creates snapshots of system resources at given intervals and marks the resource
9
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
10
- * It tracks the following resources: Memory, EventLoop, API and CPU.
11
- * The class is used by the {@link AutoscaledPool} class.
6
+ * Owns the four built-in {@link LoadSignal} instances {@link MemoryLoadSignal},
7
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} constructing the ones
8
+ * that were not switched off and driving their shared lifecycle.
12
9
  *
13
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
14
- * as collected from the running Docker container. When running locally, `Snapshotter`
15
- * makes its own statistics by querying the OS.
16
- *
17
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
18
- * when Apify platform marks it as overloaded.
19
- *
20
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
21
- * It's computed using the total memory available to the container when running on
22
- * the Apify platform and a quarter of total system memory when running locally.
23
- * Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
24
- * environment variable.
25
- *
26
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
27
- *
28
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
29
- * typically received from the request queue, exceed the set limit within the set interval.
30
- *
31
- * @category Scaling
10
+ * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
11
+ * are simply forwarded to the corresponding constructor.
12
+ * @internal
32
13
  */
33
14
  export class Snapshotter {
34
- log;
35
- client;
36
- configuration;
15
+ // Absent when switched off through the corresponding option (e.g. `client: false`).
37
16
  memorySignal;
38
17
  eventLoopSignal;
39
18
  cpuSignal;
40
19
  clientSignal;
41
20
  /**
42
- * Returns the four built-in signals as an array, so `SystemStatus` can
43
- * iterate them alongside any custom `LoadSignal` instances.
21
+ * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
22
+ * instances. Signals switched off through the options are simply absent — the system status reports them as
23
+ * not overloaded.
44
24
  */
45
25
  getLoadSignals() {
46
- return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
47
- }
48
- // Legacy public properties kept for backward compat (tests read these directly)
49
- get cpuSnapshots() {
50
- return this.cpuSignal.store.getAll();
51
- }
52
- get eventLoopSnapshots() {
53
- return this.eventLoopSignal.store.getAll();
54
- }
55
- get memorySnapshots() {
56
- return this.memorySignal.getMemorySnapshots();
57
- }
58
- get clientSnapshots() {
59
- return this.clientSignal.store.getAll();
26
+ const builtin = [
27
+ this.memorySignal,
28
+ this.eventLoopSignal,
29
+ this.cpuSignal,
30
+ this.clientSignal,
31
+ ];
32
+ return builtin.filter((signal) => signal !== undefined);
60
33
  }
61
34
  /**
62
35
  * @param [options] All `Snapshotter` configuration options.
63
36
  */
64
37
  constructor(options = {}) {
65
- ow(options, ow.object.exactShape({
66
- eventLoopSnapshotIntervalSecs: ow.optional.number,
67
- clientSnapshotIntervalSecs: ow.optional.number,
68
- snapshotHistorySecs: ow.optional.number,
69
- maxBlockedMillis: ow.optional.number,
70
- maxUsedMemoryRatio: ow.optional.number,
71
- maxClientErrors: ow.optional.number,
72
- log: ow.optional.object,
73
- client: ow.optional.object,
74
- configuration: ow.optional.object,
75
- }));
76
- const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), configuration = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
77
- this.log = log.child({ prefix: 'Snapshotter' });
78
- this.client = client;
79
- this.configuration = configuration;
80
- const snapshotHistoryMillis = snapshotHistorySecs * 1000;
81
- this.memorySignal = new MemoryLoadSignal({
82
- maxUsedMemoryRatio,
83
- snapshotHistoryMillis,
84
- configuration: this.configuration,
85
- log: this.log,
86
- });
87
- this.eventLoopSignal = createEventLoopLoadSignal({
88
- eventLoopSnapshotIntervalSecs,
89
- maxBlockedMillis,
90
- snapshotHistoryMillis,
91
- });
92
- this.cpuSignal = createCpuLoadSignal({
93
- snapshotHistoryMillis,
94
- configuration: this.configuration,
95
- });
96
- this.clientSignal = createClientLoadSignal({
97
- client: this.client,
98
- clientSnapshotIntervalSecs,
99
- maxClientErrors,
100
- snapshotHistoryMillis,
101
- });
38
+ const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
39
+ // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
40
+ // over then too - so there is nothing to thread in here beyond the caller's tuning.
41
+ if (memory !== false)
42
+ this.memorySignal = new MemoryLoadSignal(memory);
43
+ if (eventLoop !== false)
44
+ this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
45
+ if (cpu !== false)
46
+ this.cpuSignal = new CpuLoadSignal(cpu);
47
+ if (client !== false)
48
+ this.clientSignal = new ClientLoadSignal(client);
102
49
  }
103
50
  /**
104
- * Starts capturing snapshots at configured intervals.
51
+ * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
52
+ * be queried with, which is also how much history they retain.
105
53
  */
106
- async start() {
107
- await this.memorySignal.start();
108
- await this.eventLoopSignal.start();
109
- await this.cpuSignal.start();
110
- await this.clientSignal.start();
54
+ async start(context) {
55
+ await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
111
56
  }
112
57
  /**
113
58
  * Stops all resource capturing.
114
59
  */
115
60
  async stop() {
116
- await this.memorySignal.stop();
117
- await this.eventLoopSignal.stop();
118
- await this.cpuSignal.stop();
119
- await this.clientSignal.stop();
61
+ await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
120
62
  // Allow microtask queue to unwind before stop returns.
121
63
  await new Promise((resolve) => {
122
64
  setImmediate(resolve);
123
65
  });
124
66
  }
125
- /**
126
- * Returns a sample of latest memory snapshots, with the size of the sample defined
127
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
128
- */
129
- getMemorySample(sampleDurationMillis) {
130
- return this.memorySignal.getSample(sampleDurationMillis);
131
- }
132
- /**
133
- * Returns a sample of latest event loop snapshots, with the size of the sample defined
134
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
135
- */
136
- getEventLoopSample(sampleDurationMillis) {
137
- return this.eventLoopSignal.getSample(sampleDurationMillis);
138
- }
139
- /**
140
- * Returns a sample of latest CPU snapshots, with the size of the sample defined
141
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
142
- */
143
- getCpuSample(sampleDurationMillis) {
144
- return this.cpuSignal.getSample(sampleDurationMillis);
145
- }
146
- /**
147
- * Returns a sample of latest Client snapshots, with the size of the sample defined
148
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
149
- */
150
- getClientSample(sampleDurationMillis) {
151
- return this.clientSignal.getSample(sampleDurationMillis);
152
- }
153
67
  }
@@ -1,5 +1,5 @@
1
1
  import type { LoadSignal } from './load_signal.js';
2
- import { Snapshotter } from './snapshotter.js';
2
+ import type { Snapshotter } from './snapshotter.js';
3
3
  /**
4
4
  * Represents the current status of the system.
5
5
  */
@@ -33,6 +33,22 @@ export interface SystemInfo {
33
33
  */
34
34
  loadSignalInfo?: Record<string, ClientInfo>;
35
35
  }
36
+ /**
37
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
38
+ * @internal
39
+ */
40
+ export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
41
+ /**
42
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
43
+ * therefore how much history the signals are asked to retain.
44
+ * @internal
45
+ */
46
+ export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
47
+ /**
48
+ * An implementation detail of the {@link ConcurrencySystem} — configure it through
49
+ * {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
50
+ * @internal
51
+ */
36
52
  export interface SystemStatusOptions {
37
53
  /**
38
54
  * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
@@ -40,38 +56,21 @@ export interface SystemStatusOptions {
40
56
  */
41
57
  currentHistorySecs?: number;
42
58
  /**
43
- * Sets the maximum ratio of overloaded snapshots in a memory sample.
44
- * If the sample exceeds this ratio, the system will be overloaded.
45
- * @default 0.2
46
- */
47
- maxMemoryOverloadedRatio?: number;
48
- /**
49
- * Sets the maximum ratio of overloaded snapshots in an event loop sample.
50
- * If the sample exceeds this ratio, the system will be overloaded.
51
- * @default 0.6
59
+ * Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
60
+ * autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
61
+ * private retention cannot silently widen the window.
62
+ * @default 30
52
63
  */
53
- maxEventLoopOverloadedRatio?: number;
64
+ historySecs?: number;
54
65
  /**
55
- * Sets the maximum ratio of overloaded snapshots in a CPU sample.
56
- * If the sample exceeds this ratio, the system will be overloaded.
57
- * @default 0.4
66
+ * The `Snapshotter` whose built-in signals are evaluated.
58
67
  */
59
- maxCpuOverloadedRatio?: number;
60
- /**
61
- * Sets the maximum ratio of overloaded snapshots in a Client sample.
62
- * If the sample exceeds this ratio, the system will be overloaded.
63
- * @default 0.3
64
- */
65
- maxClientOverloadedRatio?: number;
66
- /**
67
- * The `Snapshotter` instance to be queried for `SystemStatus`.
68
- */
69
- snapshotter?: Snapshotter;
68
+ snapshotter: Snapshotter;
70
69
  /**
71
70
  * Additional load signals to include in the system status evaluation.
72
71
  * These are evaluated alongside the built-in memory, CPU, event loop,
73
72
  * and client signals. If any signal reports overload, the system is
74
- * considered overloaded.
73
+ * considered overloaded. Each signal carries its own overload ratio.
75
74
  */
76
75
  loadSignals?: LoadSignal[];
77
76
  }
@@ -93,38 +92,35 @@ export interface FinalStatistics {
93
92
  crawlerRuntimeMillis: number;
94
93
  }
95
94
  /**
96
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
97
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
98
- * and {@link SystemStatus.getHistoricalStatus}.
99
- * The system status is calculated using a weighted average of overloaded
100
- * messages in the snapshots, with the weights being the time intervals
101
- * between the snapshots. Each resource is calculated separately
102
- * and the system is overloaded whenever at least one resource is overloaded.
103
- * The class is used by the {@link AutoscaledPool} class.
95
+ * Reads the overload verdict of every signal the {@link Snapshotter}'s built-in four plus any custom ones — and
96
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
97
+ * is overloaded whenever at least one of them is.
104
98
  *
105
- * {@link SystemStatus.getCurrentStatus}
106
- * returns a boolean that represents the current status of the system.
107
- * The length of the current timeframe in seconds is configurable
108
- * by the `currentHistorySecs` option and represents the max age
109
- * of snapshots to be considered for the calculation.
99
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
100
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
101
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
110
102
  *
111
- * {@link SystemStatus.getHistoricalStatus}
112
- * returns a boolean that represents the long-term status
113
- * of the system. It considers the full snapshot history available
114
- * in the {@link Snapshotter} instance.
115
- * @category Scaling
103
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
104
+ * {@link ConcurrencySystemOptions}.
105
+ * @internal
116
106
  */
117
107
  export declare class SystemStatus {
118
108
  private readonly currentHistoryMillis;
119
- private readonly snapshotter;
109
+ private readonly historyMillis;
120
110
  private readonly signals;
111
+ constructor(options: SystemStatusOptions);
121
112
  /**
122
- * Per-signal ratio overrides. The built-in four get their overrides from
123
- * the legacy `max*OverloadedRatio` options; custom signals use their own
124
- * `overloadedRatio`.
113
+ * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
114
+ * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
115
+ * defaults.
125
116
  */
126
- private ratioOverrides;
127
- constructor(options?: SystemStatusOptions);
117
+ get maxSampleWindowMillis(): number;
118
+ /**
119
+ * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
120
+ * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
121
+ * only the last is reported.
122
+ */
123
+ private assertUniqueSignalNames;
128
124
  /**
129
125
  * Returns an {@link SystemInfo} object with the following structure:
130
126
  *
@@ -154,9 +150,8 @@ export declare class SystemStatus {
154
150
  * }
155
151
  * ```
156
152
  *
157
- * Where the `isSystemIdle` property is set to `false` if the system
158
- * has been overloaded in the full history of the {@link Snapshotter}
159
- * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
153
+ * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
154
+ * `historySecs` seconds and `true` otherwise.
160
155
  */
161
156
  getHistoricalStatus(): SystemInfo;
162
157
  /**