@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +1 -1
  2. package/configuration.d.ts +16 -47
  3. package/configuration.js +13 -25
  4. package/debug.js +4 -4
  5. package/errors.d.ts +28 -38
  6. package/errors.js +33 -47
  7. package/events/event_manager.d.ts +2 -2
  8. package/events/event_manager.js +7 -6
  9. package/events/index.d.ts +1 -0
  10. package/events/local_event_manager.d.ts +1 -8
  11. package/events/local_event_manager.js +13 -13
  12. package/events/system_info.d.ts +38 -0
  13. package/index.d.ts +2 -8
  14. package/index.js +4 -8
  15. package/internal.d.ts +8 -0
  16. package/internal.js +9 -0
  17. package/log.d.ts +10 -11
  18. package/log.js +52 -20
  19. package/memory-storage/memory-storage.d.ts +15 -18
  20. package/memory-storage/memory-storage.js +80 -58
  21. package/memory-storage/resource-clients/dataset.d.ts +1 -6
  22. package/memory-storage/resource-clients/dataset.js +23 -31
  23. package/memory-storage/resource-clients/key-value-store.d.ts +1 -10
  24. package/memory-storage/resource-clients/key-value-store.js +43 -67
  25. package/memory-storage/resource-clients/request-queue.d.ts +1 -42
  26. package/memory-storage/resource-clients/request-queue.js +109 -117
  27. package/owned_or_injected.d.ts +1 -3
  28. package/owned_or_injected.js +17 -17
  29. package/package.json +17 -20
  30. package/proxy_configuration.d.ts +21 -26
  31. package/proxy_configuration.js +35 -25
  32. package/recoverable_state.d.ts +104 -47
  33. package/recoverable_state.js +199 -74
  34. package/request.d.ts +20 -107
  35. package/request.js +78 -244
  36. package/serialization.js +17 -16
  37. package/service_locator.d.ts +22 -10
  38. package/service_locator.js +59 -48
  39. package/storages/batched_adds.d.ts +37 -0
  40. package/storages/batched_adds.js +73 -0
  41. package/storages/dataset.d.ts +13 -8
  42. package/storages/dataset.js +149 -40
  43. package/storages/index.d.ts +4 -4
  44. package/storages/index.js +2 -4
  45. package/storages/key_value_store.d.ts +16 -35
  46. package/storages/key_value_store.js +223 -110
  47. package/storages/key_value_store_codec.js +6 -11
  48. package/storages/request_dedup_cache.d.ts +1 -4
  49. package/storages/request_dedup_cache.js +15 -15
  50. package/storages/request_list.d.ts +9 -104
  51. package/storages/request_list.js +236 -233
  52. package/storages/request_loader.d.ts +49 -18
  53. package/storages/request_loader.js +36 -1
  54. package/storages/request_manager.d.ts +86 -0
  55. package/storages/request_manager_tandem.d.ts +14 -38
  56. package/storages/request_manager_tandem.js +67 -64
  57. package/storages/request_queue.d.ts +23 -50
  58. package/storages/request_queue.js +371 -226
  59. package/storages/storage_instance_manager.d.ts +2 -4
  60. package/storages/storage_instance_manager.js +21 -21
  61. package/storages/storage_stats.d.ts +1 -1
  62. package/storages/storage_stats.js +4 -4
  63. package/storages/transaction.d.ts +270 -0
  64. package/storages/transaction.js +296 -0
  65. package/storages/utils.d.ts +6 -3
  66. package/storages/utils.js +11 -2
  67. package/system-info/runtime.js +7 -7
  68. package/url.d.ts +9 -0
  69. package/url.js +11 -0
  70. package/validators.d.ts +23 -25
  71. package/validators.js +14 -25
  72. package/autoscaling/autoscaled_pool.d.ts +0 -213
  73. package/autoscaling/autoscaled_pool.js +0 -378
  74. package/autoscaling/client_load_signal.d.ts +0 -59
  75. package/autoscaling/client_load_signal.js +0 -73
  76. package/autoscaling/concurrency_system.d.ts +0 -283
  77. package/autoscaling/concurrency_system.js +0 -350
  78. package/autoscaling/cpu_load_signal.d.ts +0 -44
  79. package/autoscaling/cpu_load_signal.js +0 -46
  80. package/autoscaling/event_loop_load_signal.d.ts +0 -54
  81. package/autoscaling/event_loop_load_signal.js +0 -60
  82. package/autoscaling/index.d.ts +0 -9
  83. package/autoscaling/index.js +0 -9
  84. package/autoscaling/load_signal.d.ts +0 -99
  85. package/autoscaling/load_signal.js +0 -103
  86. package/autoscaling/memory_load_signal.d.ts +0 -56
  87. package/autoscaling/memory_load_signal.js +0 -106
  88. package/autoscaling/snapshotter.d.ts +0 -87
  89. package/autoscaling/snapshotter.js +0 -67
  90. package/autoscaling/system_status.d.ts +0 -161
  91. package/autoscaling/system_status.js +0 -139
  92. package/autoscaling/weighted_avg.d.ts +0 -5
  93. package/autoscaling/weighted_avg.js +0 -14
  94. package/cookie_utils.d.ts +0 -44
  95. package/cookie_utils.js +0 -122
  96. package/crawlers/context_pipeline.d.ts +0 -70
  97. package/crawlers/context_pipeline.js +0 -122
  98. package/crawlers/crawler_commons.d.ts +0 -257
  99. package/crawlers/crawler_commons.js +0 -107
  100. package/crawlers/error_snapshotter.d.ts +0 -59
  101. package/crawlers/error_snapshotter.js +0 -117
  102. package/crawlers/error_tracker.d.ts +0 -54
  103. package/crawlers/error_tracker.js +0 -308
  104. package/crawlers/index.d.ts +0 -5
  105. package/crawlers/index.js +0 -5
  106. package/crawlers/internals/types.d.ts +0 -7
  107. package/crawlers/statistics.d.ts +0 -209
  108. package/crawlers/statistics.js +0 -350
  109. package/enqueue_links/enqueue_links.d.ts +0 -264
  110. package/enqueue_links/enqueue_links.js +0 -271
  111. package/enqueue_links/index.d.ts +0 -2
  112. package/enqueue_links/index.js +0 -2
  113. package/enqueue_links/shared.d.ts +0 -83
  114. package/enqueue_links/shared.js +0 -221
  115. package/router.d.ts +0 -309
  116. package/router.js +0 -309
  117. package/session_pool/consts.d.ts +0 -3
  118. package/session_pool/consts.js +0 -3
  119. package/session_pool/errors.d.ts +0 -7
  120. package/session_pool/errors.js +0 -11
  121. package/session_pool/fingerprint.d.ts +0 -9
  122. package/session_pool/fingerprint.js +0 -30
  123. package/session_pool/index.d.ts +0 -4
  124. package/session_pool/index.js +0 -4
  125. package/session_pool/session.d.ts +0 -161
  126. package/session_pool/session.js +0 -218
  127. package/session_pool/session_pool.d.ts +0 -246
  128. package/session_pool/session_pool.js +0 -386
  129. package/storages/access_checking.d.ts +0 -12
  130. package/storages/access_checking.js +0 -17
  131. package/storages/sitemap_request_loader.d.ts +0 -249
  132. package/storages/sitemap_request_loader.js +0 -432
  133. /package/{crawlers/internals/types.js → events/system_info.js} +0 -0
@@ -1,103 +0,0 @@
1
- import { weightedAvg } from './weighted_avg.js';
2
- /**
3
- * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
4
- * and so can yours — it is the only part of their machinery worth reusing.
5
- */
6
- export class SnapshotStore {
7
- snapshots = [];
8
- /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
9
- historyMillis = Infinity;
10
- /**
11
- * Sizes retention to the window the signal will be sampled over, as handed to it in
12
- * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
13
- * its start context grows unboundedly.
14
- */
15
- useSampleWindow(maxSampleWindowMillis) {
16
- this.historyMillis = maxSampleWindowMillis;
17
- }
18
- /**
19
- * Add a snapshot and prune entries older than the history window.
20
- */
21
- push(snapshot, now = snapshot.createdAt) {
22
- // Inline pruning to avoid private-method transpilation issues
23
- let oldCount = 0;
24
- for (let i = 0; i < this.snapshots.length; i++) {
25
- const { createdAt } = this.snapshots[i];
26
- if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
27
- oldCount++;
28
- else
29
- break;
30
- }
31
- if (oldCount)
32
- this.snapshots.splice(0, oldCount);
33
- this.snapshots.push(snapshot);
34
- }
35
- /**
36
- * Return all snapshots, or only those within the given time window.
37
- */
38
- getSample(sampleDurationMillis) {
39
- if (!sampleDurationMillis)
40
- return this.snapshots;
41
- const sample = [];
42
- let idx = this.snapshots.length;
43
- if (!idx)
44
- return sample;
45
- const latestTime = this.snapshots[idx - 1].createdAt;
46
- while (idx--) {
47
- const snapshot = this.snapshots[idx];
48
- if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
49
- sample.unshift(snapshot);
50
- }
51
- else {
52
- break;
53
- }
54
- }
55
- return sample;
56
- }
57
- /**
58
- * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
59
- * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
60
- */
61
- getAll() {
62
- return this.snapshots;
63
- }
64
- /**
65
- * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
66
- * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
67
- * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
68
- * rather than on stop leaves a finished session readable.
69
- */
70
- clear() {
71
- this.snapshots = [];
72
- }
73
- }
74
- /**
75
- * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
76
- * overloaded ratio, using a time-weighted average. This is the shared
77
- * evaluation logic used by `SystemStatus` for all signal types.
78
- * @internal
79
- */
80
- export function evaluateLoadSignalSample(sample, overloadedRatio) {
81
- if (sample.length === 0) {
82
- return {
83
- isOverloaded: false,
84
- limitRatio: overloadedRatio,
85
- actualRatio: 0,
86
- };
87
- }
88
- const weights = [];
89
- const values = [];
90
- for (let i = 1; i < sample.length; i++) {
91
- const previous = sample[i - 1];
92
- const current = sample[i];
93
- const weight = +current.createdAt - +previous.createdAt;
94
- weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
95
- values.push(+current.isOverloaded);
96
- }
97
- const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
98
- return {
99
- isOverloaded: wAvg > overloadedRatio,
100
- limitRatio: overloadedRatio,
101
- actualRatio: Math.round(wAvg * 1000) / 1000,
102
- };
103
- }
@@ -1,56 +0,0 @@
1
- import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
2
- import type { SystemInfo } from './system_status.js';
3
- /**
4
- * A snapshot produced by the built-in memory signal.
5
- * @internal
6
- */
7
- export interface MemorySnapshot extends LoadSnapshot {
8
- usedBytes?: number;
9
- }
10
- /**
11
- * Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
12
- * {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
13
- */
14
- export interface MemoryLoadSignalOptions {
15
- /**
16
- * Defines the maximum ratio of total memory that can be used.
17
- * Exceeding this limit overloads the memory.
18
- * @default 0.9
19
- */
20
- maxUsedRatio?: number;
21
- /**
22
- * Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
23
- * @default 0.2
24
- */
25
- overloadedRatio?: number;
26
- }
27
- /**
28
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
29
- * threshold. Also warns when memory use becomes critical.
30
- *
31
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
32
- *
33
- * @category Scaling
34
- */
35
- export declare class MemoryLoadSignal implements LoadSignal {
36
- readonly name = "memInfo";
37
- readonly overloadedRatio: number;
38
- private readonly store;
39
- private readonly maxUsedRatio;
40
- /** All resolved in `start()`, before anything that reads them can fire. */
41
- private config;
42
- private log;
43
- private maxMemoryBytes;
44
- private events?;
45
- private maxMemoryRatio;
46
- private lastLoggedCriticalMemoryOverloadAt;
47
- constructor(options?: MemoryLoadSignalOptions);
48
- start(context: LoadSignalStartContext): Promise<void>;
49
- stop(): Promise<void>;
50
- getSample(sampleDurationMillis?: number): LoadSnapshot[];
51
- /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
52
- handle(systemInfo: SystemInfo): void;
53
- /** @internal */
54
- _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
55
- private _getTotalMemoryBytes;
56
- }
@@ -1,106 +0,0 @@
1
- import { serviceLocator } from '../service_locator.js';
2
- import { getMemoryInfo } from '../system-info/memory-info.js';
3
- import { isContainerized } from '../system-info/runtime.js';
4
- import { SnapshotStore } from './load_signal.js';
5
- const RESERVE_MEMORY_RATIO = 0.5;
6
- const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
7
- /**
8
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
9
- * threshold. Also warns when memory use becomes critical.
10
- *
11
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
12
- *
13
- * @category Scaling
14
- */
15
- export class MemoryLoadSignal {
16
- name = 'memInfo';
17
- overloadedRatio;
18
- store = new SnapshotStore();
19
- maxUsedRatio;
20
- /** All resolved in `start()`, before anything that reads them can fire. */
21
- config;
22
- log;
23
- maxMemoryBytes;
24
- events;
25
- maxMemoryRatio;
26
- lastLoggedCriticalMemoryOverloadAt = null;
27
- constructor(options = {}) {
28
- this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
29
- this.overloadedRatio = options.overloadedRatio ?? 0.2;
30
- this.handle = this.handle.bind(this);
31
- }
32
- async start(context) {
33
- this.store.useSampleWindow(context.maxSampleWindowMillis);
34
- // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
35
- this.store.clear();
36
- // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
37
- // between systems) must not capture whichever services happened to be registered at that moment.
38
- this.config = serviceLocator.getConfiguration();
39
- this.events = serviceLocator.getEventManager();
40
- this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
41
- const memoryMbytes = this.config.memoryMbytes ?? 0;
42
- if (memoryMbytes > 0) {
43
- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
44
- }
45
- else {
46
- this.maxMemoryRatio = this.config.availableMemoryRatio;
47
- if (!this.maxMemoryRatio) {
48
- throw new Error('availableMemoryRatio is not set in configuration.');
49
- }
50
- else {
51
- this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
52
- 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
53
- }
54
- // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
55
- this.maxMemoryBytes = await this._getTotalMemoryBytes();
56
- }
57
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
58
- }
59
- async stop() {
60
- this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
61
- this.events = undefined;
62
- }
63
- getSample(sampleDurationMillis) {
64
- return this.store.getSample(sampleDurationMillis);
65
- }
66
- /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
67
- handle(systemInfo) {
68
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
69
- const { memCurrentBytes, memTotalBytes } = systemInfo;
70
- let maxMemoryBytes = this.maxMemoryBytes;
71
- if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
72
- maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
73
- }
74
- const snapshot = {
75
- createdAt,
76
- isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
77
- usedBytes: memCurrentBytes,
78
- };
79
- this.store.push(snapshot, createdAt);
80
- this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
81
- }
82
- /** @internal */
83
- _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
84
- const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
85
- const { memCurrentBytes } = systemInfo;
86
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
87
- if (this.lastLoggedCriticalMemoryOverloadAt &&
88
- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
89
- return;
90
- const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
91
- const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
92
- const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
93
- const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
94
- if (isCriticalOverload) {
95
- const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
96
- const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
97
- this.log.warning('Memory is critically overloaded. ' +
98
- `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
99
- this.lastLoggedCriticalMemoryOverloadAt = createdAt;
100
- }
101
- }
102
- async _getTotalMemoryBytes() {
103
- const containerized = this.config.containerized ?? (await isContainerized());
104
- return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
105
- }
106
- }
@@ -1,87 +0,0 @@
1
- import type { ClientLoadSignalOptions } from './client_load_signal.js';
2
- import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
3
- import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
4
- import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
5
- import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
6
- /**
7
- * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
8
- *
9
- * Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
10
- * corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
11
- * `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
12
- * unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
13
- *
14
- * How far back the signals are evaluated is *not* set here, but by
15
- * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
16
- * {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
17
- */
18
- export interface LoadSignalsOptions {
19
- /**
20
- * Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
21
- * it off.
22
- */
23
- memory?: MemoryLoadSignalOptions | false;
24
- /**
25
- * Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
26
- * ratio), or `false` to switch it off — which also stops its measuring interval.
27
- */
28
- eventLoop?: EventLoopLoadSignalOptions | false;
29
- /**
30
- * Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
31
- */
32
- cpu?: CpuLoadSignalOptions | false;
33
- /**
34
- * Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
35
- * `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
36
- * signal otherwise polls it every second to no purpose.
37
- */
38
- client?: ClientLoadSignalOptions | false;
39
- /**
40
- * Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
41
- * alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
42
- * is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
43
- * not collide with an enabled built-in's.
44
- */
45
- custom?: LoadSignal[];
46
- }
47
- /**
48
- * An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
49
- * {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
50
- * them here.
51
- * @internal
52
- */
53
- export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
54
- /**
55
- * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
56
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
57
- * that were not switched off and driving their shared lifecycle.
58
- *
59
- * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
60
- * are simply forwarded to the corresponding constructor.
61
- * @internal
62
- */
63
- export declare class Snapshotter {
64
- private readonly memorySignal?;
65
- private readonly eventLoopSignal?;
66
- private readonly cpuSignal?;
67
- private readonly clientSignal?;
68
- /**
69
- * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
70
- * instances. Signals switched off through the options are simply absent — the system status reports them as
71
- * not overloaded.
72
- */
73
- getLoadSignals(): LoadSignal[];
74
- /**
75
- * @param [options] All `Snapshotter` configuration options.
76
- */
77
- constructor(options?: SnapshotterOptions);
78
- /**
79
- * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
80
- * be queried with, which is also how much history they retain.
81
- */
82
- start(context: LoadSignalStartContext): Promise<void>;
83
- /**
84
- * Stops all resource capturing.
85
- */
86
- stop(): Promise<void>;
87
- }
@@ -1,67 +0,0 @@
1
- import { ClientLoadSignal } from './client_load_signal.js';
2
- import { CpuLoadSignal } from './cpu_load_signal.js';
3
- import { EventLoopLoadSignal } from './event_loop_load_signal.js';
4
- import { MemoryLoadSignal } from './memory_load_signal.js';
5
- /**
6
- * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
7
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
8
- * that were not switched off and driving their shared lifecycle.
9
- *
10
- * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
11
- * are simply forwarded to the corresponding constructor.
12
- * @internal
13
- */
14
- export class Snapshotter {
15
- // Absent when switched off through the corresponding option (e.g. `client: false`).
16
- memorySignal;
17
- eventLoopSignal;
18
- cpuSignal;
19
- clientSignal;
20
- /**
21
- * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
22
- * instances. Signals switched off through the options are simply absent — the system status reports them as
23
- * not overloaded.
24
- */
25
- getLoadSignals() {
26
- const builtin = [
27
- this.memorySignal,
28
- this.eventLoopSignal,
29
- this.cpuSignal,
30
- this.clientSignal,
31
- ];
32
- return builtin.filter((signal) => signal !== undefined);
33
- }
34
- /**
35
- * @param [options] All `Snapshotter` configuration options.
36
- */
37
- constructor(options = {}) {
38
- const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
39
- // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
40
- // over then too - so there is nothing to thread in here beyond the caller's tuning.
41
- if (memory !== false)
42
- this.memorySignal = new MemoryLoadSignal(memory);
43
- if (eventLoop !== false)
44
- this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
45
- if (cpu !== false)
46
- this.cpuSignal = new CpuLoadSignal(cpu);
47
- if (client !== false)
48
- this.clientSignal = new ClientLoadSignal(client);
49
- }
50
- /**
51
- * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
52
- * be queried with, which is also how much history they retain.
53
- */
54
- async start(context) {
55
- await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
56
- }
57
- /**
58
- * Stops all resource capturing.
59
- */
60
- async stop() {
61
- await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
62
- // Allow microtask queue to unwind before stop returns.
63
- await new Promise((resolve) => {
64
- setImmediate(resolve);
65
- });
66
- }
67
- }
@@ -1,161 +0,0 @@
1
- import type { LoadSignal } from './load_signal.js';
2
- import type { Snapshotter } from './snapshotter.js';
3
- /**
4
- * Represents the current status of the system.
5
- */
6
- export interface SystemInfo {
7
- /** If false, system is being overloaded. */
8
- isSystemIdle: boolean;
9
- memInfo: ClientInfo;
10
- eventLoopInfo: ClientInfo;
11
- cpuInfo: ClientInfo;
12
- clientInfo: ClientInfo;
13
- memTotalBytes?: number;
14
- memCurrentBytes?: number;
15
- /**
16
- * Platform only property
17
- * @internal
18
- */
19
- cpuCurrentUsage?: number;
20
- /**
21
- * Platform only property
22
- * @internal
23
- */
24
- isCpuOverloaded?: boolean;
25
- /**
26
- * Platform only property
27
- * @internal
28
- */
29
- createdAt?: Date;
30
- /**
31
- * Status of additional load signals beyond the built-in four.
32
- * Keys are `LoadSignal.name` values, values are overload info.
33
- */
34
- loadSignalInfo?: Record<string, ClientInfo>;
35
- }
36
- /**
37
- * How far back the *current* system status looks by default — the window that gates task dispatch.
38
- * @internal
39
- */
40
- export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
41
- /**
42
- * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
43
- * therefore how much history the signals are asked to retain.
44
- * @internal
45
- */
46
- export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
47
- /**
48
- * An implementation detail of the {@link ConcurrencySystem} — configure it through
49
- * {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
50
- * @internal
51
- */
52
- export interface SystemStatusOptions {
53
- /**
54
- * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
55
- * @default 5
56
- */
57
- currentHistorySecs?: number;
58
- /**
59
- * Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
60
- * autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
61
- * private retention cannot silently widen the window.
62
- * @default 30
63
- */
64
- historySecs?: number;
65
- /**
66
- * The `Snapshotter` whose built-in signals are evaluated.
67
- */
68
- snapshotter: Snapshotter;
69
- /**
70
- * Additional load signals to include in the system status evaluation.
71
- * These are evaluated alongside the built-in memory, CPU, event loop,
72
- * and client signals. If any signal reports overload, the system is
73
- * considered overloaded. Each signal carries its own overload ratio.
74
- */
75
- loadSignals?: LoadSignal[];
76
- }
77
- export interface ClientInfo {
78
- isOverloaded: boolean;
79
- limitRatio: number;
80
- actualRatio: number;
81
- }
82
- export interface FinalStatistics {
83
- requestsFinished: number;
84
- requestsFailed: number;
85
- retryHistogram: number[];
86
- requestAvgFailedDurationMillis: number;
87
- requestAvgFinishedDurationMillis: number;
88
- requestsFinishedPerMinute: number;
89
- requestsFailedPerMinute: number;
90
- requestTotalDurationMillis: number;
91
- requestsTotal: number;
92
- crawlerRuntimeMillis: number;
93
- }
94
- /**
95
- * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
96
- * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
97
- * is overloaded whenever at least one of them is.
98
- *
99
- * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
100
- * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
101
- * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
102
- *
103
- * An implementation detail of the {@link ConcurrencySystem}, configured through
104
- * {@link ConcurrencySystemOptions}.
105
- * @internal
106
- */
107
- export declare class SystemStatus {
108
- private readonly currentHistoryMillis;
109
- private readonly historyMillis;
110
- private readonly signals;
111
- constructor(options: SystemStatusOptions);
112
- /**
113
- * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
114
- * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
115
- * defaults.
116
- */
117
- get maxSampleWindowMillis(): number;
118
- /**
119
- * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
120
- * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
121
- * only the last is reported.
122
- */
123
- private assertUniqueSignalNames;
124
- /**
125
- * Returns an {@link SystemInfo} object with the following structure:
126
- *
127
- * ```javascript
128
- * {
129
- * isSystemIdle: Boolean,
130
- * memInfo: Object,
131
- * eventLoopInfo: Object,
132
- * cpuInfo: Object
133
- * }
134
- * ```
135
- *
136
- * Where the `isSystemIdle` property is set to `false` if the system
137
- * has been overloaded in the last `options.currentHistorySecs` seconds,
138
- * and `true` otherwise.
139
- */
140
- getCurrentStatus(): SystemInfo;
141
- /**
142
- * Returns an {@link SystemInfo} object with the following structure:
143
- *
144
- * ```javascript
145
- * {
146
- * isSystemIdle: Boolean,
147
- * memInfo: Object,
148
- * eventLoopInfo: Object,
149
- * cpuInfo: Object
150
- * }
151
- * ```
152
- *
153
- * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
154
- * `historySecs` seconds and `true` otherwise.
155
- */
156
- getHistoricalStatus(): SystemInfo;
157
- /**
158
- * Returns a system status object.
159
- */
160
- private isSystemIdle;
161
- }