@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +5 -6
  3. package/autoscaling/autoscaled_pool.js +12 -11
  4. package/autoscaling/client_load_signal.d.ts +25 -0
  5. package/autoscaling/client_load_signal.js +36 -0
  6. package/autoscaling/cpu_load_signal.d.ts +28 -0
  7. package/autoscaling/cpu_load_signal.js +24 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  9. package/autoscaling/event_loop_load_signal.js +35 -0
  10. package/autoscaling/index.d.ts +5 -1
  11. package/autoscaling/index.js +5 -1
  12. package/autoscaling/load_signal.d.ts +99 -0
  13. package/autoscaling/load_signal.js +135 -0
  14. package/autoscaling/memory_load_signal.d.ts +43 -0
  15. package/autoscaling/memory_load_signal.js +102 -0
  16. package/autoscaling/snapshotter.d.ts +35 -72
  17. package/autoscaling/snapshotter.js +78 -181
  18. package/autoscaling/system_status.d.ts +21 -36
  19. package/autoscaling/system_status.js +46 -84
  20. package/configuration.d.ts +88 -228
  21. package/configuration.js +162 -224
  22. package/cookie_utils.d.ts +4 -3
  23. package/cookie_utils.js +20 -13
  24. package/crawlers/context_pipeline.d.ts +70 -0
  25. package/crawlers/context_pipeline.js +122 -0
  26. package/crawlers/crawler_commons.d.ts +20 -30
  27. package/crawlers/crawler_commons.js +12 -21
  28. package/crawlers/crawler_utils.d.ts +2 -3
  29. package/crawlers/crawler_utils.js +1 -2
  30. package/crawlers/error_snapshotter.d.ts +3 -3
  31. package/crawlers/error_snapshotter.js +2 -3
  32. package/crawlers/error_tracker.d.ts +2 -2
  33. package/crawlers/error_tracker.js +0 -1
  34. package/crawlers/index.d.ts +1 -2
  35. package/crawlers/index.js +1 -2
  36. package/crawlers/internals/types.d.ts +7 -0
  37. package/crawlers/internals/types.js +1 -0
  38. package/crawlers/statistics.d.ts +22 -17
  39. package/crawlers/statistics.js +30 -26
  40. package/enqueue_links/enqueue_links.d.ts +44 -22
  41. package/enqueue_links/enqueue_links.js +51 -30
  42. package/enqueue_links/index.d.ts +0 -1
  43. package/enqueue_links/index.js +0 -1
  44. package/enqueue_links/shared.d.ts +25 -9
  45. package/enqueue_links/shared.js +69 -38
  46. package/errors.d.ts +53 -4
  47. package/errors.js +70 -5
  48. package/events/event_manager.d.ts +34 -8
  49. package/events/event_manager.js +8 -10
  50. package/events/index.d.ts +0 -1
  51. package/events/index.js +0 -1
  52. package/events/local_event_manager.d.ts +15 -5
  53. package/events/local_event_manager.js +37 -40
  54. package/index.d.ts +4 -3
  55. package/index.js +3 -2
  56. package/log.d.ts +82 -3
  57. package/log.js +102 -1
  58. package/memory-storage/consts.d.ts +4 -0
  59. package/memory-storage/consts.js +4 -0
  60. package/memory-storage/index.d.ts +1 -0
  61. package/memory-storage/index.js +1 -0
  62. package/memory-storage/memory-storage.d.ts +46 -0
  63. package/memory-storage/memory-storage.js +136 -0
  64. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  65. package/memory-storage/resource-clients/common/base-client.js +6 -0
  66. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  67. package/memory-storage/resource-clients/dataset.js +113 -0
  68. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  69. package/memory-storage/resource-clients/key-value-store.js +203 -0
  70. package/memory-storage/resource-clients/request-queue.d.ts +96 -0
  71. package/memory-storage/resource-clients/request-queue.js +421 -0
  72. package/memory-storage/utils.d.ts +17 -0
  73. package/memory-storage/utils.js +42 -0
  74. package/package.json +12 -10
  75. package/proxy_configuration.d.ts +14 -149
  76. package/proxy_configuration.js +19 -168
  77. package/recoverable_state.d.ts +120 -0
  78. package/recoverable_state.js +141 -0
  79. package/request.d.ts +84 -13
  80. package/request.js +107 -28
  81. package/router.d.ts +117 -15
  82. package/router.js +124 -30
  83. package/serialization.d.ts +0 -1
  84. package/serialization.js +1 -2
  85. package/service_locator.d.ts +156 -0
  86. package/service_locator.js +238 -0
  87. package/session_pool/consts.d.ts +1 -2
  88. package/session_pool/consts.js +1 -2
  89. package/session_pool/errors.d.ts +0 -1
  90. package/session_pool/errors.js +0 -1
  91. package/session_pool/fingerprint.d.ts +9 -0
  92. package/session_pool/fingerprint.js +30 -0
  93. package/session_pool/index.d.ts +0 -2
  94. package/session_pool/index.js +0 -2
  95. package/session_pool/session.d.ts +35 -73
  96. package/session_pool/session.js +46 -99
  97. package/session_pool/session_pool.d.ts +71 -72
  98. package/session_pool/session_pool.js +111 -100
  99. package/storages/access_checking.d.ts +0 -1
  100. package/storages/access_checking.js +5 -2
  101. package/storages/dataset.d.ts +101 -51
  102. package/storages/dataset.js +168 -126
  103. package/storages/index.d.ts +8 -7
  104. package/storages/index.js +6 -7
  105. package/storages/key_value_store.d.ts +165 -36
  106. package/storages/key_value_store.js +269 -122
  107. package/storages/key_value_store_codec.d.ts +32 -0
  108. package/storages/key_value_store_codec.js +113 -0
  109. package/storages/request_dedup_cache.d.ts +23 -0
  110. package/storages/request_dedup_cache.js +48 -0
  111. package/storages/request_list.d.ts +38 -82
  112. package/storages/request_list.js +65 -41
  113. package/storages/request_loader.d.ts +96 -0
  114. package/storages/request_loader.js +1 -0
  115. package/storages/request_manager.d.ts +33 -0
  116. package/storages/request_manager.js +1 -0
  117. package/storages/request_manager_tandem.d.ts +106 -0
  118. package/storages/request_manager_tandem.js +197 -0
  119. package/storages/request_queue.d.ts +303 -46
  120. package/storages/request_queue.js +629 -214
  121. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
  122. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
  123. package/storages/storage_instance_manager.d.ts +91 -0
  124. package/storages/storage_instance_manager.js +258 -0
  125. package/storages/storage_stats.d.ts +48 -0
  126. package/storages/storage_stats.js +29 -0
  127. package/storages/utils.d.ts +51 -6
  128. package/storages/utils.js +60 -9
  129. package/system-info/cpu-info.d.ts +67 -0
  130. package/system-info/cpu-info.js +216 -0
  131. package/system-info/memory-info.d.ts +31 -0
  132. package/system-info/memory-info.js +115 -0
  133. package/system-info/ps-tree.d.ts +17 -0
  134. package/system-info/ps-tree.js +144 -0
  135. package/system-info/runtime.d.ts +14 -0
  136. package/system-info/runtime.js +80 -0
  137. package/typedefs.d.ts +1 -2
  138. package/typedefs.js +0 -1
  139. package/validators.d.ts +8 -1
  140. package/validators.js +8 -1
  141. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  142. package/autoscaling/autoscaled_pool.js.map +0 -1
  143. package/autoscaling/index.d.ts.map +0 -1
  144. package/autoscaling/index.js.map +0 -1
  145. package/autoscaling/snapshotter.d.ts.map +0 -1
  146. package/autoscaling/snapshotter.js.map +0 -1
  147. package/autoscaling/system_status.d.ts.map +0 -1
  148. package/autoscaling/system_status.js.map +0 -1
  149. package/configuration.d.ts.map +0 -1
  150. package/configuration.js.map +0 -1
  151. package/cookie_utils.d.ts.map +0 -1
  152. package/cookie_utils.js.map +0 -1
  153. package/crawlers/crawler_commons.d.ts.map +0 -1
  154. package/crawlers/crawler_commons.js.map +0 -1
  155. package/crawlers/crawler_extension.d.ts +0 -12
  156. package/crawlers/crawler_extension.d.ts.map +0 -1
  157. package/crawlers/crawler_extension.js +0 -14
  158. package/crawlers/crawler_extension.js.map +0 -1
  159. package/crawlers/crawler_utils.d.ts.map +0 -1
  160. package/crawlers/crawler_utils.js.map +0 -1
  161. package/crawlers/error_snapshotter.d.ts.map +0 -1
  162. package/crawlers/error_snapshotter.js.map +0 -1
  163. package/crawlers/error_tracker.d.ts.map +0 -1
  164. package/crawlers/error_tracker.js.map +0 -1
  165. package/crawlers/index.d.ts.map +0 -1
  166. package/crawlers/index.js.map +0 -1
  167. package/crawlers/statistics.d.ts.map +0 -1
  168. package/crawlers/statistics.js.map +0 -1
  169. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  170. package/enqueue_links/enqueue_links.js.map +0 -1
  171. package/enqueue_links/index.d.ts.map +0 -1
  172. package/enqueue_links/index.js.map +0 -1
  173. package/enqueue_links/shared.d.ts.map +0 -1
  174. package/enqueue_links/shared.js.map +0 -1
  175. package/errors.d.ts.map +0 -1
  176. package/errors.js.map +0 -1
  177. package/events/event_manager.d.ts.map +0 -1
  178. package/events/event_manager.js.map +0 -1
  179. package/events/index.d.ts.map +0 -1
  180. package/events/index.js.map +0 -1
  181. package/events/local_event_manager.d.ts.map +0 -1
  182. package/events/local_event_manager.js.map +0 -1
  183. package/http_clients/base-http-client.d.ts +0 -134
  184. package/http_clients/base-http-client.d.ts.map +0 -1
  185. package/http_clients/base-http-client.js +0 -33
  186. package/http_clients/base-http-client.js.map +0 -1
  187. package/http_clients/form-data-like.d.ts +0 -67
  188. package/http_clients/form-data-like.d.ts.map +0 -1
  189. package/http_clients/form-data-like.js +0 -5
  190. package/http_clients/form-data-like.js.map +0 -1
  191. package/http_clients/got-scraping-http-client.d.ts +0 -15
  192. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  193. package/http_clients/got-scraping-http-client.js +0 -69
  194. package/http_clients/got-scraping-http-client.js.map +0 -1
  195. package/http_clients/index.d.ts +0 -3
  196. package/http_clients/index.d.ts.map +0 -1
  197. package/http_clients/index.js +0 -3
  198. package/http_clients/index.js.map +0 -1
  199. package/index.d.ts.map +0 -1
  200. package/index.js.map +0 -1
  201. package/log.d.ts.map +0 -1
  202. package/log.js.map +0 -1
  203. package/proxy_configuration.d.ts.map +0 -1
  204. package/proxy_configuration.js.map +0 -1
  205. package/request.d.ts.map +0 -1
  206. package/request.js.map +0 -1
  207. package/router.d.ts.map +0 -1
  208. package/router.js.map +0 -1
  209. package/serialization.d.ts.map +0 -1
  210. package/serialization.js.map +0 -1
  211. package/session_pool/consts.d.ts.map +0 -1
  212. package/session_pool/consts.js.map +0 -1
  213. package/session_pool/errors.d.ts.map +0 -1
  214. package/session_pool/errors.js.map +0 -1
  215. package/session_pool/events.d.ts +0 -3
  216. package/session_pool/events.d.ts.map +0 -1
  217. package/session_pool/events.js +0 -3
  218. package/session_pool/events.js.map +0 -1
  219. package/session_pool/index.d.ts.map +0 -1
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts.map +0 -1
  222. package/session_pool/session.js.map +0 -1
  223. package/session_pool/session_pool.d.ts.map +0 -1
  224. package/session_pool/session_pool.js.map +0 -1
  225. package/storages/access_checking.d.ts.map +0 -1
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts.map +0 -1
  246. package/storages/sitemap_request_list.js.map +0 -1
  247. package/storages/storage_manager.d.ts +0 -58
  248. package/storages/storage_manager.d.ts.map +0 -1
  249. package/storages/storage_manager.js +0 -105
  250. package/storages/storage_manager.js.map +0 -1
  251. package/storages/utils.d.ts.map +0 -1
  252. package/storages/utils.js.map +0 -1
  253. package/tsconfig.build.tsbuildinfo +0 -1
  254. package/typedefs.d.ts.map +0 -1
  255. package/typedefs.js.map +0 -1
  256. package/validators.d.ts.map +0 -1
  257. package/validators.js.map +0 -1
@@ -0,0 +1,43 @@
1
+ import type { Configuration } from '../configuration.js';
2
+ import type { CrawleeLogger } from '../log.js';
3
+ import type { LoadSignal, LoadSnapshot } from './load_signal.js';
4
+ import type { SystemInfo } from './system_status.js';
5
+ export interface MemorySnapshot extends LoadSnapshot {
6
+ usedBytes?: number;
7
+ }
8
+ export interface MemoryLoadSignalOptions {
9
+ maxUsedMemoryRatio?: number;
10
+ overloadedRatio?: number;
11
+ snapshotHistoryMillis?: number;
12
+ config: Configuration;
13
+ log?: CrawleeLogger;
14
+ }
15
+ /**
16
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
17
+ * the used-to-available memory ratio exceeds a threshold.
18
+ */
19
+ export declare class MemoryLoadSignal implements LoadSignal {
20
+ readonly name = "memInfo";
21
+ readonly overloadedRatio: number;
22
+ private readonly store;
23
+ private readonly config;
24
+ private readonly events;
25
+ private readonly log;
26
+ private readonly maxUsedMemoryRatio;
27
+ private maxMemoryRatio;
28
+ private maxMemoryBytes;
29
+ private lastLoggedCriticalMemoryOverloadAt;
30
+ constructor(options: MemoryLoadSignalOptions);
31
+ start(): Promise<void>;
32
+ stop(): Promise<void>;
33
+ getSample(sampleDurationMillis?: number): MemorySnapshot[];
34
+ /**
35
+ * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
36
+ */
37
+ getMemorySnapshots(): MemorySnapshot[];
38
+ /** @internal */
39
+ _onSystemInfo(systemInfo: SystemInfo): void;
40
+ /** @internal */
41
+ _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
42
+ private _getTotalMemoryBytes;
43
+ }
@@ -0,0 +1,102 @@
1
+ import { serviceLocator } from '../service_locator.js';
2
+ import { getMemoryInfo } from '../system-info/memory-info.js';
3
+ import { isContainerized } from '../system-info/runtime.js';
4
+ import { SnapshotStore } from './load_signal.js';
5
+ const RESERVE_MEMORY_RATIO = 0.5;
6
+ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
7
+ /**
8
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
9
+ * the used-to-available memory ratio exceeds a threshold.
10
+ */
11
+ export class MemoryLoadSignal {
12
+ name = 'memInfo';
13
+ overloadedRatio;
14
+ store;
15
+ config;
16
+ events;
17
+ log;
18
+ maxUsedMemoryRatio;
19
+ maxMemoryRatio;
20
+ maxMemoryBytes;
21
+ lastLoggedCriticalMemoryOverloadAt = null;
22
+ constructor(options) {
23
+ this.store = new SnapshotStore(options.snapshotHistoryMillis);
24
+ this.config = options.config;
25
+ this.events = serviceLocator.getEventManager();
26
+ this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
27
+ this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
28
+ this.overloadedRatio = options.overloadedRatio ?? 0.2;
29
+ this._onSystemInfo = this._onSystemInfo.bind(this);
30
+ }
31
+ async start() {
32
+ const memoryMbytes = this.config.memoryMbytes ?? 0;
33
+ if (memoryMbytes > 0) {
34
+ this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
35
+ }
36
+ else {
37
+ this.maxMemoryRatio = this.config.availableMemoryRatio;
38
+ if (!this.maxMemoryRatio) {
39
+ throw new Error('availableMemoryRatio is not set in configuration.');
40
+ }
41
+ else {
42
+ this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
43
+ 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
44
+ }
45
+ // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
46
+ this.maxMemoryBytes = await this._getTotalMemoryBytes();
47
+ }
48
+ this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
49
+ }
50
+ async stop() {
51
+ this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
52
+ }
53
+ getSample(sampleDurationMillis) {
54
+ return this.store.getSample(sampleDurationMillis);
55
+ }
56
+ /**
57
+ * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
58
+ */
59
+ getMemorySnapshots() {
60
+ return this.store.getAll();
61
+ }
62
+ /** @internal */
63
+ _onSystemInfo(systemInfo) {
64
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
65
+ const { memCurrentBytes, memTotalBytes } = systemInfo;
66
+ let maxMemoryBytes = this.maxMemoryBytes;
67
+ if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
68
+ maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
69
+ }
70
+ const snapshot = {
71
+ createdAt,
72
+ isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
73
+ usedBytes: memCurrentBytes,
74
+ };
75
+ this.store.push(snapshot, createdAt);
76
+ this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
77
+ }
78
+ /** @internal */
79
+ _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
80
+ const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
81
+ const { memCurrentBytes } = systemInfo;
82
+ const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
83
+ if (this.lastLoggedCriticalMemoryOverloadAt &&
84
+ +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
85
+ return;
86
+ const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
87
+ const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
88
+ const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
89
+ const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
90
+ if (isCriticalOverload) {
91
+ const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
92
+ const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
93
+ this.log.warning('Memory is critically overloaded. ' +
94
+ `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
95
+ this.lastLoggedCriticalMemoryOverloadAt = createdAt;
96
+ }
97
+ }
98
+ async _getTotalMemoryBytes() {
99
+ const containerized = this.config.containerized ?? (await isContainerized());
100
+ return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
101
+ }
102
+ }
@@ -1,8 +1,11 @@
1
- import type { StorageClient } from '@crawlee/types';
2
- import type { Log } from '@apify/log';
3
- import type { BetterIntervalID } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
5
- import type { EventManager } from '../events/event_manager.js';
1
+ import type { StorageBackend } from '@crawlee/types';
2
+ import type { Configuration } from '../configuration.js';
3
+ import type { CrawleeLogger } from '../log.js';
4
+ import type { ClientSnapshot } from './client_load_signal.js';
5
+ import type { CpuSnapshot } from './cpu_load_signal.js';
6
+ import type { EventLoopSnapshot } from './event_loop_load_signal.js';
7
+ import type { LoadSignal } from './load_signal.js';
8
+ import type { MemorySnapshot } from './memory_load_signal.js';
6
9
  import type { SystemInfo } from './system_status.js';
7
10
  export interface SnapshotterOptions {
8
11
  /**
@@ -31,46 +34,22 @@ export interface SnapshotterOptions {
31
34
  /**
32
35
  * Defines the maximum number of new rate limit errors within
33
36
  * the given interval.
34
- * @default 1
37
+ * @default 3
35
38
  */
36
39
  maxClientErrors?: number;
37
40
  /**
38
41
  * Sets the interval in seconds for which a history of resource snapshots
39
42
  * will be kept. Increasing this to very high numbers will affect performance.
40
- * @default 60
43
+ * @default 30
41
44
  */
42
45
  snapshotHistorySecs?: number;
43
46
  /** @internal */
44
- log?: Log;
47
+ log?: CrawleeLogger;
45
48
  /** @internal */
46
- client?: StorageClient;
49
+ client?: StorageBackend;
47
50
  /** @internal */
48
51
  config?: Configuration;
49
52
  }
50
- interface MemorySnapshot {
51
- createdAt: Date;
52
- isOverloaded: boolean;
53
- usedBytes?: number;
54
- }
55
- interface CpuSnapshot {
56
- createdAt: Date;
57
- isOverloaded: boolean;
58
- usedRatio: number;
59
- ticks?: {
60
- idle: number;
61
- total: number;
62
- };
63
- }
64
- interface EventLoopSnapshot {
65
- createdAt: Date;
66
- isOverloaded: boolean;
67
- exceededMillis: number;
68
- }
69
- interface ClientSnapshot {
70
- createdAt: Date;
71
- isOverloaded: boolean;
72
- rateLimitErrorCount: number;
73
- }
74
53
  /**
75
54
  * Creates snapshots of system resources at given intervals and marks the resource
76
55
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -94,27 +73,26 @@ interface ClientSnapshot {
94
73
  *
95
74
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
96
75
  * typically received from the request queue, exceed the set limit within the set interval.
76
+ *
97
77
  * @category Scaling
98
78
  */
99
79
  export declare class Snapshotter {
100
- log: Log;
101
- client: StorageClient;
80
+ log: CrawleeLogger;
81
+ client: StorageBackend;
102
82
  config: Configuration;
103
- events: EventManager;
104
- eventLoopSnapshotIntervalMillis: number;
105
- clientSnapshotIntervalMillis: number;
106
- snapshotHistoryMillis: number;
107
- maxBlockedMillis: number;
108
- maxUsedMemoryRatio: number;
109
- maxClientErrors: number;
110
- maxMemoryBytes: number;
111
- cpuSnapshots: CpuSnapshot[];
112
- eventLoopSnapshots: EventLoopSnapshot[];
113
- memorySnapshots: MemorySnapshot[];
114
- clientSnapshots: ClientSnapshot[];
115
- eventLoopInterval: BetterIntervalID;
116
- clientInterval: BetterIntervalID;
117
- lastLoggedCriticalMemoryOverloadAt: Date | null;
83
+ private readonly memorySignal;
84
+ private readonly eventLoopSignal;
85
+ private readonly cpuSignal;
86
+ private readonly clientSignal;
87
+ /**
88
+ * Returns the four built-in signals as an array, so `SystemStatus` can
89
+ * iterate them alongside any custom `LoadSignal` instances.
90
+ */
91
+ getLoadSignals(): LoadSignal[];
92
+ get cpuSnapshots(): CpuSnapshot[];
93
+ get eventLoopSnapshots(): EventLoopSnapshot[];
94
+ get memorySnapshots(): MemorySnapshot[];
95
+ get clientSnapshots(): ClientSnapshot[];
118
96
  /**
119
97
  * @param [options] All `Snapshotter` configuration options.
120
98
  */
@@ -148,42 +126,27 @@ export declare class Snapshotter {
148
126
  */
149
127
  getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
150
128
  /**
151
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
152
- */
153
- protected _getSample<T extends {
154
- createdAt: Date;
155
- }>(snapshots: T[], sampleDurationMillis?: number): T[];
156
- /**
157
- * Creates a snapshot of current memory usage
158
- * using the Apify platform `systemInfo` event.
129
+ * @deprecated Kept for backward compatibility.
159
130
  */
160
131
  protected _snapshotMemory(systemInfo: SystemInfo): void;
161
132
  /**
162
- * Checks for critical memory overload and logs it to the console.
133
+ * @deprecated Kept for backward compatibility.
163
134
  */
164
135
  protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
165
136
  /**
166
- * Creates a snapshot of current event loop delay.
137
+ * @deprecated Kept for backward compatibility.
167
138
  */
168
139
  protected _snapshotEventLoop(intervalCallback: () => unknown): void;
169
140
  /**
170
- * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
141
+ * @deprecated Kept for backward compatibility.
171
142
  */
172
143
  protected _snapshotCpu(systemInfo: SystemInfo): void;
173
144
  /**
174
- * Creates a snapshot of current API state by checking for
175
- * rate limit errors. Only errors produced by a 2nd retry
176
- * of the API call are considered for snapshotting since
177
- * earlier errors may just be caused by a random spike in
178
- * number of requests and do not necessarily signify API
179
- * overloading.
145
+ * @deprecated Kept for backward compatibility.
180
146
  */
181
147
  protected _snapshotClient(intervalCallback: () => unknown): void;
182
148
  /**
183
- * Removes snapshots that are older than the snapshotHistorySecs option
184
- * from the array (destructively - in place).
149
+ * @deprecated Pruning is now handled by individual signals.
185
150
  */
186
- protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
151
+ protected _pruneSnapshots(_snapshots: any[], _now: Date): void;
187
152
  }
188
- export {};
189
- //# sourceMappingURL=snapshotter.d.ts.map
@@ -1,11 +1,9 @@
1
- import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
2
1
  import ow from 'ow';
3
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
5
- import { log as defaultLog } from '../log.js';
6
- const RESERVE_MEMORY_RATIO = 0.5;
7
- const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
8
- const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
2
+ import { serviceLocator } from '../service_locator.js';
3
+ import { createClientLoadSignal } from './client_load_signal.js';
4
+ import { createCpuLoadSignal } from './cpu_load_signal.js';
5
+ import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
6
+ import { MemoryLoadSignal } from './memory_load_signal.js';
9
7
  /**
10
8
  * Creates snapshots of system resources at given intervals and marks the resource
11
9
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -29,27 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
29
27
  *
30
28
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
31
29
  * typically received from the request queue, exceed the set limit within the set interval.
30
+ *
32
31
  * @category Scaling
33
32
  */
34
33
  export class Snapshotter {
35
34
  log;
36
35
  client;
37
36
  config;
38
- events;
39
- eventLoopSnapshotIntervalMillis;
40
- clientSnapshotIntervalMillis;
41
- snapshotHistoryMillis;
42
- maxBlockedMillis;
43
- maxUsedMemoryRatio;
44
- maxClientErrors;
45
- maxMemoryBytes;
46
- cpuSnapshots = [];
47
- eventLoopSnapshots = [];
48
- memorySnapshots = [];
49
- clientSnapshots = [];
50
- eventLoopInterval = null;
51
- clientInterval = null;
52
- lastLoggedCriticalMemoryOverloadAt = null;
37
+ memorySignal;
38
+ eventLoopSignal;
39
+ cpuSignal;
40
+ clientSignal;
41
+ /**
42
+ * Returns the four built-in signals as an array, so `SystemStatus` can
43
+ * iterate them alongside any custom `LoadSignal` instances.
44
+ */
45
+ getLoadSignals() {
46
+ return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
47
+ }
48
+ // Legacy public properties kept for backward compat (tests read these directly)
49
+ get cpuSnapshots() {
50
+ return this.cpuSignal.store.getAll();
51
+ }
52
+ get eventLoopSnapshots() {
53
+ return this.eventLoopSignal.store.getAll();
54
+ }
55
+ get memorySnapshots() {
56
+ return this.memorySignal.getMemorySnapshots();
57
+ }
58
+ get clientSnapshots() {
59
+ return this.clientSignal.store.getAll();
60
+ }
53
61
  /**
54
62
  * @param [options] All `Snapshotter` configuration options.
55
63
  */
@@ -65,58 +73,50 @@ export class Snapshotter {
65
73
  client: ow.optional.object,
66
74
  config: ow.optional.object,
67
75
  }));
68
- const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = defaultLog, config = Configuration.getGlobalConfig(), client = config.getStorageClient(), } = options;
76
+ const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
69
77
  this.log = log.child({ prefix: 'Snapshotter' });
70
78
  this.client = client;
71
79
  this.config = config;
72
- this.events = this.config.getEventManager();
73
- this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
74
- this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
75
- this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
76
- this.maxBlockedMillis = maxBlockedMillis;
77
- this.maxUsedMemoryRatio = maxUsedMemoryRatio;
78
- this.maxClientErrors = maxClientErrors;
79
- // We need to pre-bind those functions to be able to successfully remove listeners.
80
- this._snapshotCpu = this._snapshotCpu.bind(this);
81
- this._snapshotMemory = this._snapshotMemory.bind(this);
80
+ const snapshotHistoryMillis = snapshotHistorySecs * 1000;
81
+ this.memorySignal = new MemoryLoadSignal({
82
+ maxUsedMemoryRatio,
83
+ snapshotHistoryMillis,
84
+ config: this.config,
85
+ log: this.log,
86
+ });
87
+ this.eventLoopSignal = createEventLoopLoadSignal({
88
+ eventLoopSnapshotIntervalSecs,
89
+ maxBlockedMillis,
90
+ snapshotHistoryMillis,
91
+ });
92
+ this.cpuSignal = createCpuLoadSignal({
93
+ snapshotHistoryMillis,
94
+ config: this.config,
95
+ });
96
+ this.clientSignal = createClientLoadSignal({
97
+ client: this.client,
98
+ clientSnapshotIntervalSecs,
99
+ maxClientErrors,
100
+ snapshotHistoryMillis,
101
+ });
82
102
  }
83
103
  /**
84
104
  * Starts capturing snapshots at configured intervals.
85
105
  */
86
106
  async start() {
87
- const memoryMbytes = this.config.get('memoryMbytes', 0);
88
- if (memoryMbytes > 0) {
89
- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
90
- }
91
- else {
92
- let totalBytes;
93
- if (this.config.get('systemInfoV2')) {
94
- const containerized = this.config.get('containerized', await isContainerized());
95
- const memInfo = await getMemoryInfoV2(containerized);
96
- totalBytes = memInfo.totalBytes;
97
- }
98
- else {
99
- const memInfo = await getMemoryInfo();
100
- totalBytes = memInfo.totalBytes;
101
- }
102
- this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
103
- this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
104
- 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
105
- }
106
- // Start snapshotting.
107
- this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
108
- this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
109
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
110
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
107
+ await this.memorySignal.start();
108
+ await this.eventLoopSignal.start();
109
+ await this.cpuSignal.start();
110
+ await this.clientSignal.start();
111
111
  }
112
112
  /**
113
113
  * Stops all resource capturing.
114
114
  */
115
115
  async stop() {
116
- betterClearInterval(this.eventLoopInterval);
117
- betterClearInterval(this.clientInterval);
118
- this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
119
- this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
116
+ await this.memorySignal.stop();
117
+ await this.eventLoopSignal.stop();
118
+ await this.cpuSignal.stop();
119
+ await this.clientSignal.stop();
120
120
  // Allow microtask queue to unwind before stop returns.
121
121
  await new Promise((resolve) => {
122
122
  setImmediate(resolve);
@@ -127,166 +127,63 @@ export class Snapshotter {
127
127
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
128
128
  */
129
129
  getMemorySample(sampleDurationMillis) {
130
- return this._getSample(this.memorySnapshots, sampleDurationMillis);
130
+ return this.memorySignal.getSample(sampleDurationMillis);
131
131
  }
132
132
  /**
133
133
  * Returns a sample of latest event loop snapshots, with the size of the sample defined
134
134
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
135
135
  */
136
136
  getEventLoopSample(sampleDurationMillis) {
137
- return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
137
+ return this.eventLoopSignal.getSample(sampleDurationMillis);
138
138
  }
139
139
  /**
140
140
  * Returns a sample of latest CPU snapshots, with the size of the sample defined
141
141
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
142
142
  */
143
143
  getCpuSample(sampleDurationMillis) {
144
- return this._getSample(this.cpuSnapshots, sampleDurationMillis);
144
+ return this.cpuSignal.getSample(sampleDurationMillis);
145
145
  }
146
146
  /**
147
147
  * Returns a sample of latest Client snapshots, with the size of the sample defined
148
148
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
149
149
  */
150
150
  getClientSample(sampleDurationMillis) {
151
- return this._getSample(this.clientSnapshots, sampleDurationMillis);
151
+ return this.clientSignal.getSample(sampleDurationMillis);
152
152
  }
153
153
  /**
154
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
155
- */
156
- _getSample(snapshots, sampleDurationMillis) {
157
- if (!sampleDurationMillis)
158
- return snapshots;
159
- const sample = [];
160
- let idx = snapshots.length;
161
- if (!idx)
162
- return sample;
163
- const latestTime = snapshots[idx - 1].createdAt;
164
- while (idx--) {
165
- const snapshot = snapshots[idx];
166
- if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
167
- sample.unshift(snapshot);
168
- }
169
- else {
170
- break;
171
- }
172
- }
173
- return sample;
174
- }
175
- /**
176
- * Creates a snapshot of current memory usage
177
- * using the Apify platform `systemInfo` event.
154
+ * @deprecated Kept for backward compatibility.
178
155
  */
179
156
  _snapshotMemory(systemInfo) {
180
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
181
- this._pruneSnapshots(this.memorySnapshots, createdAt);
182
- const { memCurrentBytes } = systemInfo;
183
- const snapshot = {
184
- createdAt,
185
- isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
186
- usedBytes: memCurrentBytes,
187
- };
188
- this.memorySnapshots.push(snapshot);
189
- this._memoryOverloadWarning(systemInfo);
157
+ this.memorySignal._onSystemInfo(systemInfo);
190
158
  }
191
159
  /**
192
- * Checks for critical memory overload and logs it to the console.
160
+ * @deprecated Kept for backward compatibility.
193
161
  */
194
162
  _memoryOverloadWarning(systemInfo) {
195
- const { memCurrentBytes } = systemInfo;
196
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
197
- if (this.lastLoggedCriticalMemoryOverloadAt &&
198
- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
199
- return;
200
- const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
201
- const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
202
- const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
203
- const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
204
- if (isCriticalOverload) {
205
- const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
206
- const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
207
- this.log.warning('Memory is critically overloaded. ' +
208
- `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
209
- this.lastLoggedCriticalMemoryOverloadAt = createdAt;
210
- }
163
+ this.memorySignal._memoryOverloadWarning(systemInfo);
211
164
  }
212
165
  /**
213
- * Creates a snapshot of current event loop delay.
166
+ * @deprecated Kept for backward compatibility.
214
167
  */
215
168
  _snapshotEventLoop(intervalCallback) {
216
- const now = new Date();
217
- this._pruneSnapshots(this.eventLoopSnapshots, now);
218
- const snapshot = {
219
- createdAt: now,
220
- isOverloaded: false,
221
- exceededMillis: 0,
222
- };
223
- const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
224
- if (previousSnapshot) {
225
- const { createdAt } = previousSnapshot;
226
- const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
227
- if (delta > this.maxBlockedMillis)
228
- snapshot.isOverloaded = true;
229
- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
230
- }
231
- this.eventLoopSnapshots.push(snapshot);
232
- intervalCallback();
169
+ this.eventLoopSignal.handle(intervalCallback);
233
170
  }
234
171
  /**
235
- * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
172
+ * @deprecated Kept for backward compatibility.
236
173
  */
237
174
  _snapshotCpu(systemInfo) {
238
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
239
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
240
- this._pruneSnapshots(this.cpuSnapshots, createdAt);
241
- this.cpuSnapshots.push({
242
- createdAt,
243
- isOverloaded: isCpuOverloaded,
244
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
245
- });
175
+ this.cpuSignal.handle(systemInfo);
246
176
  }
247
177
  /**
248
- * Creates a snapshot of current API state by checking for
249
- * rate limit errors. Only errors produced by a 2nd retry
250
- * of the API call are considered for snapshotting since
251
- * earlier errors may just be caused by a random spike in
252
- * number of requests and do not necessarily signify API
253
- * overloading.
178
+ * @deprecated Kept for backward compatibility.
254
179
  */
255
180
  _snapshotClient(intervalCallback) {
256
- const now = new Date();
257
- this._pruneSnapshots(this.clientSnapshots, now);
258
- const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
259
- const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
260
- // Handle empty snapshots array
261
- const snapshot = {
262
- createdAt: now,
263
- isOverloaded: false,
264
- rateLimitErrorCount: currentErrCount,
265
- };
266
- const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
267
- if (previousSnapshot) {
268
- const { rateLimitErrorCount } = previousSnapshot;
269
- const delta = currentErrCount - rateLimitErrorCount;
270
- if (delta > this.maxClientErrors)
271
- snapshot.isOverloaded = true;
272
- }
273
- this.clientSnapshots.push(snapshot);
274
- intervalCallback();
181
+ this.clientSignal.handle(intervalCallback);
275
182
  }
276
183
  /**
277
- * Removes snapshots that are older than the snapshotHistorySecs option
278
- * from the array (destructively - in place).
184
+ * @deprecated Pruning is now handled by individual signals.
279
185
  */
280
- _pruneSnapshots(snapshots, now) {
281
- let oldCount = 0;
282
- for (let i = 0; i < snapshots.length; i++) {
283
- const { createdAt } = snapshots[i];
284
- if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
285
- oldCount++;
286
- else
287
- break;
288
- }
289
- snapshots.splice(0, oldCount);
186
+ _pruneSnapshots(_snapshots, _now) {
187
+ // no-op signals prune themselves
290
188
  }
291
189
  }
292
- //# sourceMappingURL=snapshotter.js.map