@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +14 -15
  3. package/autoscaling/autoscaled_pool.js +37 -36
  4. package/autoscaling/client_load_signal.d.ts +25 -0
  5. package/autoscaling/client_load_signal.js +36 -0
  6. package/autoscaling/cpu_load_signal.d.ts +28 -0
  7. package/autoscaling/cpu_load_signal.js +24 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  9. package/autoscaling/event_loop_load_signal.js +35 -0
  10. package/autoscaling/index.d.ts +5 -1
  11. package/autoscaling/index.js +5 -1
  12. package/autoscaling/load_signal.d.ts +99 -0
  13. package/autoscaling/load_signal.js +135 -0
  14. package/autoscaling/memory_load_signal.d.ts +43 -0
  15. package/autoscaling/memory_load_signal.js +102 -0
  16. package/autoscaling/snapshotter.d.ts +29 -91
  17. package/autoscaling/snapshotter.js +65 -204
  18. package/autoscaling/system_status.d.ts +22 -37
  19. package/autoscaling/system_status.js +49 -87
  20. package/configuration.d.ts +88 -228
  21. package/configuration.js +162 -224
  22. package/cookie_utils.d.ts +4 -3
  23. package/cookie_utils.js +20 -13
  24. package/crawlers/context_pipeline.d.ts +70 -0
  25. package/crawlers/context_pipeline.js +122 -0
  26. package/crawlers/crawler_commons.d.ts +20 -30
  27. package/crawlers/crawler_commons.js +12 -21
  28. package/crawlers/crawler_utils.d.ts +2 -3
  29. package/crawlers/crawler_utils.js +1 -2
  30. package/crawlers/error_snapshotter.d.ts +3 -3
  31. package/crawlers/error_snapshotter.js +2 -3
  32. package/crawlers/error_tracker.d.ts +2 -2
  33. package/crawlers/error_tracker.js +0 -1
  34. package/crawlers/index.d.ts +1 -2
  35. package/crawlers/index.js +1 -2
  36. package/crawlers/internals/types.d.ts +7 -0
  37. package/crawlers/internals/types.js +1 -0
  38. package/crawlers/statistics.d.ts +28 -23
  39. package/crawlers/statistics.js +38 -34
  40. package/enqueue_links/enqueue_links.d.ts +44 -22
  41. package/enqueue_links/enqueue_links.js +51 -30
  42. package/enqueue_links/index.d.ts +0 -1
  43. package/enqueue_links/index.js +0 -1
  44. package/enqueue_links/shared.d.ts +25 -9
  45. package/enqueue_links/shared.js +69 -38
  46. package/errors.d.ts +53 -4
  47. package/errors.js +70 -5
  48. package/events/event_manager.d.ts +34 -8
  49. package/events/event_manager.js +8 -10
  50. package/events/index.d.ts +0 -1
  51. package/events/index.js +0 -1
  52. package/events/local_event_manager.d.ts +15 -5
  53. package/events/local_event_manager.js +37 -40
  54. package/index.d.ts +4 -3
  55. package/index.js +3 -2
  56. package/log.d.ts +82 -3
  57. package/log.js +102 -1
  58. package/memory-storage/consts.d.ts +4 -0
  59. package/memory-storage/consts.js +4 -0
  60. package/memory-storage/index.d.ts +1 -0
  61. package/memory-storage/index.js +1 -0
  62. package/memory-storage/memory-storage.d.ts +46 -0
  63. package/memory-storage/memory-storage.js +136 -0
  64. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  65. package/memory-storage/resource-clients/common/base-client.js +6 -0
  66. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  67. package/memory-storage/resource-clients/dataset.js +113 -0
  68. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  69. package/memory-storage/resource-clients/key-value-store.js +203 -0
  70. package/memory-storage/resource-clients/request-queue.d.ts +96 -0
  71. package/memory-storage/resource-clients/request-queue.js +421 -0
  72. package/memory-storage/utils.d.ts +17 -0
  73. package/memory-storage/utils.js +42 -0
  74. package/package.json +12 -10
  75. package/proxy_configuration.d.ts +29 -152
  76. package/proxy_configuration.js +21 -173
  77. package/recoverable_state.d.ts +120 -0
  78. package/recoverable_state.js +141 -0
  79. package/request.d.ts +84 -13
  80. package/request.js +107 -28
  81. package/router.d.ts +118 -16
  82. package/router.js +124 -30
  83. package/serialization.d.ts +0 -1
  84. package/serialization.js +1 -2
  85. package/service_locator.d.ts +156 -0
  86. package/service_locator.js +238 -0
  87. package/session_pool/consts.d.ts +1 -2
  88. package/session_pool/consts.js +1 -2
  89. package/session_pool/errors.d.ts +0 -1
  90. package/session_pool/errors.js +0 -1
  91. package/session_pool/fingerprint.d.ts +9 -0
  92. package/session_pool/fingerprint.js +30 -0
  93. package/session_pool/index.d.ts +0 -2
  94. package/session_pool/index.js +0 -2
  95. package/session_pool/session.d.ts +37 -75
  96. package/session_pool/session.js +49 -102
  97. package/session_pool/session_pool.d.ts +85 -90
  98. package/session_pool/session_pool.js +131 -120
  99. package/storages/access_checking.d.ts +0 -1
  100. package/storages/access_checking.js +5 -2
  101. package/storages/dataset.d.ts +101 -51
  102. package/storages/dataset.js +168 -126
  103. package/storages/index.d.ts +8 -7
  104. package/storages/index.js +6 -7
  105. package/storages/key_value_store.d.ts +165 -36
  106. package/storages/key_value_store.js +269 -122
  107. package/storages/key_value_store_codec.d.ts +32 -0
  108. package/storages/key_value_store_codec.js +113 -0
  109. package/storages/request_dedup_cache.d.ts +23 -0
  110. package/storages/request_dedup_cache.js +48 -0
  111. package/storages/request_list.d.ts +53 -97
  112. package/storages/request_list.js +99 -75
  113. package/storages/request_loader.d.ts +96 -0
  114. package/storages/request_loader.js +1 -0
  115. package/storages/request_manager.d.ts +33 -0
  116. package/storages/request_manager.js +1 -0
  117. package/storages/request_manager_tandem.d.ts +106 -0
  118. package/storages/request_manager_tandem.js +197 -0
  119. package/storages/request_queue.d.ts +290 -47
  120. package/storages/request_queue.js +628 -215
  121. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
  122. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
  123. package/storages/storage_instance_manager.d.ts +91 -0
  124. package/storages/storage_instance_manager.js +258 -0
  125. package/storages/storage_stats.d.ts +48 -0
  126. package/storages/storage_stats.js +29 -0
  127. package/storages/utils.d.ts +51 -6
  128. package/storages/utils.js +60 -9
  129. package/system-info/cpu-info.d.ts +67 -0
  130. package/system-info/cpu-info.js +216 -0
  131. package/system-info/memory-info.d.ts +31 -0
  132. package/system-info/memory-info.js +115 -0
  133. package/system-info/ps-tree.d.ts +17 -0
  134. package/system-info/ps-tree.js +144 -0
  135. package/system-info/runtime.d.ts +14 -0
  136. package/system-info/runtime.js +80 -0
  137. package/typedefs.d.ts +1 -2
  138. package/typedefs.js +0 -1
  139. package/validators.d.ts +8 -1
  140. package/validators.js +10 -3
  141. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  142. package/autoscaling/autoscaled_pool.js.map +0 -1
  143. package/autoscaling/index.d.ts.map +0 -1
  144. package/autoscaling/index.js.map +0 -1
  145. package/autoscaling/snapshotter.d.ts.map +0 -1
  146. package/autoscaling/snapshotter.js.map +0 -1
  147. package/autoscaling/system_status.d.ts.map +0 -1
  148. package/autoscaling/system_status.js.map +0 -1
  149. package/configuration.d.ts.map +0 -1
  150. package/configuration.js.map +0 -1
  151. package/cookie_utils.d.ts.map +0 -1
  152. package/cookie_utils.js.map +0 -1
  153. package/crawlers/crawler_commons.d.ts.map +0 -1
  154. package/crawlers/crawler_commons.js.map +0 -1
  155. package/crawlers/crawler_extension.d.ts +0 -12
  156. package/crawlers/crawler_extension.d.ts.map +0 -1
  157. package/crawlers/crawler_extension.js +0 -14
  158. package/crawlers/crawler_extension.js.map +0 -1
  159. package/crawlers/crawler_utils.d.ts.map +0 -1
  160. package/crawlers/crawler_utils.js.map +0 -1
  161. package/crawlers/error_snapshotter.d.ts.map +0 -1
  162. package/crawlers/error_snapshotter.js.map +0 -1
  163. package/crawlers/error_tracker.d.ts.map +0 -1
  164. package/crawlers/error_tracker.js.map +0 -1
  165. package/crawlers/index.d.ts.map +0 -1
  166. package/crawlers/index.js.map +0 -1
  167. package/crawlers/statistics.d.ts.map +0 -1
  168. package/crawlers/statistics.js.map +0 -1
  169. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  170. package/enqueue_links/enqueue_links.js.map +0 -1
  171. package/enqueue_links/index.d.ts.map +0 -1
  172. package/enqueue_links/index.js.map +0 -1
  173. package/enqueue_links/shared.d.ts.map +0 -1
  174. package/enqueue_links/shared.js.map +0 -1
  175. package/errors.d.ts.map +0 -1
  176. package/errors.js.map +0 -1
  177. package/events/event_manager.d.ts.map +0 -1
  178. package/events/event_manager.js.map +0 -1
  179. package/events/index.d.ts.map +0 -1
  180. package/events/index.js.map +0 -1
  181. package/events/local_event_manager.d.ts.map +0 -1
  182. package/events/local_event_manager.js.map +0 -1
  183. package/http_clients/base-http-client.d.ts +0 -134
  184. package/http_clients/base-http-client.d.ts.map +0 -1
  185. package/http_clients/base-http-client.js +0 -33
  186. package/http_clients/base-http-client.js.map +0 -1
  187. package/http_clients/form-data-like.d.ts +0 -67
  188. package/http_clients/form-data-like.d.ts.map +0 -1
  189. package/http_clients/form-data-like.js +0 -5
  190. package/http_clients/form-data-like.js.map +0 -1
  191. package/http_clients/got-scraping-http-client.d.ts +0 -15
  192. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  193. package/http_clients/got-scraping-http-client.js +0 -69
  194. package/http_clients/got-scraping-http-client.js.map +0 -1
  195. package/http_clients/index.d.ts +0 -3
  196. package/http_clients/index.d.ts.map +0 -1
  197. package/http_clients/index.js +0 -3
  198. package/http_clients/index.js.map +0 -1
  199. package/index.d.ts.map +0 -1
  200. package/index.js.map +0 -1
  201. package/log.d.ts.map +0 -1
  202. package/log.js.map +0 -1
  203. package/proxy_configuration.d.ts.map +0 -1
  204. package/proxy_configuration.js.map +0 -1
  205. package/request.d.ts.map +0 -1
  206. package/request.js.map +0 -1
  207. package/router.d.ts.map +0 -1
  208. package/router.js.map +0 -1
  209. package/serialization.d.ts.map +0 -1
  210. package/serialization.js.map +0 -1
  211. package/session_pool/consts.d.ts.map +0 -1
  212. package/session_pool/consts.js.map +0 -1
  213. package/session_pool/errors.d.ts.map +0 -1
  214. package/session_pool/errors.js.map +0 -1
  215. package/session_pool/events.d.ts +0 -3
  216. package/session_pool/events.d.ts.map +0 -1
  217. package/session_pool/events.js +0 -3
  218. package/session_pool/events.js.map +0 -1
  219. package/session_pool/index.d.ts.map +0 -1
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts.map +0 -1
  222. package/session_pool/session.js.map +0 -1
  223. package/session_pool/session_pool.d.ts.map +0 -1
  224. package/session_pool/session_pool.js.map +0 -1
  225. package/storages/access_checking.d.ts.map +0 -1
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts.map +0 -1
  246. package/storages/sitemap_request_list.js.map +0 -1
  247. package/storages/storage_manager.d.ts +0 -58
  248. package/storages/storage_manager.d.ts.map +0 -1
  249. package/storages/storage_manager.js +0 -105
  250. package/storages/storage_manager.js.map +0 -1
  251. package/storages/utils.d.ts.map +0 -1
  252. package/storages/utils.js.map +0 -1
  253. package/tsconfig.build.tsbuildinfo +0 -1
  254. package/typedefs.d.ts.map +0 -1
  255. package/typedefs.js.map +0 -1
  256. package/validators.d.ts.map +0 -1
  257. package/validators.js.map +0 -1
@@ -1,11 +1,9 @@
1
- import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
2
1
  import ow from 'ow';
3
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
5
- import { log as defaultLog } from '../log.js';
6
- const RESERVE_MEMORY_RATIO = 0.5;
7
- const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
8
- const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
2
+ import { serviceLocator } from '../service_locator.js';
3
+ import { createClientLoadSignal } from './client_load_signal.js';
4
+ import { createCpuLoadSignal } from './cpu_load_signal.js';
5
+ import { createEventLoopLoadSignal } from './event_loop_load_signal.js';
6
+ import { MemoryLoadSignal } from './memory_load_signal.js';
9
7
  /**
10
8
  * Creates snapshots of system resources at given intervals and marks the resource
11
9
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -29,27 +27,37 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
29
27
  *
30
28
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
31
29
  * typically received from the request queue, exceed the set limit within the set interval.
30
+ *
32
31
  * @category Scaling
33
32
  */
34
33
  export class Snapshotter {
35
34
  log;
36
35
  client;
37
36
  config;
38
- events;
39
- eventLoopSnapshotIntervalMillis;
40
- clientSnapshotIntervalMillis;
41
- snapshotHistoryMillis;
42
- maxBlockedMillis;
43
- maxUsedMemoryRatio;
44
- maxClientErrors;
45
- maxMemoryBytes;
46
- cpuSnapshots = [];
47
- eventLoopSnapshots = [];
48
- memorySnapshots = [];
49
- clientSnapshots = [];
50
- eventLoopInterval = null;
51
- clientInterval = null;
52
- lastLoggedCriticalMemoryOverloadAt = null;
37
+ memorySignal;
38
+ eventLoopSignal;
39
+ cpuSignal;
40
+ clientSignal;
41
+ /**
42
+ * Returns the four built-in signals as an array, so `SystemStatus` can
43
+ * iterate them alongside any custom `LoadSignal` instances.
44
+ */
45
+ getLoadSignals() {
46
+ return [this.memorySignal, this.eventLoopSignal, this.cpuSignal, this.clientSignal];
47
+ }
48
+ // Legacy public properties kept for backward compat (tests read these directly)
49
+ get cpuSnapshots() {
50
+ return this.cpuSignal.store.getAll();
51
+ }
52
+ get eventLoopSnapshots() {
53
+ return this.eventLoopSignal.store.getAll();
54
+ }
55
+ get memorySnapshots() {
56
+ return this.memorySignal.getMemorySnapshots();
57
+ }
58
+ get clientSnapshots() {
59
+ return this.clientSignal.store.getAll();
60
+ }
53
61
  /**
54
62
  * @param [options] All `Snapshotter` configuration options.
55
63
  */
@@ -65,58 +73,50 @@ export class Snapshotter {
65
73
  client: ow.optional.object,
66
74
  config: ow.optional.object,
67
75
  }));
68
- const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = defaultLog, config = Configuration.getGlobalConfig(), client = config.getStorageClient(), } = options;
76
+ const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), config = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
69
77
  this.log = log.child({ prefix: 'Snapshotter' });
70
78
  this.client = client;
71
79
  this.config = config;
72
- this.events = this.config.getEventManager();
73
- this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
74
- this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
75
- this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
76
- this.maxBlockedMillis = maxBlockedMillis;
77
- this.maxUsedMemoryRatio = maxUsedMemoryRatio;
78
- this.maxClientErrors = maxClientErrors;
79
- // We need to pre-bind those functions to be able to successfully remove listeners.
80
- this._snapshotCpu = this._snapshotCpu.bind(this);
81
- this._snapshotMemory = this._snapshotMemory.bind(this);
80
+ const snapshotHistoryMillis = snapshotHistorySecs * 1000;
81
+ this.memorySignal = new MemoryLoadSignal({
82
+ maxUsedMemoryRatio,
83
+ snapshotHistoryMillis,
84
+ config: this.config,
85
+ log: this.log,
86
+ });
87
+ this.eventLoopSignal = createEventLoopLoadSignal({
88
+ eventLoopSnapshotIntervalSecs,
89
+ maxBlockedMillis,
90
+ snapshotHistoryMillis,
91
+ });
92
+ this.cpuSignal = createCpuLoadSignal({
93
+ snapshotHistoryMillis,
94
+ config: this.config,
95
+ });
96
+ this.clientSignal = createClientLoadSignal({
97
+ client: this.client,
98
+ clientSnapshotIntervalSecs,
99
+ maxClientErrors,
100
+ snapshotHistoryMillis,
101
+ });
82
102
  }
83
103
  /**
84
104
  * Starts capturing snapshots at configured intervals.
85
105
  */
86
106
  async start() {
87
- const memoryMbytes = this.config.get('memoryMbytes', 0);
88
- if (memoryMbytes > 0) {
89
- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
90
- }
91
- else {
92
- let totalBytes;
93
- if (this.config.get('systemInfoV2')) {
94
- const containerized = this.config.get('containerized', await isContainerized());
95
- const memInfo = await getMemoryInfoV2(containerized);
96
- totalBytes = memInfo.totalBytes;
97
- }
98
- else {
99
- const memInfo = await getMemoryInfo();
100
- totalBytes = memInfo.totalBytes;
101
- }
102
- this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
103
- this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
104
- 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
105
- }
106
- // Start snapshotting.
107
- this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
108
- this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
109
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
110
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
107
+ await this.memorySignal.start();
108
+ await this.eventLoopSignal.start();
109
+ await this.cpuSignal.start();
110
+ await this.clientSignal.start();
111
111
  }
112
112
  /**
113
113
  * Stops all resource capturing.
114
114
  */
115
115
  async stop() {
116
- betterClearInterval(this.eventLoopInterval);
117
- betterClearInterval(this.clientInterval);
118
- this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
119
- this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
116
+ await this.memorySignal.stop();
117
+ await this.eventLoopSignal.stop();
118
+ await this.cpuSignal.stop();
119
+ await this.clientSignal.stop();
120
120
  // Allow microtask queue to unwind before stop returns.
121
121
  await new Promise((resolve) => {
122
122
  setImmediate(resolve);
@@ -127,166 +127,27 @@ export class Snapshotter {
127
127
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
128
128
  */
129
129
  getMemorySample(sampleDurationMillis) {
130
- return this._getSample(this.memorySnapshots, sampleDurationMillis);
130
+ return this.memorySignal.getSample(sampleDurationMillis);
131
131
  }
132
132
  /**
133
133
  * Returns a sample of latest event loop snapshots, with the size of the sample defined
134
134
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
135
135
  */
136
136
  getEventLoopSample(sampleDurationMillis) {
137
- return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
137
+ return this.eventLoopSignal.getSample(sampleDurationMillis);
138
138
  }
139
139
  /**
140
140
  * Returns a sample of latest CPU snapshots, with the size of the sample defined
141
141
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
142
142
  */
143
143
  getCpuSample(sampleDurationMillis) {
144
- return this._getSample(this.cpuSnapshots, sampleDurationMillis);
144
+ return this.cpuSignal.getSample(sampleDurationMillis);
145
145
  }
146
146
  /**
147
147
  * Returns a sample of latest Client snapshots, with the size of the sample defined
148
148
  * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
149
149
  */
150
150
  getClientSample(sampleDurationMillis) {
151
- return this._getSample(this.clientSnapshots, sampleDurationMillis);
152
- }
153
- /**
154
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
155
- */
156
- _getSample(snapshots, sampleDurationMillis) {
157
- if (!sampleDurationMillis)
158
- return snapshots;
159
- const sample = [];
160
- let idx = snapshots.length;
161
- if (!idx)
162
- return sample;
163
- const latestTime = snapshots[idx - 1].createdAt;
164
- while (idx--) {
165
- const snapshot = snapshots[idx];
166
- if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
167
- sample.unshift(snapshot);
168
- }
169
- else {
170
- break;
171
- }
172
- }
173
- return sample;
174
- }
175
- /**
176
- * Creates a snapshot of current memory usage
177
- * using the Apify platform `systemInfo` event.
178
- */
179
- _snapshotMemory(systemInfo) {
180
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
181
- this._pruneSnapshots(this.memorySnapshots, createdAt);
182
- const { memCurrentBytes } = systemInfo;
183
- const snapshot = {
184
- createdAt,
185
- isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
186
- usedBytes: memCurrentBytes,
187
- };
188
- this.memorySnapshots.push(snapshot);
189
- this._memoryOverloadWarning(systemInfo);
190
- }
191
- /**
192
- * Checks for critical memory overload and logs it to the console.
193
- */
194
- _memoryOverloadWarning(systemInfo) {
195
- const { memCurrentBytes } = systemInfo;
196
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
197
- if (this.lastLoggedCriticalMemoryOverloadAt &&
198
- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
199
- return;
200
- const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
201
- const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
202
- const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
203
- const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
204
- if (isCriticalOverload) {
205
- const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
206
- const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
207
- this.log.warning('Memory is critically overloaded. ' +
208
- `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
209
- this.lastLoggedCriticalMemoryOverloadAt = createdAt;
210
- }
211
- }
212
- /**
213
- * Creates a snapshot of current event loop delay.
214
- */
215
- _snapshotEventLoop(intervalCallback) {
216
- const now = new Date();
217
- this._pruneSnapshots(this.eventLoopSnapshots, now);
218
- const snapshot = {
219
- createdAt: now,
220
- isOverloaded: false,
221
- exceededMillis: 0,
222
- };
223
- const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
224
- if (previousSnapshot) {
225
- const { createdAt } = previousSnapshot;
226
- const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
227
- if (delta > this.maxBlockedMillis)
228
- snapshot.isOverloaded = true;
229
- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
230
- }
231
- this.eventLoopSnapshots.push(snapshot);
232
- intervalCallback();
233
- }
234
- /**
235
- * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
236
- */
237
- _snapshotCpu(systemInfo) {
238
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
239
- const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
240
- this._pruneSnapshots(this.cpuSnapshots, createdAt);
241
- this.cpuSnapshots.push({
242
- createdAt,
243
- isOverloaded: isCpuOverloaded,
244
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
245
- });
246
- }
247
- /**
248
- * Creates a snapshot of current API state by checking for
249
- * rate limit errors. Only errors produced by a 2nd retry
250
- * of the API call are considered for snapshotting since
251
- * earlier errors may just be caused by a random spike in
252
- * number of requests and do not necessarily signify API
253
- * overloading.
254
- */
255
- _snapshotClient(intervalCallback) {
256
- const now = new Date();
257
- this._pruneSnapshots(this.clientSnapshots, now);
258
- const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
259
- const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
260
- // Handle empty snapshots array
261
- const snapshot = {
262
- createdAt: now,
263
- isOverloaded: false,
264
- rateLimitErrorCount: currentErrCount,
265
- };
266
- const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
267
- if (previousSnapshot) {
268
- const { rateLimitErrorCount } = previousSnapshot;
269
- const delta = currentErrCount - rateLimitErrorCount;
270
- if (delta > this.maxClientErrors)
271
- snapshot.isOverloaded = true;
272
- }
273
- this.clientSnapshots.push(snapshot);
274
- intervalCallback();
275
- }
276
- /**
277
- * Removes snapshots that are older than the snapshotHistorySecs option
278
- * from the array (destructively - in place).
279
- */
280
- _pruneSnapshots(snapshots, now) {
281
- let oldCount = 0;
282
- for (let i = 0; i < snapshots.length; i++) {
283
- const { createdAt } = snapshots[i];
284
- if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
285
- oldCount++;
286
- else
287
- break;
288
- }
289
- snapshots.splice(0, oldCount);
151
+ return this.clientSignal.getSample(sampleDurationMillis);
290
152
  }
291
153
  }
292
- //# sourceMappingURL=snapshotter.js.map
@@ -1,4 +1,4 @@
1
- import type { Configuration } from '../configuration.js';
1
+ import type { LoadSignal } from './load_signal.js';
2
2
  import { Snapshotter } from './snapshotter.js';
3
3
  /**
4
4
  * Represents the current status of the system.
@@ -10,6 +10,7 @@ export interface SystemInfo {
10
10
  eventLoopInfo: ClientInfo;
11
11
  cpuInfo: ClientInfo;
12
12
  clientInfo: ClientInfo;
13
+ memTotalBytes?: number;
13
14
  memCurrentBytes?: number;
14
15
  /**
15
16
  * Platform only property
@@ -26,6 +27,11 @@ export interface SystemInfo {
26
27
  * @internal
27
28
  */
28
29
  createdAt?: Date;
30
+ /**
31
+ * Status of additional load signals beyond the built-in four.
32
+ * Keys are `LoadSignal.name` values, values are overload info.
33
+ */
34
+ loadSignalInfo?: Record<string, ClientInfo>;
29
35
  }
30
36
  export interface SystemStatusOptions {
31
37
  /**
@@ -61,8 +67,13 @@ export interface SystemStatusOptions {
61
67
  * The `Snapshotter` instance to be queried for `SystemStatus`.
62
68
  */
63
69
  snapshotter?: Snapshotter;
64
- /** @internal */
65
- config?: Configuration;
70
+ /**
71
+ * Additional load signals to include in the system status evaluation.
72
+ * These are evaluated alongside the built-in memory, CPU, event loop,
73
+ * and client signals. If any signal reports overload, the system is
74
+ * considered overloaded.
75
+ */
76
+ loadSignals?: LoadSignal[];
66
77
  }
67
78
  export interface ClientInfo {
68
79
  isOverloaded: boolean;
@@ -105,11 +116,14 @@ export interface FinalStatistics {
105
116
  */
106
117
  export declare class SystemStatus {
107
118
  private readonly currentHistoryMillis;
108
- private readonly maxMemoryOverloadedRatio;
109
- private readonly maxEventLoopOverloadedRatio;
110
- private readonly maxCpuOverloadedRatio;
111
- private readonly maxClientOverloadedRatio;
112
119
  private readonly snapshotter;
120
+ private readonly signals;
121
+ /**
122
+ * Per-signal ratio overrides. The built-in four get their overrides from
123
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
124
+ * `overloadedRatio`.
125
+ */
126
+ private ratioOverrides;
113
127
  constructor(options?: SystemStatusOptions);
114
128
  /**
115
129
  * Returns an {@link SystemInfo} object with the following structure:
@@ -148,34 +162,5 @@ export declare class SystemStatus {
148
162
  /**
149
163
  * Returns a system status object.
150
164
  */
151
- protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
152
- /**
153
- * Returns an object with an isOverloaded property set to true
154
- * if the memory has been overloaded in the last sampleDurationMillis.
155
- */
156
- protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
157
- /**
158
- * Returns an object with an isOverloaded property set to true
159
- * if the event loop has been overloaded in the last sampleDurationMillis.
160
- */
161
- protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
162
- /**
163
- * Returns an object with an isOverloaded property set to true
164
- * if the CPU has been overloaded in the last sampleDurationMillis.
165
- */
166
- protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
167
- /**
168
- * Returns an object with an isOverloaded property set to true
169
- * if the client has been overloaded in the last sampleDurationMillis.
170
- */
171
- protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
172
- /**
173
- * Returns an object with sample information and an isOverloaded property
174
- * set to true if at least the ratio of snapshots in the sample are overloaded.
175
- */
176
- protected _isSampleOverloaded<T extends {
177
- createdAt: Date;
178
- isOverloaded: boolean;
179
- }>(sample: T[], ratio: number): ClientInfo;
165
+ private isSystemIdle;
180
166
  }
181
- //# sourceMappingURL=system_status.d.ts.map
@@ -1,6 +1,8 @@
1
- import { weightedAvg } from '@crawlee/utils';
2
1
  import ow from 'ow';
2
+ import { evaluateLoadSignalSample } from './load_signal.js';
3
3
  import { Snapshotter } from './snapshotter.js';
4
+ /** The four built-in signal names that map to typed `SystemInfo` fields. */
5
+ const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
4
6
  /**
5
7
  * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
6
8
  * It only exposes two functions {@link SystemStatus.getCurrentStatus}
@@ -25,11 +27,14 @@ import { Snapshotter } from './snapshotter.js';
25
27
  */
26
28
  export class SystemStatus {
27
29
  currentHistoryMillis;
28
- maxMemoryOverloadedRatio;
29
- maxEventLoopOverloadedRatio;
30
- maxCpuOverloadedRatio;
31
- maxClientOverloadedRatio;
32
30
  snapshotter;
31
+ signals;
32
+ /**
33
+ * Per-signal ratio overrides. The built-in four get their overrides from
34
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
35
+ * `overloadedRatio`.
36
+ */
37
+ ratioOverrides;
33
38
  constructor(options = {}) {
34
39
  ow(options, ow.object.exactShape({
35
40
  currentHistorySecs: ow.optional.number,
@@ -38,15 +43,20 @@ export class SystemStatus {
38
43
  maxCpuOverloadedRatio: ow.optional.number,
39
44
  maxClientOverloadedRatio: ow.optional.number,
40
45
  snapshotter: ow.optional.object,
41
- config: ow.optional.object,
46
+ loadSignals: ow.optional.array,
42
47
  }));
43
- const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
48
+ const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
44
49
  this.currentHistoryMillis = currentHistorySecs * 1000;
45
- this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
46
- this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
47
- this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
48
- this.maxClientOverloadedRatio = maxClientOverloadedRatio;
49
- this.snapshotter = snapshotter || new Snapshotter({ config });
50
+ this.snapshotter = snapshotter || new Snapshotter();
51
+ // Built-in signals from the snapshotter + any custom signals
52
+ this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
53
+ // Allow legacy options to override the built-in signal ratios
54
+ this.ratioOverrides = {
55
+ memInfo: maxMemoryOverloadedRatio,
56
+ eventLoopInfo: maxEventLoopOverloadedRatio,
57
+ cpuInfo: maxCpuOverloadedRatio,
58
+ clientInfo: maxClientOverloadedRatio,
59
+ };
50
60
  }
51
61
  /**
52
62
  * Returns an {@link SystemInfo} object with the following structure:
@@ -65,7 +75,7 @@ export class SystemStatus {
65
75
  * and `true` otherwise.
66
76
  */
67
77
  getCurrentStatus() {
68
- return this._isSystemIdle(this.currentHistoryMillis);
78
+ return this.isSystemIdle(this.currentHistoryMillis);
69
79
  }
70
80
  /**
71
81
  * Returns an {@link SystemInfo} object with the following structure:
@@ -84,86 +94,38 @@ export class SystemStatus {
84
94
  * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
85
95
  */
86
96
  getHistoricalStatus() {
87
- return this._isSystemIdle();
97
+ return this.isSystemIdle();
88
98
  }
89
99
  /**
90
100
  * Returns a system status object.
91
101
  */
92
- _isSystemIdle(sampleDurationMillis) {
93
- const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
94
- const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
95
- const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
96
- const clientInfo = this._isClientOverloaded(sampleDurationMillis);
97
- return {
98
- isSystemIdle: !memInfo.isOverloaded &&
99
- !eventLoopInfo.isOverloaded &&
100
- !cpuInfo.isOverloaded &&
101
- !clientInfo.isOverloaded,
102
- memInfo,
103
- eventLoopInfo,
104
- cpuInfo,
105
- clientInfo,
102
+ isSystemIdle(sampleDurationMillis) {
103
+ const result = {
104
+ isSystemIdle: true,
105
+ memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
106
+ eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
107
+ cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
108
+ clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
106
109
  };
107
- }
108
- /**
109
- * Returns an object with an isOverloaded property set to true
110
- * if the memory has been overloaded in the last sampleDurationMillis.
111
- */
112
- _isMemoryOverloaded(sampleDurationMillis) {
113
- const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
114
- return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
115
- }
116
- /**
117
- * Returns an object with an isOverloaded property set to true
118
- * if the event loop has been overloaded in the last sampleDurationMillis.
119
- */
120
- _isEventLoopOverloaded(sampleDurationMillis) {
121
- const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
122
- return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
123
- }
124
- /**
125
- * Returns an object with an isOverloaded property set to true
126
- * if the CPU has been overloaded in the last sampleDurationMillis.
127
- */
128
- _isCpuOverloaded(sampleDurationMillis) {
129
- const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
130
- return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
131
- }
132
- /**
133
- * Returns an object with an isOverloaded property set to true
134
- * if the client has been overloaded in the last sampleDurationMillis.
135
- */
136
- _isClientOverloaded(sampleDurationMillis) {
137
- const sample = this.snapshotter.getClientSample(sampleDurationMillis);
138
- return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
139
- }
140
- /**
141
- * Returns an object with sample information and an isOverloaded property
142
- * set to true if at least the ratio of snapshots in the sample are overloaded.
143
- */
144
- _isSampleOverloaded(sample, ratio) {
145
- if (sample.length === 0) {
146
- return {
147
- isOverloaded: false,
148
- limitRatio: ratio,
149
- actualRatio: 0,
150
- };
110
+ let loadSignalInfo;
111
+ for (const signal of this.signals) {
112
+ const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
113
+ const sample = signal.getSample(sampleDurationMillis);
114
+ const info = evaluateLoadSignalSample(sample, ratio);
115
+ if (info.isOverloaded) {
116
+ result.isSystemIdle = false;
117
+ }
118
+ if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
119
+ result[signal.name] = info;
120
+ }
121
+ else {
122
+ loadSignalInfo ??= {};
123
+ loadSignalInfo[signal.name] = info;
124
+ }
151
125
  }
152
- const weights = [];
153
- const values = [];
154
- for (let i = 1; i < sample.length; i++) {
155
- const previous = sample[i - 1];
156
- const current = sample[i];
157
- const weight = +current.createdAt - +previous.createdAt;
158
- weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
159
- values.push(+current.isOverloaded);
126
+ if (loadSignalInfo) {
127
+ result.loadSignalInfo = loadSignalInfo;
160
128
  }
161
- const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
162
- return {
163
- isOverloaded: wAvg > ratio,
164
- limitRatio: ratio,
165
- actualRatio: Math.round(wAvg * 1000) / 1000,
166
- };
129
+ return result;
167
130
  }
168
131
  }
169
- //# sourceMappingURL=system_status.js.map