@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +5 -6
  3. package/autoscaling/autoscaled_pool.js +12 -11
  4. package/autoscaling/client_load_signal.d.ts +25 -0
  5. package/autoscaling/client_load_signal.js +36 -0
  6. package/autoscaling/cpu_load_signal.d.ts +28 -0
  7. package/autoscaling/cpu_load_signal.js +24 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  9. package/autoscaling/event_loop_load_signal.js +35 -0
  10. package/autoscaling/index.d.ts +5 -1
  11. package/autoscaling/index.js +5 -1
  12. package/autoscaling/load_signal.d.ts +99 -0
  13. package/autoscaling/load_signal.js +135 -0
  14. package/autoscaling/memory_load_signal.d.ts +43 -0
  15. package/autoscaling/memory_load_signal.js +102 -0
  16. package/autoscaling/snapshotter.d.ts +35 -72
  17. package/autoscaling/snapshotter.js +78 -181
  18. package/autoscaling/system_status.d.ts +21 -36
  19. package/autoscaling/system_status.js +46 -84
  20. package/configuration.d.ts +88 -228
  21. package/configuration.js +162 -224
  22. package/cookie_utils.d.ts +4 -3
  23. package/cookie_utils.js +20 -13
  24. package/crawlers/context_pipeline.d.ts +70 -0
  25. package/crawlers/context_pipeline.js +122 -0
  26. package/crawlers/crawler_commons.d.ts +20 -30
  27. package/crawlers/crawler_commons.js +12 -21
  28. package/crawlers/crawler_utils.d.ts +2 -3
  29. package/crawlers/crawler_utils.js +1 -2
  30. package/crawlers/error_snapshotter.d.ts +3 -3
  31. package/crawlers/error_snapshotter.js +2 -3
  32. package/crawlers/error_tracker.d.ts +2 -2
  33. package/crawlers/error_tracker.js +0 -1
  34. package/crawlers/index.d.ts +1 -2
  35. package/crawlers/index.js +1 -2
  36. package/crawlers/internals/types.d.ts +7 -0
  37. package/crawlers/internals/types.js +1 -0
  38. package/crawlers/statistics.d.ts +22 -17
  39. package/crawlers/statistics.js +30 -26
  40. package/enqueue_links/enqueue_links.d.ts +44 -22
  41. package/enqueue_links/enqueue_links.js +51 -30
  42. package/enqueue_links/index.d.ts +0 -1
  43. package/enqueue_links/index.js +0 -1
  44. package/enqueue_links/shared.d.ts +25 -9
  45. package/enqueue_links/shared.js +69 -38
  46. package/errors.d.ts +53 -4
  47. package/errors.js +70 -5
  48. package/events/event_manager.d.ts +34 -8
  49. package/events/event_manager.js +8 -10
  50. package/events/index.d.ts +0 -1
  51. package/events/index.js +0 -1
  52. package/events/local_event_manager.d.ts +15 -5
  53. package/events/local_event_manager.js +37 -40
  54. package/index.d.ts +4 -3
  55. package/index.js +3 -2
  56. package/log.d.ts +82 -3
  57. package/log.js +102 -1
  58. package/memory-storage/consts.d.ts +4 -0
  59. package/memory-storage/consts.js +4 -0
  60. package/memory-storage/index.d.ts +1 -0
  61. package/memory-storage/index.js +1 -0
  62. package/memory-storage/memory-storage.d.ts +46 -0
  63. package/memory-storage/memory-storage.js +136 -0
  64. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  65. package/memory-storage/resource-clients/common/base-client.js +6 -0
  66. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  67. package/memory-storage/resource-clients/dataset.js +113 -0
  68. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  69. package/memory-storage/resource-clients/key-value-store.js +203 -0
  70. package/memory-storage/resource-clients/request-queue.d.ts +96 -0
  71. package/memory-storage/resource-clients/request-queue.js +421 -0
  72. package/memory-storage/utils.d.ts +17 -0
  73. package/memory-storage/utils.js +42 -0
  74. package/package.json +12 -10
  75. package/proxy_configuration.d.ts +14 -149
  76. package/proxy_configuration.js +19 -168
  77. package/recoverable_state.d.ts +120 -0
  78. package/recoverable_state.js +141 -0
  79. package/request.d.ts +84 -13
  80. package/request.js +107 -28
  81. package/router.d.ts +117 -15
  82. package/router.js +124 -30
  83. package/serialization.d.ts +0 -1
  84. package/serialization.js +1 -2
  85. package/service_locator.d.ts +156 -0
  86. package/service_locator.js +238 -0
  87. package/session_pool/consts.d.ts +1 -2
  88. package/session_pool/consts.js +1 -2
  89. package/session_pool/errors.d.ts +0 -1
  90. package/session_pool/errors.js +0 -1
  91. package/session_pool/fingerprint.d.ts +9 -0
  92. package/session_pool/fingerprint.js +30 -0
  93. package/session_pool/index.d.ts +0 -2
  94. package/session_pool/index.js +0 -2
  95. package/session_pool/session.d.ts +35 -73
  96. package/session_pool/session.js +46 -99
  97. package/session_pool/session_pool.d.ts +71 -72
  98. package/session_pool/session_pool.js +111 -100
  99. package/storages/access_checking.d.ts +0 -1
  100. package/storages/access_checking.js +5 -2
  101. package/storages/dataset.d.ts +101 -51
  102. package/storages/dataset.js +168 -126
  103. package/storages/index.d.ts +8 -7
  104. package/storages/index.js +6 -7
  105. package/storages/key_value_store.d.ts +165 -36
  106. package/storages/key_value_store.js +269 -122
  107. package/storages/key_value_store_codec.d.ts +32 -0
  108. package/storages/key_value_store_codec.js +113 -0
  109. package/storages/request_dedup_cache.d.ts +23 -0
  110. package/storages/request_dedup_cache.js +48 -0
  111. package/storages/request_list.d.ts +38 -82
  112. package/storages/request_list.js +65 -41
  113. package/storages/request_loader.d.ts +96 -0
  114. package/storages/request_loader.js +1 -0
  115. package/storages/request_manager.d.ts +33 -0
  116. package/storages/request_manager.js +1 -0
  117. package/storages/request_manager_tandem.d.ts +106 -0
  118. package/storages/request_manager_tandem.js +197 -0
  119. package/storages/request_queue.d.ts +303 -46
  120. package/storages/request_queue.js +629 -214
  121. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
  122. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
  123. package/storages/storage_instance_manager.d.ts +91 -0
  124. package/storages/storage_instance_manager.js +258 -0
  125. package/storages/storage_stats.d.ts +48 -0
  126. package/storages/storage_stats.js +29 -0
  127. package/storages/utils.d.ts +51 -6
  128. package/storages/utils.js +60 -9
  129. package/system-info/cpu-info.d.ts +67 -0
  130. package/system-info/cpu-info.js +216 -0
  131. package/system-info/memory-info.d.ts +31 -0
  132. package/system-info/memory-info.js +115 -0
  133. package/system-info/ps-tree.d.ts +17 -0
  134. package/system-info/ps-tree.js +144 -0
  135. package/system-info/runtime.d.ts +14 -0
  136. package/system-info/runtime.js +80 -0
  137. package/typedefs.d.ts +1 -2
  138. package/typedefs.js +0 -1
  139. package/validators.d.ts +8 -1
  140. package/validators.js +8 -1
  141. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  142. package/autoscaling/autoscaled_pool.js.map +0 -1
  143. package/autoscaling/index.d.ts.map +0 -1
  144. package/autoscaling/index.js.map +0 -1
  145. package/autoscaling/snapshotter.d.ts.map +0 -1
  146. package/autoscaling/snapshotter.js.map +0 -1
  147. package/autoscaling/system_status.d.ts.map +0 -1
  148. package/autoscaling/system_status.js.map +0 -1
  149. package/configuration.d.ts.map +0 -1
  150. package/configuration.js.map +0 -1
  151. package/cookie_utils.d.ts.map +0 -1
  152. package/cookie_utils.js.map +0 -1
  153. package/crawlers/crawler_commons.d.ts.map +0 -1
  154. package/crawlers/crawler_commons.js.map +0 -1
  155. package/crawlers/crawler_extension.d.ts +0 -12
  156. package/crawlers/crawler_extension.d.ts.map +0 -1
  157. package/crawlers/crawler_extension.js +0 -14
  158. package/crawlers/crawler_extension.js.map +0 -1
  159. package/crawlers/crawler_utils.d.ts.map +0 -1
  160. package/crawlers/crawler_utils.js.map +0 -1
  161. package/crawlers/error_snapshotter.d.ts.map +0 -1
  162. package/crawlers/error_snapshotter.js.map +0 -1
  163. package/crawlers/error_tracker.d.ts.map +0 -1
  164. package/crawlers/error_tracker.js.map +0 -1
  165. package/crawlers/index.d.ts.map +0 -1
  166. package/crawlers/index.js.map +0 -1
  167. package/crawlers/statistics.d.ts.map +0 -1
  168. package/crawlers/statistics.js.map +0 -1
  169. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  170. package/enqueue_links/enqueue_links.js.map +0 -1
  171. package/enqueue_links/index.d.ts.map +0 -1
  172. package/enqueue_links/index.js.map +0 -1
  173. package/enqueue_links/shared.d.ts.map +0 -1
  174. package/enqueue_links/shared.js.map +0 -1
  175. package/errors.d.ts.map +0 -1
  176. package/errors.js.map +0 -1
  177. package/events/event_manager.d.ts.map +0 -1
  178. package/events/event_manager.js.map +0 -1
  179. package/events/index.d.ts.map +0 -1
  180. package/events/index.js.map +0 -1
  181. package/events/local_event_manager.d.ts.map +0 -1
  182. package/events/local_event_manager.js.map +0 -1
  183. package/http_clients/base-http-client.d.ts +0 -134
  184. package/http_clients/base-http-client.d.ts.map +0 -1
  185. package/http_clients/base-http-client.js +0 -33
  186. package/http_clients/base-http-client.js.map +0 -1
  187. package/http_clients/form-data-like.d.ts +0 -67
  188. package/http_clients/form-data-like.d.ts.map +0 -1
  189. package/http_clients/form-data-like.js +0 -5
  190. package/http_clients/form-data-like.js.map +0 -1
  191. package/http_clients/got-scraping-http-client.d.ts +0 -15
  192. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  193. package/http_clients/got-scraping-http-client.js +0 -69
  194. package/http_clients/got-scraping-http-client.js.map +0 -1
  195. package/http_clients/index.d.ts +0 -3
  196. package/http_clients/index.d.ts.map +0 -1
  197. package/http_clients/index.js +0 -3
  198. package/http_clients/index.js.map +0 -1
  199. package/index.d.ts.map +0 -1
  200. package/index.js.map +0 -1
  201. package/log.d.ts.map +0 -1
  202. package/log.js.map +0 -1
  203. package/proxy_configuration.d.ts.map +0 -1
  204. package/proxy_configuration.js.map +0 -1
  205. package/request.d.ts.map +0 -1
  206. package/request.js.map +0 -1
  207. package/router.d.ts.map +0 -1
  208. package/router.js.map +0 -1
  209. package/serialization.d.ts.map +0 -1
  210. package/serialization.js.map +0 -1
  211. package/session_pool/consts.d.ts.map +0 -1
  212. package/session_pool/consts.js.map +0 -1
  213. package/session_pool/errors.d.ts.map +0 -1
  214. package/session_pool/errors.js.map +0 -1
  215. package/session_pool/events.d.ts +0 -3
  216. package/session_pool/events.d.ts.map +0 -1
  217. package/session_pool/events.js +0 -3
  218. package/session_pool/events.js.map +0 -1
  219. package/session_pool/index.d.ts.map +0 -1
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts.map +0 -1
  222. package/session_pool/session.js.map +0 -1
  223. package/session_pool/session_pool.d.ts.map +0 -1
  224. package/session_pool/session_pool.js.map +0 -1
  225. package/storages/access_checking.d.ts.map +0 -1
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts.map +0 -1
  246. package/storages/sitemap_request_list.js.map +0 -1
  247. package/storages/storage_manager.d.ts +0 -58
  248. package/storages/storage_manager.d.ts.map +0 -1
  249. package/storages/storage_manager.js +0 -105
  250. package/storages/storage_manager.js.map +0 -1
  251. package/storages/utils.d.ts.map +0 -1
  252. package/storages/utils.js.map +0 -1
  253. package/tsconfig.build.tsbuildinfo +0 -1
  254. package/typedefs.d.ts.map +0 -1
  255. package/typedefs.js.map +0 -1
  256. package/validators.d.ts.map +0 -1
  257. package/validators.js.map +0 -1
@@ -1,4 +1,4 @@
1
- import type { Configuration } from '../configuration.js';
1
+ import type { LoadSignal } from './load_signal.js';
2
2
  import { Snapshotter } from './snapshotter.js';
3
3
  /**
4
4
  * Represents the current status of the system.
@@ -10,6 +10,7 @@ export interface SystemInfo {
10
10
  eventLoopInfo: ClientInfo;
11
11
  cpuInfo: ClientInfo;
12
12
  clientInfo: ClientInfo;
13
+ memTotalBytes?: number;
13
14
  memCurrentBytes?: number;
14
15
  /**
15
16
  * Platform only property
@@ -26,6 +27,11 @@ export interface SystemInfo {
26
27
  * @internal
27
28
  */
28
29
  createdAt?: Date;
30
+ /**
31
+ * Status of additional load signals beyond the built-in four.
32
+ * Keys are `LoadSignal.name` values, values are overload info.
33
+ */
34
+ loadSignalInfo?: Record<string, ClientInfo>;
29
35
  }
30
36
  export interface SystemStatusOptions {
31
37
  /**
@@ -61,8 +67,13 @@ export interface SystemStatusOptions {
61
67
  * The `Snapshotter` instance to be queried for `SystemStatus`.
62
68
  */
63
69
  snapshotter?: Snapshotter;
64
- /** @internal */
65
- config?: Configuration;
70
+ /**
71
+ * Additional load signals to include in the system status evaluation.
72
+ * These are evaluated alongside the built-in memory, CPU, event loop,
73
+ * and client signals. If any signal reports overload, the system is
74
+ * considered overloaded.
75
+ */
76
+ loadSignals?: LoadSignal[];
66
77
  }
67
78
  export interface ClientInfo {
68
79
  isOverloaded: boolean;
@@ -105,11 +116,14 @@ export interface FinalStatistics {
105
116
  */
106
117
  export declare class SystemStatus {
107
118
  private readonly currentHistoryMillis;
108
- private readonly maxMemoryOverloadedRatio;
109
- private readonly maxEventLoopOverloadedRatio;
110
- private readonly maxCpuOverloadedRatio;
111
- private readonly maxClientOverloadedRatio;
112
119
  private readonly snapshotter;
120
+ private readonly signals;
121
+ /**
122
+ * Per-signal ratio overrides. The built-in four get their overrides from
123
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
124
+ * `overloadedRatio`.
125
+ */
126
+ private ratioOverrides;
113
127
  constructor(options?: SystemStatusOptions);
114
128
  /**
115
129
  * Returns an {@link SystemInfo} object with the following structure:
@@ -149,33 +163,4 @@ export declare class SystemStatus {
149
163
  * Returns a system status object.
150
164
  */
151
165
  protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
152
- /**
153
- * Returns an object with an isOverloaded property set to true
154
- * if the memory has been overloaded in the last sampleDurationMillis.
155
- */
156
- protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
157
- /**
158
- * Returns an object with an isOverloaded property set to true
159
- * if the event loop has been overloaded in the last sampleDurationMillis.
160
- */
161
- protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
162
- /**
163
- * Returns an object with an isOverloaded property set to true
164
- * if the CPU has been overloaded in the last sampleDurationMillis.
165
- */
166
- protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
167
- /**
168
- * Returns an object with an isOverloaded property set to true
169
- * if the client has been overloaded in the last sampleDurationMillis.
170
- */
171
- protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
172
- /**
173
- * Returns an object with sample information and an isOverloaded property
174
- * set to true if at least the ratio of snapshots in the sample are overloaded.
175
- */
176
- protected _isSampleOverloaded<T extends {
177
- createdAt: Date;
178
- isOverloaded: boolean;
179
- }>(sample: T[], ratio: number): ClientInfo;
180
166
  }
181
- //# sourceMappingURL=system_status.d.ts.map
@@ -1,6 +1,8 @@
1
- import { weightedAvg } from '@crawlee/utils';
2
1
  import ow from 'ow';
2
+ import { evaluateLoadSignalSample } from './load_signal.js';
3
3
  import { Snapshotter } from './snapshotter.js';
4
+ /** The four built-in signal names that map to typed `SystemInfo` fields. */
5
+ const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
4
6
  /**
5
7
  * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
6
8
  * It only exposes two functions {@link SystemStatus.getCurrentStatus}
@@ -25,11 +27,14 @@ import { Snapshotter } from './snapshotter.js';
25
27
  */
26
28
  export class SystemStatus {
27
29
  currentHistoryMillis;
28
- maxMemoryOverloadedRatio;
29
- maxEventLoopOverloadedRatio;
30
- maxCpuOverloadedRatio;
31
- maxClientOverloadedRatio;
32
30
  snapshotter;
31
+ signals;
32
+ /**
33
+ * Per-signal ratio overrides. The built-in four get their overrides from
34
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
35
+ * `overloadedRatio`.
36
+ */
37
+ ratioOverrides;
33
38
  constructor(options = {}) {
34
39
  ow(options, ow.object.exactShape({
35
40
  currentHistorySecs: ow.optional.number,
@@ -38,15 +43,20 @@ export class SystemStatus {
38
43
  maxCpuOverloadedRatio: ow.optional.number,
39
44
  maxClientOverloadedRatio: ow.optional.number,
40
45
  snapshotter: ow.optional.object,
41
- config: ow.optional.object,
46
+ loadSignals: ow.optional.array,
42
47
  }));
43
- const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
48
+ const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
44
49
  this.currentHistoryMillis = currentHistorySecs * 1000;
45
- this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
46
- this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
47
- this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
48
- this.maxClientOverloadedRatio = maxClientOverloadedRatio;
49
- this.snapshotter = snapshotter || new Snapshotter({ config });
50
+ this.snapshotter = snapshotter || new Snapshotter();
51
+ // Built-in signals from the snapshotter + any custom signals
52
+ this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
53
+ // Allow legacy options to override the built-in signal ratios
54
+ this.ratioOverrides = {
55
+ memInfo: maxMemoryOverloadedRatio,
56
+ eventLoopInfo: maxEventLoopOverloadedRatio,
57
+ cpuInfo: maxCpuOverloadedRatio,
58
+ clientInfo: maxClientOverloadedRatio,
59
+ };
50
60
  }
51
61
  /**
52
62
  * Returns an {@link SystemInfo} object with the following structure:
@@ -90,80 +100,32 @@ export class SystemStatus {
90
100
  * Returns a system status object.
91
101
  */
92
102
  _isSystemIdle(sampleDurationMillis) {
93
- const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
94
- const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
95
- const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
96
- const clientInfo = this._isClientOverloaded(sampleDurationMillis);
97
- return {
98
- isSystemIdle: !memInfo.isOverloaded &&
99
- !eventLoopInfo.isOverloaded &&
100
- !cpuInfo.isOverloaded &&
101
- !clientInfo.isOverloaded,
102
- memInfo,
103
- eventLoopInfo,
104
- cpuInfo,
105
- clientInfo,
103
+ const result = {
104
+ isSystemIdle: true,
105
+ memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
106
+ eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
107
+ cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
108
+ clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
106
109
  };
107
- }
108
- /**
109
- * Returns an object with an isOverloaded property set to true
110
- * if the memory has been overloaded in the last sampleDurationMillis.
111
- */
112
- _isMemoryOverloaded(sampleDurationMillis) {
113
- const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
114
- return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
115
- }
116
- /**
117
- * Returns an object with an isOverloaded property set to true
118
- * if the event loop has been overloaded in the last sampleDurationMillis.
119
- */
120
- _isEventLoopOverloaded(sampleDurationMillis) {
121
- const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
122
- return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
123
- }
124
- /**
125
- * Returns an object with an isOverloaded property set to true
126
- * if the CPU has been overloaded in the last sampleDurationMillis.
127
- */
128
- _isCpuOverloaded(sampleDurationMillis) {
129
- const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
130
- return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
131
- }
132
- /**
133
- * Returns an object with an isOverloaded property set to true
134
- * if the client has been overloaded in the last sampleDurationMillis.
135
- */
136
- _isClientOverloaded(sampleDurationMillis) {
137
- const sample = this.snapshotter.getClientSample(sampleDurationMillis);
138
- return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
139
- }
140
- /**
141
- * Returns an object with sample information and an isOverloaded property
142
- * set to true if at least the ratio of snapshots in the sample are overloaded.
143
- */
144
- _isSampleOverloaded(sample, ratio) {
145
- if (sample.length === 0) {
146
- return {
147
- isOverloaded: false,
148
- limitRatio: ratio,
149
- actualRatio: 0,
150
- };
110
+ let loadSignalInfo;
111
+ for (const signal of this.signals) {
112
+ const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
113
+ const sample = signal.getSample(sampleDurationMillis);
114
+ const info = evaluateLoadSignalSample(sample, ratio);
115
+ if (info.isOverloaded) {
116
+ result.isSystemIdle = false;
117
+ }
118
+ if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
119
+ result[signal.name] = info;
120
+ }
121
+ else {
122
+ loadSignalInfo ??= {};
123
+ loadSignalInfo[signal.name] = info;
124
+ }
151
125
  }
152
- const weights = [];
153
- const values = [];
154
- for (let i = 1; i < sample.length; i++) {
155
- const previous = sample[i - 1];
156
- const current = sample[i];
157
- const weight = +current.createdAt - +previous.createdAt;
158
- weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
159
- values.push(+current.isOverloaded);
126
+ if (loadSignalInfo) {
127
+ result.loadSignalInfo = loadSignalInfo;
160
128
  }
161
- const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
162
- return {
163
- isOverloaded: wAvg > ratio,
164
- limitRatio: ratio,
165
- actualRatio: Math.round(wAvg * 1000) / 1000,
166
- };
129
+ return result;
167
130
  }
168
131
  }
169
- //# sourceMappingURL=system_status.js.map
@@ -1,153 +1,59 @@
1
- import { AsyncLocalStorage } from 'node:async_hooks';
2
- import type { MemoryStorageOptions } from '@crawlee/memory-storage';
3
- import { MemoryStorage } from '@crawlee/memory-storage';
4
- import type { Dictionary, StorageClient } from '@crawlee/types';
5
- import { LogLevel } from '@apify/log';
6
- import { type EventManager } from './events/event_manager.js';
7
- import type { StorageManager } from './storages/storage_manager.js';
8
- import { type Constructor } from './typedefs.js';
9
- export interface ConfigurationOptions {
10
- /**
11
- * Defines storage client to be used.
12
- * @default {@link MemoryStorage}
13
- */
14
- storageClient?: StorageClient;
15
- /**
16
- * Defines the Event Manager to be used.
17
- * @default {@link EventManager}
18
- */
19
- eventManager?: EventManager;
20
- /**
21
- * Could be used to adjust the storage client behavior
22
- * e.g. {@link MemoryStorageOptions} could be used to adjust the {@link MemoryStorage} behavior.
23
- */
24
- storageClientOptions?: Dictionary;
25
- /**
26
- * Default dataset id.
27
- *
28
- * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
29
- * @default 'default'
30
- */
31
- defaultDatasetId?: string;
32
- /**
33
- * Defines whether to purge the default storage folders before starting the crawler run.
34
- *
35
- * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
36
- * @default true
37
- */
38
- purgeOnStart?: boolean;
39
- /**
40
- * Default key-value store id.
41
- *
42
- * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
43
- * @default 'default'
44
- */
45
- defaultKeyValueStoreId?: string;
46
- /**
47
- * Default request queue id.
48
- *
49
- * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
50
- * @default 'default'
51
- */
52
- defaultRequestQueueId?: string;
53
- /**
54
- * Sets the ratio, defining the maximum CPU usage.
55
- * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
56
- * @default 0.95
57
- */
58
- maxUsedCpuRatio?: number;
59
- /**
60
- * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
61
- * When the memory usage is more than the provided ratio, the memory is considered overloaded.
62
- *
63
- * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
64
- * @default 0.25
65
- */
66
- availableMemoryRatio?: number;
67
- /**
68
- * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
69
- * By default, the maximum memory is set to one quarter of total system memory.
70
- *
71
- * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
72
- */
73
- memoryMbytes?: number;
74
- /**
75
- * Defines the interval of emitting the `persistState` event.
76
- *
77
- * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
78
- * @default 60_000
79
- */
80
- persistStateIntervalMillis?: number;
81
- /**
82
- Defines the interval of emitting the `systemInfo` event.
83
- @default 1_000
84
- */
85
- systemInfoIntervalMillis?: number;
86
- /**
87
- * Defines the default input key, i.e. the key that is used to get the crawler input value
88
- * from the default {@link KeyValueStore} associated with the current crawler run.
89
- *
90
- * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
91
- * @default 'INPUT'
92
- */
93
- inputKey?: string;
94
- /**
95
- * Defines whether web browsers launched by Crawlee will run in the headless mode.
96
- *
97
- * Alternative to `CRAWLEE_HEADLESS` environment variable.
98
- * @default true
99
- */
100
- headless?: boolean;
101
- /**
102
- * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
103
- *
104
- * Alternative to `CRAWLEE_XVFB` environment variable.
105
- * @default false
106
- */
107
- xvfb?: boolean;
108
- /**
109
- * Defines a path to Chrome executable.
110
- *
111
- * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
112
- */
113
- chromeExecutablePath?: string;
114
- /**
115
- * Defines a path to default browser executable.
116
- *
117
- * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
118
- */
119
- defaultBrowserPath?: string;
120
- /**
121
- * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
122
- *
123
- * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
124
- */
125
- disableBrowserSandbox?: boolean;
126
- /**
127
- * Sets the log level to the given value.
128
- *
129
- * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
130
- * @default 'INFO'
131
- */
132
- logLevel?: LogLevel | LogLevel[keyof LogLevel];
133
- /**
134
- * Defines whether the storage client used should persist the data it stores.
135
- *
136
- * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
137
- */
138
- persistStorage?: boolean;
139
- /**
140
- * Defines whether to use the systemInfoV2 metric collection experiment.
141
- *
142
- * Alternative to `CRAWLEE_SYSTEM_INFO_V2` environment variable.
143
- */
144
- systemInfoV2?: boolean;
145
- /**
146
- * Used in place of `isContainerized()` when collecting system metrics.
147
- *
148
- * Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
149
- */
150
- containerized?: boolean;
1
+ import { z } from 'zod';
2
+ import { LogLevel } from './log.js';
3
+ export interface ConfigField<T extends z.ZodType = z.ZodType> {
4
+ schema: T;
5
+ envVar?: string | string[];
6
+ }
7
+ export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
8
+ /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
9
+ export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
10
+ export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
11
+ export declare const crawleeConfigFields: {
12
+ /** @default 'default' */
13
+ defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
14
+ /** @default true */
15
+ purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
16
+ /** @default 'default' */
17
+ defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
18
+ /** @default 'default' */
19
+ defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
20
+ /** @default 0.95 */
21
+ maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
22
+ /** @default 0.25 */
23
+ availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
24
+ memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
25
+ /** @default 60_000 */
26
+ persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
27
+ /** @default 1_000 */
28
+ systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
29
+ /** @default 'INPUT' */
30
+ inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
31
+ /** @default true */
32
+ headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
33
+ /** @default false */
34
+ xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
35
+ chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
36
+ defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
37
+ /** @default false */
38
+ disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
39
+ logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
40
+ /** @default true */
41
+ persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
42
+ /** @default './storage' */
43
+ storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
44
+ containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
45
+ };
46
+ export type FieldsInput<F extends Record<string, ConfigField>> = {
47
+ [K in keyof F]?: z.output<F[K]['schema']>;
48
+ };
49
+ export type FieldsOutput<F extends Record<string, ConfigField>> = {
50
+ [K in keyof F]: z.output<F[K]['schema']>;
51
+ };
52
+ export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
53
+ export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
54
+ /** @deprecated Use {@link ConfigurationInput} instead. */
55
+ export type ConfigurationOptions = ConfigurationInput;
56
+ export interface Configuration extends ResolvedConfigValues {
151
57
  }
152
58
  /**
153
59
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
@@ -161,13 +67,9 @@ export interface ConfigurationOptions {
161
67
  *
162
68
  * // Get the global configuration
163
69
  * const config = Configuration.getGlobalConfig();
164
- * // Set the 'persistStateIntervalMillis' option
165
- * // of global configuration to 10 seconds
166
- * config.set('persistStateIntervalMillis', 10_000);
167
- *
168
- * // No need to pass the configuration to the crawler,
169
- * // as it's using the global configuration by default
170
- * const crawler = new BasicCrawler();
70
+ * // Access configuration values directly as properties
71
+ * console.log(config.headless);
72
+ * console.log(config.persistStateIntervalMillis);
171
73
  * ```
172
74
  *
173
75
  * *Using custom configuration:*
@@ -177,15 +79,14 @@ export interface ConfigurationOptions {
177
79
  * // Create a new configuration
178
80
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
179
81
  * // Pass the configuration to the crawler
180
- * const crawler = new BasicCrawler({ ... }, config);
82
+ * const crawler = new BasicCrawler({ configuration: config });
181
83
  * ```
182
84
  *
183
- * The configuration provided via environment variables always takes precedence. We can also
184
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
185
- * so the options provided in constructor will override those. In other words, the precedence is:
85
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
86
+ * The priority order for resolving values is (highest to lowest):
186
87
  *
187
88
  * ```text
188
- * crawlee.json < constructor options < environment variables
89
+ * constructor options > environment variables > crawlee.json > schema defaults
189
90
  * ```
190
91
  *
191
92
  * ## Supported Configuration Options
@@ -201,97 +102,56 @@ export interface ConfigurationOptions {
201
102
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
202
103
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
203
104
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
105
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
204
106
  *
205
107
  * ## Advanced Configuration Options
206
108
  *
207
109
  * Key | Environment Variable | Default Value
208
110
  * ---|---|---
209
111
  * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
210
- * `xvfb` | `CRAWLEE_XVFB` | -
112
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
211
113
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
212
114
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
213
115
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
214
116
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
215
- * `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
216
- * `containerized | `CRAWLEE_CONTAINERIZED | -
117
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
217
118
  */
218
119
  export declare class Configuration {
219
120
  /**
220
- * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
121
+ * Field definitions for this configuration class.
122
+ * Subclasses override this to register additional fields.
221
123
  */
222
- protected static ENV_MAP: Dictionary;
223
- protected static BOOLEAN_VARS: string[];
224
- protected static INTEGER_VARS: string[];
225
- protected static COMMA_SEPARATED_LIST_VARS: string[];
226
- protected static DEFAULTS: Dictionary;
124
+ protected static fields: Record<string, ConfigField>;
125
+ private resolvedValues;
227
126
  /**
228
- * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
229
- * @internal
127
+ * Creates new `Configuration` instance with provided options.
128
+ * Constructor options take precedence over environment variables, which take precedence
129
+ * over crawlee.json values, which take precedence over schema defaults.
230
130
  */
231
- static storage: AsyncLocalStorage<Configuration>;
232
- protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
233
- protected services: Map<string, unknown>;
234
- /** @internal */
235
- static globalConfig?: Configuration;
236
- // @ts-ignore optional peer dependency or compatibility with es2022
237
- readonly storageManagers: Map<Constructor, StorageManager<import("./storages/storage_manager.js").IStorage>>;
131
+ constructor(options?: ConfigurationInput);
238
132
  /**
239
- * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
240
- */
241
- constructor(options?: ConfigurationOptions);
242
- /**
243
- * Returns configured value. First checks the environment variables, then provided configuration,
244
- * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
245
- * in the above section.
246
- */
247
- get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
248
- protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean | string[];
249
- /**
250
- * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
251
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
252
- */
253
- set(key: keyof ConfigurationOptions, value?: any): void;
254
- /**
255
- * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
256
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
257
- */
258
- static set(key: keyof ConfigurationOptions, value?: any): void;
259
- /**
260
- * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
261
- * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
262
- * return the same client instance.
133
+ * Returns the global configuration instance. It will respect the environment variables.
263
134
  *
264
- * Caching works based on the `storageClientOptions`, so calling this method with different options will return
265
- * multiple instances, one for each variant of the options.
266
- * @internal
135
+ * Delegates to the global ServiceLocator, making it the single source of truth for service management.
267
136
  */
268
- getStorageClient(): StorageClient;
269
- getEventManager(): EventManager;
270
- /**
271
- * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
272
- * @internal
273
- */
274
- createMemoryStorage(options?: MemoryStorageOptions): MemoryStorage;
275
- useStorageClient(client: StorageClient): void;
276
- static useStorageClient(client: StorageClient): void;
277
- useEventManager(events: EventManager): void;
137
+ static getGlobalConfig(): Configuration;
278
138
  /**
279
- * Returns the global configuration instance. It will respect the environment variables.
139
+ * Resolves all field values once using the priority chain:
140
+ * constructor options > env vars > crawlee.json > schema defaults.
280
141
  */
281
- static getGlobalConfig(): Configuration;
142
+ private static resolveAll;
282
143
  /**
283
- * Gets default {@link StorageClient} instance.
144
+ * Registers getters (and throwing setters) on the instance for each field.
284
145
  */
285
- static getStorageClient(): StorageClient;
146
+ private registerAccessors;
286
147
  /**
287
- * Gets default {@link EventManager} instance.
148
+ * Reads the first defined env var value for a field definition.
149
+ * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
150
+ * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
288
151
  */
289
- static getEventManager(): EventManager;
152
+ private static readEnvVar;
290
153
  /**
291
- * Resets global configuration instance. The default instance holds configuration based on env vars,
292
- * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
154
+ * Loads config options from crawlee.json in the current working directory.
293
155
  */
294
- static resetGlobalState(): void;
295
- protected buildOptions(options: ConfigurationOptions): void;
156
+ private static loadFileOptions;
296
157
  }
297
- //# sourceMappingURL=configuration.d.ts.map