@crawlee/core 4.0.0-beta.70 → 4.0.0-beta.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +2 -1
  2. package/autoscaling/autoscaled_pool.js +9 -2
  3. package/autoscaling/client_load_signal.d.ts +25 -0
  4. package/autoscaling/client_load_signal.js +36 -0
  5. package/autoscaling/cpu_load_signal.d.ts +28 -0
  6. package/autoscaling/cpu_load_signal.js +24 -0
  7. package/autoscaling/event_loop_load_signal.d.ts +23 -0
  8. package/autoscaling/event_loop_load_signal.js +35 -0
  9. package/autoscaling/index.d.ts +5 -1
  10. package/autoscaling/index.js +5 -1
  11. package/autoscaling/load_signal.d.ts +99 -0
  12. package/autoscaling/load_signal.js +135 -0
  13. package/autoscaling/memory_load_signal.d.ts +43 -0
  14. package/autoscaling/memory_load_signal.js +101 -0
  15. package/autoscaling/snapshotter.d.ts +36 -63
  16. package/autoscaling/snapshotter.js +83 -175
  17. package/autoscaling/system_status.d.ts +21 -33
  18. package/autoscaling/system_status.js +45 -82
  19. package/configuration.d.ts +0 -1
  20. package/configuration.js +0 -1
  21. package/cookie_utils.d.ts +0 -1
  22. package/cookie_utils.js +2 -1
  23. package/crawlers/context_pipeline.d.ts +0 -1
  24. package/crawlers/context_pipeline.js +0 -1
  25. package/crawlers/crawler_commons.d.ts +0 -1
  26. package/crawlers/crawler_commons.js +0 -1
  27. package/crawlers/crawler_utils.d.ts +0 -1
  28. package/crawlers/crawler_utils.js +0 -1
  29. package/crawlers/error_snapshotter.d.ts +0 -1
  30. package/crawlers/error_snapshotter.js +0 -1
  31. package/crawlers/error_tracker.d.ts +0 -1
  32. package/crawlers/error_tracker.js +0 -1
  33. package/crawlers/index.d.ts +0 -1
  34. package/crawlers/index.js +0 -1
  35. package/crawlers/internals/types.d.ts +0 -1
  36. package/crawlers/internals/types.js +0 -1
  37. package/crawlers/statistics.d.ts +6 -1
  38. package/crawlers/statistics.js +8 -1
  39. package/enqueue_links/enqueue_links.d.ts +8 -1
  40. package/enqueue_links/enqueue_links.js +9 -9
  41. package/enqueue_links/index.d.ts +0 -1
  42. package/enqueue_links/index.js +0 -1
  43. package/enqueue_links/shared.d.ts +0 -1
  44. package/enqueue_links/shared.js +0 -1
  45. package/errors.d.ts +20 -1
  46. package/errors.js +22 -1
  47. package/events/event_manager.d.ts +0 -1
  48. package/events/event_manager.js +0 -1
  49. package/events/index.d.ts +0 -1
  50. package/events/index.js +0 -1
  51. package/events/local_event_manager.d.ts +1 -1
  52. package/events/local_event_manager.js +9 -6
  53. package/index.d.ts +0 -1
  54. package/index.js +0 -1
  55. package/log.d.ts +0 -1
  56. package/log.js +0 -1
  57. package/memory-storage/consts.d.ts +0 -1
  58. package/memory-storage/consts.js +0 -1
  59. package/memory-storage/index.d.ts +0 -1
  60. package/memory-storage/index.js +0 -1
  61. package/memory-storage/memory-storage.d.ts +0 -1
  62. package/memory-storage/memory-storage.js +0 -1
  63. package/memory-storage/resource-clients/common/base-client.d.ts +0 -1
  64. package/memory-storage/resource-clients/common/base-client.js +0 -1
  65. package/memory-storage/resource-clients/dataset.d.ts +0 -1
  66. package/memory-storage/resource-clients/dataset.js +0 -1
  67. package/memory-storage/resource-clients/key-value-store.d.ts +0 -1
  68. package/memory-storage/resource-clients/key-value-store.js +0 -1
  69. package/memory-storage/resource-clients/request-queue.d.ts +0 -1
  70. package/memory-storage/resource-clients/request-queue.js +0 -1
  71. package/memory-storage/utils.d.ts +6 -1
  72. package/memory-storage/utils.js +15 -1
  73. package/package.json +5 -5
  74. package/proxy_configuration.d.ts +0 -1
  75. package/proxy_configuration.js +0 -1
  76. package/recoverable_state.d.ts +0 -1
  77. package/recoverable_state.js +0 -1
  78. package/request.d.ts +0 -1
  79. package/request.js +0 -1
  80. package/router.d.ts +117 -15
  81. package/router.js +124 -30
  82. package/serialization.d.ts +0 -1
  83. package/serialization.js +0 -1
  84. package/service_locator.d.ts +0 -1
  85. package/service_locator.js +0 -1
  86. package/session_pool/consts.d.ts +0 -1
  87. package/session_pool/consts.js +0 -1
  88. package/session_pool/errors.d.ts +0 -1
  89. package/session_pool/errors.js +0 -1
  90. package/session_pool/fingerprint.d.ts +0 -1
  91. package/session_pool/fingerprint.js +0 -1
  92. package/session_pool/index.d.ts +0 -1
  93. package/session_pool/index.js +0 -1
  94. package/session_pool/session.d.ts +0 -1
  95. package/session_pool/session.js +0 -1
  96. package/session_pool/session_pool.d.ts +0 -1
  97. package/session_pool/session_pool.js +0 -1
  98. package/storages/access_checking.d.ts +0 -1
  99. package/storages/access_checking.js +0 -1
  100. package/storages/dataset.d.ts +1 -2
  101. package/storages/dataset.js +0 -1
  102. package/storages/index.d.ts +0 -1
  103. package/storages/index.js +0 -1
  104. package/storages/key_value_store.d.ts +0 -1
  105. package/storages/key_value_store.js +0 -1
  106. package/storages/key_value_store_codec.d.ts +0 -1
  107. package/storages/key_value_store_codec.js +0 -1
  108. package/storages/request_dedup_cache.d.ts +23 -0
  109. package/storages/request_dedup_cache.js +48 -0
  110. package/storages/request_list.d.ts +6 -1
  111. package/storages/request_list.js +16 -3
  112. package/storages/request_loader.d.ts +0 -1
  113. package/storages/request_loader.js +0 -1
  114. package/storages/request_manager.d.ts +0 -1
  115. package/storages/request_manager.js +0 -1
  116. package/storages/request_manager_tandem.d.ts +0 -1
  117. package/storages/request_manager_tandem.js +0 -1
  118. package/storages/request_queue.d.ts +24 -1
  119. package/storages/request_queue.js +80 -27
  120. package/storages/sitemap_request_loader.d.ts +0 -1
  121. package/storages/sitemap_request_loader.js +0 -1
  122. package/storages/storage_instance_manager.d.ts +0 -1
  123. package/storages/storage_instance_manager.js +0 -1
  124. package/storages/storage_stats.d.ts +0 -1
  125. package/storages/storage_stats.js +0 -1
  126. package/storages/utils.d.ts +0 -1
  127. package/storages/utils.js +0 -1
  128. package/typedefs.d.ts +0 -1
  129. package/typedefs.js +0 -1
  130. package/validators.d.ts +0 -1
  131. package/validators.js +0 -1
  132. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  133. package/autoscaling/autoscaled_pool.js.map +0 -1
  134. package/autoscaling/index.d.ts.map +0 -1
  135. package/autoscaling/index.js.map +0 -1
  136. package/autoscaling/snapshotter.d.ts.map +0 -1
  137. package/autoscaling/snapshotter.js.map +0 -1
  138. package/autoscaling/system_status.d.ts.map +0 -1
  139. package/autoscaling/system_status.js.map +0 -1
  140. package/configuration.d.ts.map +0 -1
  141. package/configuration.js.map +0 -1
  142. package/cookie_utils.d.ts.map +0 -1
  143. package/cookie_utils.js.map +0 -1
  144. package/crawlers/context_pipeline.d.ts.map +0 -1
  145. package/crawlers/context_pipeline.js.map +0 -1
  146. package/crawlers/crawler_commons.d.ts.map +0 -1
  147. package/crawlers/crawler_commons.js.map +0 -1
  148. package/crawlers/crawler_utils.d.ts.map +0 -1
  149. package/crawlers/crawler_utils.js.map +0 -1
  150. package/crawlers/error_snapshotter.d.ts.map +0 -1
  151. package/crawlers/error_snapshotter.js.map +0 -1
  152. package/crawlers/error_tracker.d.ts.map +0 -1
  153. package/crawlers/error_tracker.js.map +0 -1
  154. package/crawlers/index.d.ts.map +0 -1
  155. package/crawlers/index.js.map +0 -1
  156. package/crawlers/internals/types.d.ts.map +0 -1
  157. package/crawlers/internals/types.js.map +0 -1
  158. package/crawlers/statistics.d.ts.map +0 -1
  159. package/crawlers/statistics.js.map +0 -1
  160. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  161. package/enqueue_links/enqueue_links.js.map +0 -1
  162. package/enqueue_links/index.d.ts.map +0 -1
  163. package/enqueue_links/index.js.map +0 -1
  164. package/enqueue_links/shared.d.ts.map +0 -1
  165. package/enqueue_links/shared.js.map +0 -1
  166. package/errors.d.ts.map +0 -1
  167. package/errors.js.map +0 -1
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts.map +0 -1
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/memory-storage/consts.d.ts.map +0 -1
  179. package/memory-storage/consts.js.map +0 -1
  180. package/memory-storage/index.d.ts.map +0 -1
  181. package/memory-storage/index.js.map +0 -1
  182. package/memory-storage/memory-storage.d.ts.map +0 -1
  183. package/memory-storage/memory-storage.js.map +0 -1
  184. package/memory-storage/resource-clients/common/base-client.d.ts.map +0 -1
  185. package/memory-storage/resource-clients/common/base-client.js.map +0 -1
  186. package/memory-storage/resource-clients/dataset.d.ts.map +0 -1
  187. package/memory-storage/resource-clients/dataset.js.map +0 -1
  188. package/memory-storage/resource-clients/key-value-store.d.ts.map +0 -1
  189. package/memory-storage/resource-clients/key-value-store.js.map +0 -1
  190. package/memory-storage/resource-clients/request-queue.d.ts.map +0 -1
  191. package/memory-storage/resource-clients/request-queue.js.map +0 -1
  192. package/memory-storage/utils.d.ts.map +0 -1
  193. package/memory-storage/utils.js.map +0 -1
  194. package/proxy_configuration.d.ts.map +0 -1
  195. package/proxy_configuration.js.map +0 -1
  196. package/recoverable_state.d.ts.map +0 -1
  197. package/recoverable_state.js.map +0 -1
  198. package/request.d.ts.map +0 -1
  199. package/request.js.map +0 -1
  200. package/router.d.ts.map +0 -1
  201. package/router.js.map +0 -1
  202. package/serialization.d.ts.map +0 -1
  203. package/serialization.js.map +0 -1
  204. package/service_locator.d.ts.map +0 -1
  205. package/service_locator.js.map +0 -1
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js.map +0 -1
  208. package/session_pool/errors.d.ts.map +0 -1
  209. package/session_pool/errors.js.map +0 -1
  210. package/session_pool/fingerprint.d.ts.map +0 -1
  211. package/session_pool/fingerprint.js.map +0 -1
  212. package/session_pool/index.d.ts.map +0 -1
  213. package/session_pool/index.js.map +0 -1
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts.map +0 -1
  217. package/session_pool/session_pool.js.map +0 -1
  218. package/storages/access_checking.d.ts.map +0 -1
  219. package/storages/access_checking.js.map +0 -1
  220. package/storages/dataset.d.ts.map +0 -1
  221. package/storages/dataset.js.map +0 -1
  222. package/storages/index.d.ts.map +0 -1
  223. package/storages/index.js.map +0 -1
  224. package/storages/key_value_store.d.ts.map +0 -1
  225. package/storages/key_value_store.js.map +0 -1
  226. package/storages/key_value_store_codec.d.ts.map +0 -1
  227. package/storages/key_value_store_codec.js.map +0 -1
  228. package/storages/request_list.d.ts.map +0 -1
  229. package/storages/request_list.js.map +0 -1
  230. package/storages/request_loader.d.ts.map +0 -1
  231. package/storages/request_loader.js.map +0 -1
  232. package/storages/request_manager.d.ts.map +0 -1
  233. package/storages/request_manager.js.map +0 -1
  234. package/storages/request_manager_tandem.d.ts.map +0 -1
  235. package/storages/request_manager_tandem.js.map +0 -1
  236. package/storages/request_queue.d.ts.map +0 -1
  237. package/storages/request_queue.js.map +0 -1
  238. package/storages/sitemap_request_loader.d.ts.map +0 -1
  239. package/storages/sitemap_request_loader.js.map +0 -1
  240. package/storages/storage_instance_manager.d.ts.map +0 -1
  241. package/storages/storage_instance_manager.js.map +0 -1
  242. package/storages/storage_stats.d.ts.map +0 -1
  243. package/storages/storage_stats.js.map +0 -1
  244. package/storages/utils.d.ts.map +0 -1
  245. package/storages/utils.js.map +0 -1
  246. package/typedefs.d.ts.map +0 -1
  247. package/typedefs.js.map +0 -1
  248. package/validators.d.ts.map +0 -1
  249. package/validators.js.map +0 -1
@@ -1,6 +1,8 @@
1
- import { weightedAvg } from '@crawlee/utils';
2
1
  import ow from 'ow';
2
+ import { evaluateLoadSignalSample } from './load_signal.js';
3
3
  import { Snapshotter } from './snapshotter.js';
4
+ /** The four built-in signal names that map to typed `SystemInfo` fields. */
5
+ const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
4
6
  /**
5
7
  * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
6
8
  * It only exposes two functions {@link SystemStatus.getCurrentStatus}
@@ -25,11 +27,14 @@ import { Snapshotter } from './snapshotter.js';
25
27
  */
26
28
  export class SystemStatus {
27
29
  currentHistoryMillis;
28
- maxMemoryOverloadedRatio;
29
- maxEventLoopOverloadedRatio;
30
- maxCpuOverloadedRatio;
31
- maxClientOverloadedRatio;
32
30
  snapshotter;
31
+ signals;
32
+ /**
33
+ * Per-signal ratio overrides. The built-in four get their overrides from
34
+ * the legacy `max*OverloadedRatio` options; custom signals use their own
35
+ * `overloadedRatio`.
36
+ */
37
+ ratioOverrides;
33
38
  constructor(options = {}) {
34
39
  ow(options, ow.object.exactShape({
35
40
  currentHistorySecs: ow.optional.number,
@@ -38,14 +43,20 @@ export class SystemStatus {
38
43
  maxCpuOverloadedRatio: ow.optional.number,
39
44
  maxClientOverloadedRatio: ow.optional.number,
40
45
  snapshotter: ow.optional.object,
46
+ loadSignals: ow.optional.array,
41
47
  }));
42
- const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, } = options;
48
+ const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
43
49
  this.currentHistoryMillis = currentHistorySecs * 1000;
44
- this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
45
- this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
46
- this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
47
- this.maxClientOverloadedRatio = maxClientOverloadedRatio;
48
50
  this.snapshotter = snapshotter || new Snapshotter();
51
+ // Built-in signals from the snapshotter + any custom signals
52
+ this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
53
+ // Allow legacy options to override the built-in signal ratios
54
+ this.ratioOverrides = {
55
+ memInfo: maxMemoryOverloadedRatio,
56
+ eventLoopInfo: maxEventLoopOverloadedRatio,
57
+ cpuInfo: maxCpuOverloadedRatio,
58
+ clientInfo: maxClientOverloadedRatio,
59
+ };
49
60
  }
50
61
  /**
51
62
  * Returns an {@link SystemInfo} object with the following structure:
@@ -89,80 +100,32 @@ export class SystemStatus {
89
100
  * Returns a system status object.
90
101
  */
91
102
  _isSystemIdle(sampleDurationMillis) {
92
- const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
93
- const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
94
- const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
95
- const clientInfo = this._isClientOverloaded(sampleDurationMillis);
96
- return {
97
- isSystemIdle: !memInfo.isOverloaded &&
98
- !eventLoopInfo.isOverloaded &&
99
- !cpuInfo.isOverloaded &&
100
- !clientInfo.isOverloaded,
101
- memInfo,
102
- eventLoopInfo,
103
- cpuInfo,
104
- clientInfo,
103
+ const result = {
104
+ isSystemIdle: true,
105
+ memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
106
+ eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
107
+ cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
108
+ clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
105
109
  };
106
- }
107
- /**
108
- * Returns an object with an isOverloaded property set to true
109
- * if the memory has been overloaded in the last sampleDurationMillis.
110
- */
111
- _isMemoryOverloaded(sampleDurationMillis) {
112
- const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
113
- return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
114
- }
115
- /**
116
- * Returns an object with an isOverloaded property set to true
117
- * if the event loop has been overloaded in the last sampleDurationMillis.
118
- */
119
- _isEventLoopOverloaded(sampleDurationMillis) {
120
- const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
121
- return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
122
- }
123
- /**
124
- * Returns an object with an isOverloaded property set to true
125
- * if the CPU has been overloaded in the last sampleDurationMillis.
126
- */
127
- _isCpuOverloaded(sampleDurationMillis) {
128
- const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
129
- return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
130
- }
131
- /**
132
- * Returns an object with an isOverloaded property set to true
133
- * if the client has been overloaded in the last sampleDurationMillis.
134
- */
135
- _isClientOverloaded(sampleDurationMillis) {
136
- const sample = this.snapshotter.getClientSample(sampleDurationMillis);
137
- return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
138
- }
139
- /**
140
- * Returns an object with sample information and an isOverloaded property
141
- * set to true if at least the ratio of snapshots in the sample are overloaded.
142
- */
143
- _isSampleOverloaded(sample, ratio) {
144
- if (sample.length === 0) {
145
- return {
146
- isOverloaded: false,
147
- limitRatio: ratio,
148
- actualRatio: 0,
149
- };
110
+ let loadSignalInfo;
111
+ for (const signal of this.signals) {
112
+ const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
113
+ const sample = signal.getSample(sampleDurationMillis);
114
+ const info = evaluateLoadSignalSample(sample, ratio);
115
+ if (info.isOverloaded) {
116
+ result.isSystemIdle = false;
117
+ }
118
+ if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
119
+ result[signal.name] = info;
120
+ }
121
+ else {
122
+ loadSignalInfo ??= {};
123
+ loadSignalInfo[signal.name] = info;
124
+ }
150
125
  }
151
- const weights = [];
152
- const values = [];
153
- for (let i = 1; i < sample.length; i++) {
154
- const previous = sample[i - 1];
155
- const current = sample[i];
156
- const weight = +current.createdAt - +previous.createdAt;
157
- weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
158
- values.push(+current.isOverloaded);
126
+ if (loadSignalInfo) {
127
+ result.loadSignalInfo = loadSignalInfo;
159
128
  }
160
- const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
161
- return {
162
- isOverloaded: wAvg > ratio,
163
- limitRatio: ratio,
164
- actualRatio: Math.round(wAvg * 1000) / 1000,
165
- };
129
+ return result;
166
130
  }
167
131
  }
168
- //# sourceMappingURL=system_status.js.map
@@ -155,4 +155,3 @@ export declare class Configuration {
155
155
  */
156
156
  private static loadFileOptions;
157
157
  }
158
- //# sourceMappingURL=configuration.d.ts.map
package/configuration.js CHANGED
@@ -242,4 +242,3 @@ export class Configuration {
242
242
  }
243
243
  }
244
244
  }
245
- //# sourceMappingURL=configuration.js.map
package/cookie_utils.d.ts CHANGED
@@ -42,4 +42,3 @@ export declare function cookieStringToToughCookie(cookieString: string): CookieO
42
42
  * @internal
43
43
  */
44
44
  export declare function mergeCookies(url: string, sourceCookies: string[]): string;
45
- //# sourceMappingURL=cookie_utils.d.ts.map
package/cookie_utils.js CHANGED
@@ -103,6 +103,8 @@ export function mergeCookies(url, sourceCookies) {
103
103
  if (!cookieString)
104
104
  continue;
105
105
  const cookie = Cookie.parse(cookieString);
106
+ if (!cookie)
107
+ throw new CookieParseError(cookieString);
106
108
  const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
107
109
  return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
108
110
  });
@@ -116,4 +118,3 @@ export function mergeCookies(url, sourceCookies) {
116
118
  }
117
119
  return jar.getCookieStringSync(url);
118
120
  }
119
- //# sourceMappingURL=cookie_utils.js.map
@@ -68,4 +68,3 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
68
68
  */
69
69
  abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
70
70
  }
71
- //# sourceMappingURL=context_pipeline.d.ts.map
@@ -120,4 +120,3 @@ class ContextPipelineImpl extends ContextPipeline {
120
120
  }
121
121
  }
122
122
  }
123
- //# sourceMappingURL=context_pipeline.js.map
@@ -187,4 +187,3 @@ export declare class RequestHandlerResult {
187
187
  private getKeyValueStoreChangedValue;
188
188
  private setKeyValueStoreChangedValue;
189
189
  }
190
- //# sourceMappingURL=crawler_commons.d.ts.map
@@ -105,4 +105,3 @@ export class RequestHandlerResult {
105
105
  this._keyValueStoreChanges[id][key] = { changedValue, options };
106
106
  };
107
107
  }
108
- //# sourceMappingURL=crawler_commons.js.map
@@ -7,4 +7,3 @@ export declare function handleRequestTimeout({ session, errorMessage }: {
7
7
  session?: ISession;
8
8
  errorMessage: string;
9
9
  }): void;
10
- //# sourceMappingURL=crawler_utils.d.ts.map
@@ -9,4 +9,3 @@ export function handleRequestTimeout({ session, errorMessage }) {
9
9
  const timeoutSecs = Number(timeoutMillis) / 1000;
10
10
  throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
11
11
  }
12
- //# sourceMappingURL=crawler_utils.js.map
@@ -57,4 +57,3 @@ export declare class ErrorSnapshotter {
57
57
  generateFilename(error: ErrnoException): string;
58
58
  }
59
59
  export {};
60
- //# sourceMappingURL=error_snapshotter.d.ts.map
@@ -115,4 +115,3 @@ export class ErrorSnapshotter {
115
115
  return fileName;
116
116
  }
117
117
  }
118
- //# sourceMappingURL=error_snapshotter.js.map
@@ -52,4 +52,3 @@ export declare class ErrorTracker {
52
52
  captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
53
53
  reset(): void;
54
54
  }
55
- //# sourceMappingURL=error_tracker.d.ts.map
@@ -306,4 +306,3 @@ export class ErrorTracker {
306
306
  }
307
307
  }
308
308
  }
309
- //# sourceMappingURL=error_tracker.js.map
@@ -4,4 +4,3 @@ export * from './crawler_utils.js';
4
4
  export * from './statistics.js';
5
5
  export * from './error_tracker.js';
6
6
  export * from './error_snapshotter.js';
7
- //# sourceMappingURL=index.d.ts.map
package/crawlers/index.js CHANGED
@@ -4,4 +4,3 @@ export * from './crawler_utils.js';
4
4
  export * from './statistics.js';
5
5
  export * from './error_tracker.js';
6
6
  export * from './error_snapshotter.js';
7
- //# sourceMappingURL=index.js.map
@@ -5,4 +5,3 @@ export interface SnapshottableProperties {
5
5
  body?: unknown;
6
6
  page?: BrowserPage;
7
7
  }
8
- //# sourceMappingURL=types.d.ts.map
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=types.js.map
@@ -86,6 +86,12 @@ export declare class Statistics {
86
86
  * @ignore
87
87
  */
88
88
  failJob(id: number | string, retryCount: number): void;
89
+ /**
90
+ * Discards a started job without affecting the finished/failed counters, e.g. when a request
91
+ * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
92
+ * @ignore
93
+ */
94
+ discardJob(id: number | string): void;
89
95
  /**
90
96
  * Calculate the current statistics
91
97
  */
@@ -201,4 +207,3 @@ export interface StatisticState {
201
207
  retryErrors: Record<string, unknown>;
202
208
  requestsWithStatusCode: Record<string, number>;
203
209
  }
204
- //# sourceMappingURL=statistics.d.ts.map
@@ -195,6 +195,14 @@ export class Statistics {
195
195
  this._saveRetryCountForJob(retryCount);
196
196
  this.requestsInProgress.delete(id);
197
197
  }
198
+ /**
199
+ * Discards a started job without affecting the finished/failed counters, e.g. when a request
200
+ * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
201
+ * @ignore
202
+ */
203
+ discardJob(id) {
204
+ this.requestsInProgress.delete(id);
205
+ }
198
206
  /**
199
207
  * Calculate the current statistics
200
208
  */
@@ -340,4 +348,3 @@ export class Statistics {
340
348
  return result;
341
349
  }
342
350
  }
343
- //# sourceMappingURL=statistics.js.map
@@ -147,6 +147,14 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
147
147
  * If provided, disallowed URLs will be ignored.
148
148
  */
149
149
  robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
150
+ /**
151
+ * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
152
+ * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
153
+ * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
154
+ */
155
+ respectRobotsTxtFile?: boolean | {
156
+ userAgent?: string;
157
+ };
150
158
  /**
151
159
  * When a request is skipped for some reason, you can use this callback to act on it.
152
160
  * This is currently fired for requests skipped
@@ -254,4 +262,3 @@ export interface ResolveBaseUrl {
254
262
  originalRequestUrl: string;
255
263
  finalRequestUrl?: string;
256
264
  }
257
- //# sourceMappingURL=enqueue_links.d.ts.map
@@ -90,6 +90,7 @@ export async function enqueueLinks(options) {
90
90
  urls: ow.array.ofType(ow.string),
91
91
  requestManager: ow.object.hasKeys('addRequestsBatched'),
92
92
  robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
93
+ respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
93
94
  onSkippedRequest: ow.optional.function,
94
95
  forefront: ow.optional.boolean,
95
96
  skipNavigation: ow.optional.boolean,
@@ -182,10 +183,11 @@ export async function enqueueLinks(options) {
182
183
  }
183
184
  }
184
185
  let requestOptions = createRequestOptions(urls, options);
185
- if (robotsTxtFile) {
186
+ if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
187
+ const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
186
188
  const skippedRequests = [];
187
189
  requestOptions = requestOptions.filter((request) => {
188
- if (robotsTxtFile.isAllowed(request.url)) {
190
+ if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
189
191
  return true;
190
192
  }
191
193
  skippedRequests.push(request);
@@ -217,15 +219,14 @@ export async function enqueueLinks(options) {
217
219
  // Step 3: Create Request instances from the final request options
218
220
  return filteredOptions.map((opts) => new Request(opts));
219
221
  }
220
- let requests = await createFilteredRequests();
221
- if (typeof limit === 'number' && limit < requests.length) {
222
- await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
223
- requests = requests.slice(0, limit);
224
- }
225
- const { addedRequests } = await requestManager.addRequestsBatched(requests, {
222
+ const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
226
223
  forefront,
227
224
  waitForAllRequestsToBeAdded,
225
+ maxNewRequests: limit,
228
226
  });
227
+ if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
228
+ await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
229
+ }
229
230
  return { processedRequests: addedRequests, unprocessedRequests: [] };
230
231
  }
231
232
  /**
@@ -268,4 +269,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
268
269
  function ignoreHttpSchema(pattern) {
269
270
  return pattern.replace(/^(https?):\/\//, 'http{s,}://');
270
271
  }
271
- //# sourceMappingURL=enqueue_links.js.map
@@ -1,3 +1,2 @@
1
1
  export * from './enqueue_links.js';
2
2
  export * from './shared.js';
3
- //# sourceMappingURL=index.d.ts.map
@@ -1,3 +1,2 @@
1
1
  export * from './enqueue_links.js';
2
2
  export * from './shared.js';
3
- //# sourceMappingURL=index.js.map
@@ -81,4 +81,3 @@ export interface RequestTransform {
81
81
  * @internal
82
82
  */
83
83
  export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
84
- //# sourceMappingURL=shared.d.ts.map
@@ -219,4 +219,3 @@ export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
219
219
  })
220
220
  .filter((r) => r !== null);
221
221
  }
222
- //# sourceMappingURL=shared.js.map
package/errors.d.ts CHANGED
@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
14
14
  */
15
15
  export declare class MissingRouteError extends CriticalError {
16
16
  }
17
+ /**
18
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
19
+ *
20
+ * As the `userData` does not change between attempts, this error is non-retryable.
21
+ */
22
+ export declare class RequestValidationError extends NonRetryableError {
23
+ readonly label: string | symbol;
24
+ readonly issues: readonly {
25
+ readonly message: string;
26
+ readonly path?: readonly (PropertyKey | {
27
+ key: PropertyKey;
28
+ })[];
29
+ }[];
30
+ constructor(label: string | symbol, issues: readonly {
31
+ readonly message: string;
32
+ readonly path?: readonly (PropertyKey | {
33
+ key: PropertyKey;
34
+ })[];
35
+ }[]);
36
+ }
17
37
  /**
18
38
  * Errors of `RetryRequestError` type will always be retried by the crawler.
19
39
  *
@@ -60,4 +80,3 @@ export declare class ServiceConflictError extends Error {
60
80
  */
61
81
  export declare class NavigationSkippedError extends NonRetryableError {
62
82
  }
63
- //# sourceMappingURL=errors.d.ts.map
package/errors.js CHANGED
@@ -15,6 +15,28 @@ export class CriticalError extends NonRetryableError {
15
15
  */
16
16
  export class MissingRouteError extends CriticalError {
17
17
  }
18
+ /**
19
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
20
+ *
21
+ * As the `userData` does not change between attempts, this error is non-retryable.
22
+ */
23
+ export class RequestValidationError extends NonRetryableError {
24
+ label;
25
+ issues;
26
+ constructor(label, issues) {
27
+ const details = issues
28
+ .map((issue) => {
29
+ const path = (issue.path ?? [])
30
+ .map((segment) => (typeof segment === 'object' ? segment.key : segment))
31
+ .join('.');
32
+ return `- ${path ? `${path}: ` : ''}${issue.message}`;
33
+ })
34
+ .join('\n');
35
+ super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
36
+ this.label = label;
37
+ this.issues = issues;
38
+ }
39
+ }
18
40
  /**
19
41
  * Errors of `RetryRequestError` type will always be retried by the crawler.
20
42
  *
@@ -78,4 +100,3 @@ export class ServiceConflictError extends Error {
78
100
  */
79
101
  export class NavigationSkippedError extends NonRetryableError {
80
102
  }
81
- //# sourceMappingURL=errors.js.map
@@ -76,4 +76,3 @@ export declare abstract class EventManager {
76
76
  waitForAllListenersToComplete(): Promise<boolean>;
77
77
  }
78
78
  export {};
79
- //# sourceMappingURL=event_manager.d.ts.map
@@ -85,4 +85,3 @@ export class EventManager {
85
85
  return this.events.waitForAllListenersToComplete();
86
86
  }
87
87
  }
88
- //# sourceMappingURL=event_manager.js.map
package/events/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.d.ts.map
package/events/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.js.map
@@ -35,5 +35,5 @@ export declare class LocalEventManager extends EventManager {
35
35
  private createSystemInfo;
36
36
  private createCpuInfo;
37
37
  private createMemoryInfo;
38
+ private getMemoryInfo;
38
39
  }
39
- //# sourceMappingURL=local_event_manager.d.ts.map
@@ -80,12 +80,9 @@ export class LocalEventManager extends EventManager {
80
80
  }
81
81
  async createMemoryInfo() {
82
82
  try {
83
- const { getMemoryInfo } = await import('@crawlee/utils');
84
- const memInfo = await getMemoryInfo({
85
- containerized: await this.isContainerizedWrapper(),
86
- logger: serviceLocator.getLogger(),
87
- });
83
+ const memInfo = await this.getMemoryInfo();
88
84
  return {
85
+ memTotalBytes: memInfo.totalBytes,
89
86
  memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
90
87
  };
91
88
  }
@@ -94,5 +91,11 @@ export class LocalEventManager extends EventManager {
94
91
  return {};
95
92
  }
96
93
  }
94
+ async getMemoryInfo() {
95
+ const { getMemoryInfo } = await import('@crawlee/utils');
96
+ return getMemoryInfo({
97
+ containerized: await this.isContainerizedWrapper(),
98
+ logger: serviceLocator.getLogger(),
99
+ });
100
+ }
97
101
  }
98
- //# sourceMappingURL=local_event_manager.js.map
package/index.d.ts CHANGED
@@ -18,4 +18,3 @@ export * from './cookie_utils.js';
18
18
  export * from './recoverable_state.js';
19
19
  export { PseudoUrl } from '@apify/pseudo_url';
20
20
  export type { Dictionary, Awaitable, Constructor, StorageBackend, Cookie, QueueOperationInfo } from '@crawlee/types';
21
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -17,4 +17,3 @@ export * from './validators.js';
17
17
  export * from './cookie_utils.js';
18
18
  export * from './recoverable_state.js';
19
19
  export { PseudoUrl } from '@apify/pseudo_url';
20
- //# sourceMappingURL=index.js.map
package/log.d.ts CHANGED
@@ -80,4 +80,3 @@ export declare class ApifyLogAdapter extends BaseCrawleeLogger {
80
80
  }
81
81
  export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
82
82
  export type { LoggerOptions };
83
- //# sourceMappingURL=log.d.ts.map
package/log.js CHANGED
@@ -102,4 +102,3 @@ export class ApifyLogAdapter extends BaseCrawleeLogger {
102
102
  }
103
103
  }
104
104
  export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
105
- //# sourceMappingURL=log.js.map
@@ -2,4 +2,3 @@
2
2
  * Length of id property of a Request instance in characters.
3
3
  */
4
4
  export declare const REQUEST_ID_LENGTH = 15;
5
- //# sourceMappingURL=consts.d.ts.map
@@ -2,4 +2,3 @@
2
2
  * Length of id property of a Request instance in characters.
3
3
  */
4
4
  export const REQUEST_ID_LENGTH = 15;
5
- //# sourceMappingURL=consts.js.map
@@ -1,2 +1 @@
1
1
  export * from './memory-storage.js';
2
- //# sourceMappingURL=index.d.ts.map
@@ -1,2 +1 @@
1
1
  export * from './memory-storage.js';
2
- //# sourceMappingURL=index.js.map
@@ -44,4 +44,3 @@ export declare class MemoryStorageBackend implements storage.StorageBackend {
44
44
  */
45
45
  teardown(): Promise<void>;
46
46
  }
47
- //# sourceMappingURL=memory-storage.d.ts.map
@@ -134,4 +134,3 @@ export class MemoryStorageBackend {
134
134
  // Nothing to tear down for in-memory storage.
135
135
  }
136
136
  }
137
- //# sourceMappingURL=memory-storage.js.map
@@ -2,4 +2,3 @@ export declare class BaseClient {
2
2
  id: string;
3
3
  constructor(id: string);
4
4
  }
5
- //# sourceMappingURL=base-client.d.ts.map
@@ -4,4 +4,3 @@ export class BaseClient {
4
4
  this.id = id;
5
5
  }
6
6
  }
7
- //# sourceMappingURL=base-client.js.map
@@ -38,4 +38,3 @@ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extend
38
38
  private getStartAndEndIndexes;
39
39
  private updateTimestamps;
40
40
  }
41
- //# sourceMappingURL=dataset.d.ts.map
@@ -111,4 +111,3 @@ export class DatasetBackend extends BaseClient {
111
111
  }
112
112
  }
113
113
  }
114
- //# sourceMappingURL=dataset.js.map
@@ -61,4 +61,3 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
61
61
  toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
62
62
  private updateTimestamps;
63
63
  }
64
- //# sourceMappingURL=key-value-store.d.ts.map
@@ -201,4 +201,3 @@ export class KeyValueStoreBackend extends BaseClient {
201
201
  }
202
202
  }
203
203
  }
204
- //# sourceMappingURL=key-value-store.js.map
@@ -94,4 +94,3 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
94
94
  private _createInternalRequest;
95
95
  private _calculateOrderNo;
96
96
  }
97
- //# sourceMappingURL=request-queue.d.ts.map
@@ -419,4 +419,3 @@ export class RequestQueueBackend extends BaseClient {
419
419
  return forefront ? -timestamp : timestamp;
420
420
  }
421
421
  }
422
- //# sourceMappingURL=request-queue.js.map