@crawlee/core 4.0.0-beta.15 → 4.0.0-beta.150

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/README.md +14 -14
  2. package/autoscaling/autoscaled_pool.d.ts +73 -177
  3. package/autoscaling/autoscaled_pool.js +182 -329
  4. package/autoscaling/concurrency_system.d.ts +268 -0
  5. package/autoscaling/concurrency_system.js +364 -0
  6. package/autoscaling/cpu_load_signal.d.ts +43 -0
  7. package/autoscaling/cpu_load_signal.js +47 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  9. package/autoscaling/event_loop_load_signal.js +60 -0
  10. package/autoscaling/index.d.ts +6 -1
  11. package/autoscaling/index.js +6 -1
  12. package/autoscaling/load_signal.d.ts +98 -0
  13. package/autoscaling/load_signal.js +103 -0
  14. package/autoscaling/memory_load_signal.d.ts +47 -0
  15. package/autoscaling/memory_load_signal.js +106 -0
  16. package/autoscaling/snapshotter.d.ts +58 -163
  17. package/autoscaling/snapshotter.js +45 -263
  18. package/autoscaling/storage_backend_load_signal.d.ts +56 -0
  19. package/autoscaling/storage_backend_load_signal.js +73 -0
  20. package/autoscaling/system_status.d.ts +67 -89
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +3 -2
  29. package/cookie_utils.js +18 -7
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +90 -126
  33. package/crawlers/crawler_commons.js +1 -108
  34. package/crawlers/error_snapshotter.d.ts +2 -5
  35. package/crawlers/error_snapshotter.js +7 -8
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +187 -64
  43. package/crawlers/statistics.js +354 -165
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +61 -154
  47. package/enqueue_links/enqueue_links.js +40 -232
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +55 -28
  51. package/enqueue_links/shared.js +116 -69
  52. package/errors.d.ts +72 -4
  53. package/errors.js +89 -5
  54. package/events/event_manager.d.ts +35 -9
  55. package/events/event_manager.js +12 -13
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +39 -13
  60. package/http.d.ts +9 -0
  61. package/http.js +28 -0
  62. package/index.d.ts +7 -5
  63. package/index.js +8 -4
  64. package/iterables.d.ts +79 -0
  65. package/iterables.js +134 -0
  66. package/log.d.ts +77 -1
  67. package/log.js +104 -1
  68. package/memory-storage/consts.d.ts +4 -0
  69. package/memory-storage/consts.js +4 -0
  70. package/memory-storage/index.d.ts +1 -0
  71. package/memory-storage/index.js +1 -0
  72. package/memory-storage/memory-storage.d.ts +44 -0
  73. package/memory-storage/memory-storage.js +160 -0
  74. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  75. package/memory-storage/resource-clients/common/base-client.js +6 -0
  76. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  77. package/memory-storage/resource-clients/dataset.js +106 -0
  78. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  79. package/memory-storage/resource-clients/key-value-store.js +199 -0
  80. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  81. package/memory-storage/resource-clients/request-queue.js +407 -0
  82. package/memory-storage/utils.d.ts +16 -0
  83. package/memory-storage/utils.js +41 -0
  84. package/owned_or_injected.d.ts +58 -0
  85. package/owned_or_injected.js +98 -0
  86. package/package.json +13 -12
  87. package/proxy_configuration.d.ts +22 -128
  88. package/proxy_configuration.js +32 -144
  89. package/recoverable_state.d.ts +83 -51
  90. package/recoverable_state.js +163 -72
  91. package/request.d.ts +57 -16
  92. package/request.js +130 -69
  93. package/router.d.ts +193 -21
  94. package/router.js +188 -43
  95. package/serialization.d.ts +0 -1
  96. package/serialization.js +15 -15
  97. package/service_locator.d.ts +165 -0
  98. package/service_locator.js +253 -0
  99. package/session_pool/consts.d.ts +1 -2
  100. package/session_pool/consts.js +1 -2
  101. package/session_pool/errors.d.ts +0 -1
  102. package/session_pool/errors.js +0 -1
  103. package/session_pool/fingerprint.d.ts +9 -0
  104. package/session_pool/fingerprint.js +30 -0
  105. package/session_pool/index.d.ts +0 -2
  106. package/session_pool/index.js +0 -2
  107. package/session_pool/session.d.ts +39 -89
  108. package/session_pool/session.js +102 -159
  109. package/session_pool/session_pool.d.ts +67 -91
  110. package/session_pool/session_pool.js +196 -187
  111. package/storages/batched_adds.d.ts +37 -0
  112. package/storages/batched_adds.js +73 -0
  113. package/storages/dataset.d.ts +109 -56
  114. package/storages/dataset.js +284 -149
  115. package/storages/index.d.ts +9 -9
  116. package/storages/index.js +7 -9
  117. package/storages/key_value_store.d.ts +183 -48
  118. package/storages/key_value_store.js +444 -171
  119. package/storages/key_value_store_codec.d.ts +32 -0
  120. package/storages/key_value_store_codec.js +113 -0
  121. package/storages/request_dedup_cache.d.ts +22 -0
  122. package/storages/request_dedup_cache.js +48 -0
  123. package/storages/request_list.d.ts +53 -115
  124. package/storages/request_list.js +194 -167
  125. package/storages/request_loader.d.ts +133 -0
  126. package/storages/request_loader.js +36 -0
  127. package/storages/request_manager.d.ts +109 -0
  128. package/storages/request_manager.js +1 -0
  129. package/storages/request_manager_tandem.d.ts +55 -23
  130. package/storages/request_manager_tandem.js +139 -57
  131. package/storages/request_queue.d.ts +286 -48
  132. package/storages/request_queue.js +757 -218
  133. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +43 -91
  134. package/storages/sitemap_request_loader.js +439 -0
  135. package/storages/storage_instance_manager.d.ts +87 -0
  136. package/storages/storage_instance_manager.js +256 -0
  137. package/storages/storage_stats.d.ts +48 -0
  138. package/storages/storage_stats.js +29 -0
  139. package/storages/throttling_request_manager.d.ts +222 -0
  140. package/storages/throttling_request_manager.js +780 -0
  141. package/storages/transaction.d.ts +252 -0
  142. package/storages/transaction.js +251 -0
  143. package/storages/utils.d.ts +59 -11
  144. package/storages/utils.js +75 -15
  145. package/system-info/cpu-info.d.ts +67 -0
  146. package/system-info/cpu-info.js +216 -0
  147. package/system-info/memory-info.d.ts +31 -0
  148. package/system-info/memory-info.js +115 -0
  149. package/system-info/ps-tree.d.ts +17 -0
  150. package/system-info/ps-tree.js +144 -0
  151. package/system-info/runtime.d.ts +14 -0
  152. package/system-info/runtime.js +80 -0
  153. package/typedefs.d.ts +0 -6
  154. package/typedefs.js +0 -1
  155. package/url.d.ts +9 -0
  156. package/url.js +11 -0
  157. package/validators.d.ts +22 -18
  158. package/validators.js +13 -18
  159. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  160. package/autoscaling/autoscaled_pool.js.map +0 -1
  161. package/autoscaling/index.d.ts.map +0 -1
  162. package/autoscaling/index.js.map +0 -1
  163. package/autoscaling/snapshotter.d.ts.map +0 -1
  164. package/autoscaling/snapshotter.js.map +0 -1
  165. package/autoscaling/system_status.d.ts.map +0 -1
  166. package/autoscaling/system_status.js.map +0 -1
  167. package/configuration.d.ts.map +0 -1
  168. package/configuration.js.map +0 -1
  169. package/cookie_utils.d.ts.map +0 -1
  170. package/cookie_utils.js.map +0 -1
  171. package/crawlers/context_pipeline.d.ts.map +0 -1
  172. package/crawlers/context_pipeline.js.map +0 -1
  173. package/crawlers/crawler_commons.d.ts.map +0 -1
  174. package/crawlers/crawler_commons.js.map +0 -1
  175. package/crawlers/crawler_utils.d.ts +0 -10
  176. package/crawlers/crawler_utils.d.ts.map +0 -1
  177. package/crawlers/crawler_utils.js +0 -12
  178. package/crawlers/crawler_utils.js.map +0 -1
  179. package/crawlers/error_snapshotter.d.ts.map +0 -1
  180. package/crawlers/error_snapshotter.js.map +0 -1
  181. package/crawlers/error_tracker.d.ts.map +0 -1
  182. package/crawlers/error_tracker.js.map +0 -1
  183. package/crawlers/index.d.ts.map +0 -1
  184. package/crawlers/index.js.map +0 -1
  185. package/crawlers/internals/types.d.ts.map +0 -1
  186. package/crawlers/internals/types.js.map +0 -1
  187. package/crawlers/statistics.d.ts.map +0 -1
  188. package/crawlers/statistics.js.map +0 -1
  189. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  190. package/enqueue_links/enqueue_links.js.map +0 -1
  191. package/enqueue_links/index.d.ts.map +0 -1
  192. package/enqueue_links/index.js.map +0 -1
  193. package/enqueue_links/shared.d.ts.map +0 -1
  194. package/enqueue_links/shared.js.map +0 -1
  195. package/errors.d.ts.map +0 -1
  196. package/errors.js.map +0 -1
  197. package/events/event_manager.d.ts.map +0 -1
  198. package/events/event_manager.js.map +0 -1
  199. package/events/index.d.ts.map +0 -1
  200. package/events/index.js.map +0 -1
  201. package/events/local_event_manager.d.ts.map +0 -1
  202. package/events/local_event_manager.js.map +0 -1
  203. package/http_clients/base-http-client.d.ts +0 -140
  204. package/http_clients/base-http-client.d.ts.map +0 -1
  205. package/http_clients/base-http-client.js +0 -40
  206. package/http_clients/base-http-client.js.map +0 -1
  207. package/http_clients/form-data-like.d.ts +0 -67
  208. package/http_clients/form-data-like.d.ts.map +0 -1
  209. package/http_clients/form-data-like.js +0 -5
  210. package/http_clients/form-data-like.js.map +0 -1
  211. package/http_clients/got-scraping-http-client.d.ts +0 -20
  212. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  213. package/http_clients/got-scraping-http-client.js +0 -85
  214. package/http_clients/got-scraping-http-client.js.map +0 -1
  215. package/http_clients/index.d.ts +0 -3
  216. package/http_clients/index.d.ts.map +0 -1
  217. package/http_clients/index.js +0 -3
  218. package/http_clients/index.js.map +0 -1
  219. package/index.d.ts.map +0 -1
  220. package/index.js.map +0 -1
  221. package/log.d.ts.map +0 -1
  222. package/log.js.map +0 -1
  223. package/proxy_configuration.d.ts.map +0 -1
  224. package/proxy_configuration.js.map +0 -1
  225. package/recoverable_state.d.ts.map +0 -1
  226. package/recoverable_state.js.map +0 -1
  227. package/request.d.ts.map +0 -1
  228. package/request.js.map +0 -1
  229. package/router.d.ts.map +0 -1
  230. package/router.js.map +0 -1
  231. package/serialization.d.ts.map +0 -1
  232. package/serialization.js.map +0 -1
  233. package/session_pool/consts.d.ts.map +0 -1
  234. package/session_pool/consts.js.map +0 -1
  235. package/session_pool/errors.d.ts.map +0 -1
  236. package/session_pool/errors.js.map +0 -1
  237. package/session_pool/events.d.ts +0 -3
  238. package/session_pool/events.d.ts.map +0 -1
  239. package/session_pool/events.js +0 -3
  240. package/session_pool/events.js.map +0 -1
  241. package/session_pool/index.d.ts.map +0 -1
  242. package/session_pool/index.js.map +0 -1
  243. package/session_pool/session.d.ts.map +0 -1
  244. package/session_pool/session.js.map +0 -1
  245. package/session_pool/session_pool.d.ts.map +0 -1
  246. package/session_pool/session_pool.js.map +0 -1
  247. package/storages/access_checking.d.ts +0 -13
  248. package/storages/access_checking.d.ts.map +0 -1
  249. package/storages/access_checking.js +0 -14
  250. package/storages/access_checking.js.map +0 -1
  251. package/storages/dataset.d.ts.map +0 -1
  252. package/storages/dataset.js.map +0 -1
  253. package/storages/index.d.ts.map +0 -1
  254. package/storages/index.js.map +0 -1
  255. package/storages/key_value_store.d.ts.map +0 -1
  256. package/storages/key_value_store.js.map +0 -1
  257. package/storages/request_list.d.ts.map +0 -1
  258. package/storages/request_list.js.map +0 -1
  259. package/storages/request_list_adapter.d.ts +0 -58
  260. package/storages/request_list_adapter.d.ts.map +0 -1
  261. package/storages/request_list_adapter.js +0 -81
  262. package/storages/request_list_adapter.js.map +0 -1
  263. package/storages/request_manager_tandem.d.ts.map +0 -1
  264. package/storages/request_manager_tandem.js.map +0 -1
  265. package/storages/request_provider.d.ts +0 -371
  266. package/storages/request_provider.d.ts.map +0 -1
  267. package/storages/request_provider.js +0 -585
  268. package/storages/request_provider.js.map +0 -1
  269. package/storages/request_queue.d.ts.map +0 -1
  270. package/storages/request_queue.js.map +0 -1
  271. package/storages/request_queue_v2.d.ts +0 -87
  272. package/storages/request_queue_v2.d.ts.map +0 -1
  273. package/storages/request_queue_v2.js +0 -438
  274. package/storages/request_queue_v2.js.map +0 -1
  275. package/storages/sitemap_request_list.d.ts.map +0 -1
  276. package/storages/sitemap_request_list.js +0 -430
  277. package/storages/sitemap_request_list.js.map +0 -1
  278. package/storages/storage_manager.d.ts +0 -58
  279. package/storages/storage_manager.d.ts.map +0 -1
  280. package/storages/storage_manager.js +0 -105
  281. package/storages/storage_manager.js.map +0 -1
  282. package/storages/utils.d.ts.map +0 -1
  283. package/storages/utils.js.map +0 -1
  284. package/typedefs.d.ts.map +0 -1
  285. package/typedefs.js.map +0 -1
  286. package/validators.d.ts.map +0 -1
  287. package/validators.js.map +0 -1
@@ -1,16 +1,45 @@
1
1
  import { URL } from 'node:url';
2
2
  import { Minimatch } from 'minimatch';
3
- import { purlToRegExp } from '@apify/pseudo_url';
3
+ import { z } from 'zod';
4
4
  import { Request } from '../request.js';
5
- export { tryAbsoluteURL } from '@crawlee/utils';
5
+ import { schemas } from '../validators.js';
6
6
  const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
7
7
  /**
8
- * To enable direct use of the Actor UI `globs`/`regexps`/`pseudoUrls` output while keeping high performance,
9
- * all the regexps from the output are only constructed once and kept in a cache
10
- * by the `enqueueLinks()` function.
8
+ * To keep high performance when the same patterns are passed on every `enqueueLinks()` call,
9
+ * each glob/regexp is only compiled once and then kept in a cache.
11
10
  * @ignore
12
11
  */
13
12
  const enqueueLinksPatternCache = new Map();
13
+ /**
14
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
15
+ * @internal
16
+ */
17
+ export const urlPatternSchema = z.union([
18
+ z.string(),
19
+ z.instanceof(RegExp),
20
+ schemas.objectWithKeys(['glob']),
21
+ schemas.objectWithKeys(['regexp']),
22
+ ]);
23
+ /**
24
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
25
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
26
+ * real callback and building a `Request` isn't free.
27
+ * @ignore
28
+ */
29
+ export function createSkippedRequestArgs(source, reason) {
30
+ const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
31
+ let request;
32
+ return {
33
+ reason: sourceReason ?? reason,
34
+ get request() {
35
+ request ??=
36
+ source instanceof Request
37
+ ? source
38
+ : new Request(typeof source === 'string' ? { url: source } : source);
39
+ return request;
40
+ },
41
+ };
42
+ }
14
43
  /**
15
44
  * @ignore
16
45
  */
@@ -21,28 +50,6 @@ export function updateEnqueueLinksPatternCache(item, pattern) {
21
50
  enqueueLinksPatternCache.delete(key);
22
51
  }
23
52
  }
24
- /**
25
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
26
- * to construct RegExps from PseudoUrl strings.
27
- * @ignore
28
- */
29
- export function constructRegExpObjectsFromPseudoUrls(pseudoUrls) {
30
- return pseudoUrls.map((item) => {
31
- // Get pseudoUrl object from cache.
32
- let regexpObject = enqueueLinksPatternCache.get(item);
33
- if (regexpObject)
34
- return regexpObject;
35
- if (typeof item === 'string') {
36
- regexpObject = { regexp: purlToRegExp(item) };
37
- }
38
- else {
39
- const { purl, ...requestOptions } = item;
40
- regexpObject = { regexp: purlToRegExp(purl), ...requestOptions };
41
- }
42
- updateEnqueueLinksPatternCache(item, regexpObject);
43
- return regexpObject;
44
- });
45
- }
46
53
  /**
47
54
  * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
48
55
  * to construct Glob objects from Glob pattern strings.
@@ -72,8 +79,7 @@ export function constructGlobObjectsFromGlobs(globs) {
72
79
  globObject = { glob: validateGlobPattern(item) };
73
80
  }
74
81
  else {
75
- const { glob, ...requestOptions } = item;
76
- globObject = { glob: validateGlobPattern(glob), ...requestOptions };
82
+ globObject = { glob: validateGlobPattern(item.glob) };
77
83
  }
78
84
  updateEnqueueLinksPatternCache(item, globObject);
79
85
  return globObject;
@@ -103,62 +109,70 @@ export function constructRegExpObjectsFromRegExps(regexps) {
103
109
  regexpObject = { regexp: item };
104
110
  }
105
111
  else {
106
- regexpObject = item;
112
+ regexpObject = { regexp: item.regexp };
107
113
  }
108
114
  updateEnqueueLinksPatternCache(item, regexpObject);
109
115
  return regexpObject;
110
116
  });
111
117
  }
112
118
  /**
119
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
120
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
113
121
  * @ignore
114
122
  */
115
- export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy, onSkippedUrl) {
116
- const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
117
- const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
123
+ export function constructUrlPatternObjects(patterns) {
124
+ const result = [];
125
+ for (const item of patterns) {
126
+ if (typeof item === 'string' || 'glob' in item) {
127
+ result.push(...constructGlobObjectsFromGlobs([item]));
128
+ }
129
+ else if (item instanceof RegExp || 'regexp' in item) {
130
+ result.push(...constructRegExpObjectsFromRegExps([item]));
131
+ }
132
+ }
133
+ return result;
134
+ }
135
+ /**
136
+ * Filters request options by URL patterns.
137
+ *
138
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
139
+ * @ignore
140
+ */
141
+ export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
142
+ const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
143
+ const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
118
144
  return requestOptions
119
- .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
120
- .filter(({ url }) => {
121
- const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
122
- if (matchesExcludePatterns) {
123
- onSkippedUrl?.(url);
145
+ .filter((opts) => {
146
+ const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
147
+ if (matchesExclude) {
148
+ onSkippedRequestOptions?.(opts);
124
149
  }
125
- return !matchesExcludePatterns;
150
+ return !matchesExclude;
126
151
  })
127
- .map(({ url, opts }) => {
128
- if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
129
- return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
130
- }
131
- for (const urlPatternObject of urlPatternObjectMatchers) {
132
- const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
133
- if (match(url)) {
134
- const request = typeof opts === 'string'
135
- ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
136
- : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
137
- return new Request(request);
152
+ .map((opts) => {
153
+ if (!includeMatchers) {
154
+ return { ...opts, enqueueStrategy: strategy };
155
+ }
156
+ for (const { match } of includeMatchers) {
157
+ if (match(opts.url)) {
158
+ return { ...opts, enqueueStrategy: strategy };
138
159
  }
139
160
  }
140
161
  // didn't match any positive pattern
141
- onSkippedUrl?.(url);
162
+ onSkippedRequestOptions?.(opts);
142
163
  return null;
143
164
  })
144
- .filter((request) => request);
165
+ .filter((opts) => opts !== null);
145
166
  }
146
- export function filterRequestsByPatterns(requests, patterns, onSkippedUrl) {
147
- if (!patterns?.length) {
148
- return requests;
167
+ function isAbsoluteUrl(url) {
168
+ try {
169
+ // eslint-disable-next-line no-new
170
+ new URL(url);
171
+ return true;
149
172
  }
150
- const filtered = [];
151
- const patternMatchers = patterns?.map(createPatternObjectMatcher);
152
- for (const request of requests) {
153
- const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
154
- if (matchingPattern !== undefined) {
155
- filtered.push(request);
156
- }
157
- else {
158
- onSkippedUrl?.(request.url);
159
- }
173
+ catch {
174
+ return false;
160
175
  }
161
- return filtered;
162
176
  }
163
177
  /**
164
178
  * @ignore
@@ -177,7 +191,12 @@ export function createRequestOptions(sources, options = {}) {
177
191
  }
178
192
  })
179
193
  .map((requestOptions) => {
180
- requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
194
+ // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
195
+ // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
196
+ // resolving against `baseUrl` in the first place.
197
+ if (!isAbsoluteUrl(requestOptions.url)) {
198
+ requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
199
+ }
181
200
  requestOptions.userData ??= options.userData ?? {};
182
201
  if (typeof options.label === 'string') {
183
202
  requestOptions.userData = {
@@ -188,6 +207,9 @@ export function createRequestOptions(sources, options = {}) {
188
207
  if (options.skipNavigation) {
189
208
  requestOptions.skipNavigation = true;
190
209
  }
210
+ if (options.sessionId) {
211
+ requestOptions.sessionId = options.sessionId;
212
+ }
191
213
  return requestOptions;
192
214
  });
193
215
  }
@@ -207,6 +229,31 @@ function createPatternObjectMatcher(urlPatternObject) {
207
229
  else {
208
230
  match = () => false;
209
231
  }
210
- return { ...urlPatternObject, match };
232
+ return { match };
233
+ }
234
+ /**
235
+ * Applies a {@link RequestTransform} function to a list of request options.
236
+ * Options for which the transform returns a falsy value are removed from the list.
237
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
238
+ * @ignore
239
+ * @internal
240
+ */
241
+ export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
242
+ return requestOptions
243
+ .map((opts) => {
244
+ const transformed = transformFn(opts);
245
+ if (transformed === 'skip') {
246
+ onSkipped?.(opts);
247
+ return null;
248
+ }
249
+ if (transformed === 'unchanged') {
250
+ return opts;
251
+ }
252
+ if (!transformed) {
253
+ onSkipped?.(opts);
254
+ return null;
255
+ }
256
+ return transformed;
257
+ })
258
+ .filter((r) => r !== null);
211
259
  }
212
- //# sourceMappingURL=shared.js.map
package/errors.d.ts CHANGED
@@ -14,6 +14,37 @@ export declare class CriticalError extends NonRetryableError {
14
14
  */
15
15
  export declare class MissingRouteError extends CriticalError {
16
16
  }
17
+ /**
18
+ * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
19
+ * error types do not have to depend on `@standard-schema/spec`.
20
+ */
21
+ export interface SchemaIssue {
22
+ readonly message: string;
23
+ readonly path?: readonly (PropertyKey | {
24
+ key: PropertyKey;
25
+ })[];
26
+ }
27
+ /**
28
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
29
+ *
30
+ * As the `userData` does not change between attempts, this error is non-retryable.
31
+ */
32
+ export declare class RequestValidationError extends NonRetryableError {
33
+ readonly label: string | symbol;
34
+ readonly issues: readonly SchemaIssue[];
35
+ constructor(label: string | symbol, issues: readonly SchemaIssue[]);
36
+ }
37
+ /**
38
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
39
+ *
40
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
41
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
42
+ */
43
+ export declare class StateValidationError extends Error {
44
+ readonly persistStateKey: string;
45
+ readonly issues: readonly SchemaIssue[];
46
+ constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
47
+ }
17
48
  /**
18
49
  * Errors of `RetryRequestError` type will always be retried by the crawler.
19
50
  *
@@ -23,13 +54,39 @@ export declare class RetryRequestError extends Error {
23
54
  constructor(message?: string);
24
55
  }
25
56
  /**
26
- * Errors of `SessionError` type will trigger a session rotation.
57
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
27
58
  *
28
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
59
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
60
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
61
+ * the domain's backoff expires, so retries are paced rather than immediate.
29
62
  */
30
- export declare class SessionError extends RetryRequestError {
63
+ export declare class RequestThrottledError extends RetryRequestError {
31
64
  constructor(message?: string);
32
65
  }
66
+ /**
67
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
68
+ * abandoned rather than kept waiting.
69
+ *
70
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
71
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
72
+ * resumes them once the domain recovers.
73
+ */
74
+ export declare class PersistentRateLimitError extends CriticalError {
75
+ }
76
+ /**
77
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
78
+ *
79
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
80
+ */
81
+ export declare class SessionError extends Error {
82
+ constructor(message?: string);
83
+ }
84
+ /**
85
+ * Thrown when a requested session is not found in the referenced SessionPool.
86
+ */
87
+ export declare class MissingSessionError extends Error {
88
+ constructor(sessionId?: string);
89
+ }
33
90
  export declare class ContextPipelineInterruptedError extends Error {
34
91
  constructor(message?: string);
35
92
  }
@@ -42,4 +99,15 @@ export declare class ContextPipelineCleanupError extends CriticalError {
42
99
  export declare class RequestHandlerError extends Error {
43
100
  constructor(error: unknown, options?: ErrorOptions);
44
101
  }
45
- //# sourceMappingURL=errors.d.ts.map
102
+ /**
103
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
104
+ */
105
+ export declare class ServiceConflictError extends Error {
106
+ constructor(serviceName: string, newValue: unknown, existingValue: unknown);
107
+ }
108
+ /**
109
+ * Thrown by crawlers when `skipNavigation` is used on a request.
110
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
111
+ */
112
+ export declare class NavigationSkippedError extends NonRetryableError {
113
+ }
package/errors.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { inspectValue } from './debug.js';
1
2
  /**
2
3
  * Errors of `NonRetryableError` type will never be retried by the crawler.
3
4
  */
@@ -14,6 +15,45 @@ export class CriticalError extends NonRetryableError {
14
15
  */
15
16
  export class MissingRouteError extends CriticalError {
16
17
  }
18
+ function formatIssues(issues) {
19
+ return issues
20
+ .map((issue) => {
21
+ const path = (issue.path ?? [])
22
+ .map((segment) => (typeof segment === 'object' ? segment.key : segment))
23
+ .join('.');
24
+ return `- ${path ? `${path}: ` : ''}${issue.message}`;
25
+ })
26
+ .join('\n');
27
+ }
28
+ /**
29
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
30
+ *
31
+ * As the `userData` does not change between attempts, this error is non-retryable.
32
+ */
33
+ export class RequestValidationError extends NonRetryableError {
34
+ label;
35
+ issues;
36
+ constructor(label, issues) {
37
+ super(`Request userData for label '${String(label)}' failed schema validation:\n${formatIssues(issues)}`);
38
+ this.label = label;
39
+ this.issues = issues;
40
+ }
41
+ }
42
+ /**
43
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
44
+ *
45
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
46
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
47
+ */
48
+ export class StateValidationError extends Error {
49
+ persistStateKey;
50
+ issues;
51
+ constructor(persistStateKey, issues) {
52
+ super(`State persisted under key '${persistStateKey}' failed schema validation:\n${formatIssues(issues)}`);
53
+ this.persistStateKey = persistStateKey;
54
+ this.issues = issues;
55
+ }
56
+ }
17
57
  /**
18
58
  * Errors of `RetryRequestError` type will always be retried by the crawler.
19
59
  *
@@ -25,13 +65,43 @@ export class RetryRequestError extends Error {
25
65
  }
26
66
  }
27
67
  /**
28
- * Errors of `SessionError` type will trigger a session rotation.
68
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
69
+ *
70
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
71
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
72
+ * the domain's backoff expires, so retries are paced rather than immediate.
73
+ */
74
+ export class RequestThrottledError extends RetryRequestError {
75
+ constructor(message) {
76
+ super(message ?? 'Request is being retried later because its domain is rate-limiting us');
77
+ }
78
+ }
79
+ /**
80
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
81
+ * abandoned rather than kept waiting.
82
+ *
83
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
84
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
85
+ * resumes them once the domain recovers.
86
+ */
87
+ export class PersistentRateLimitError extends CriticalError {
88
+ }
89
+ /**
90
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
29
91
  *
30
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
92
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
31
93
  */
32
- export class SessionError extends RetryRequestError {
94
+ export class SessionError extends Error {
33
95
  constructor(message) {
34
- super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
96
+ super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
97
+ }
98
+ }
99
+ /**
100
+ * Thrown when a requested session is not found in the referenced SessionPool.
101
+ */
102
+ export class MissingSessionError extends Error {
103
+ constructor(sessionId) {
104
+ super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
35
105
  }
36
106
  }
37
107
  export class ContextPipelineInterruptedError extends Error {
@@ -54,4 +124,18 @@ export class RequestHandlerError extends Error {
54
124
  super(undefined, { cause: error, ...options });
55
125
  }
56
126
  }
57
- //# sourceMappingURL=errors.js.map
127
+ /**
128
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
129
+ */
130
+ export class ServiceConflictError extends Error {
131
+ constructor(serviceName, newValue, existingValue) {
132
+ super(`Service ${serviceName} is already in use. ` +
133
+ `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
134
+ }
135
+ }
136
+ /**
137
+ * Thrown by crawlers when `skipNavigation` is used on a request.
138
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
139
+ */
140
+ export class NavigationSkippedError extends NonRetryableError {
141
+ }
@@ -1,28 +1,55 @@
1
1
  import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
2
2
  import type { BetterIntervalID } from '@apify/utilities';
3
- import { Configuration } from '../configuration.js';
4
- export declare const enum EventType {
3
+ export interface EventManagerOptions {
4
+ /** Interval between emitted `persistState` events in milliseconds. */
5
+ persistStateIntervalMillis: number;
6
+ }
7
+ export declare enum EventType {
5
8
  PERSIST_STATE = "persistState",
6
9
  SYSTEM_INFO = "systemInfo",
7
10
  MIGRATING = "migrating",
8
11
  ABORTING = "aborting",
9
- EXIT = "exit"
12
+ EXIT = "exit",
13
+ STATUS_MESSAGE = "statusMessage"
14
+ }
15
+ export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
16
+ /**
17
+ * Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
18
+ *
19
+ * The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
20
+ * start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
21
+ * to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
22
+ */
23
+ export interface EventStatusMessageData {
24
+ /**
25
+ * Identifies the crawler that emitted the message.
26
+ *
27
+ * Either the user-provided `id` from the crawler options, or a randomly generated one.
28
+ * Since a single event manager may be shared by multiple crawlers, consumers can use this
29
+ * to attribute the message to a specific crawler instance.
30
+ */
31
+ crawlerId: string;
32
+ /** The human-readable status message. */
33
+ message: string;
34
+ /** Whether this is the final status message of the run. */
35
+ isStatusMessageTerminal?: boolean;
36
+ /** The log level the message was logged with. */
37
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
10
38
  }
11
- export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
12
39
  interface Intervals {
13
40
  persistState?: BetterIntervalID;
14
41
  systemInfo?: BetterIntervalID;
15
42
  }
16
43
  export declare abstract class EventManager {
17
- readonly config: Configuration;
44
+ #private;
18
45
  protected events: AsyncEventEmitter<{}>;
19
46
  protected initialized: boolean;
20
47
  protected intervals: Intervals;
21
48
  // @ts-ignore optional peer dependency or compatibility with es2022
22
- protected log: import("@apify/log").Log;
23
- constructor(config?: Configuration);
49
+ protected log: import("@crawlee/types").CrawleeLogger;
50
+ constructor(options: EventManagerOptions);
24
51
  /**
25
- * Initializes the event manager by creating the `persistState` event interval.
52
+ * Initializes the event manager by starting the `persistState` event interval.
26
53
  * This is automatically called at the beginning of `crawler.run()`.
27
54
  */
28
55
  init(): Promise<void>;
@@ -49,4 +76,3 @@ export declare abstract class EventManager {
49
76
  waitForAllListenersToComplete(): Promise<boolean>;
50
77
  }
51
78
  export {};
52
- //# sourceMappingURL=event_manager.d.ts.map
@@ -1,7 +1,6 @@
1
1
  import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
2
- import log from '@apify/log';
3
2
  import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
3
+ import { serviceLocator } from '../service_locator.js';
5
4
  export var EventType;
6
5
  (function (EventType) {
7
6
  EventType["PERSIST_STATE"] = "persistState";
@@ -9,30 +8,31 @@ export var EventType;
9
8
  EventType["MIGRATING"] = "migrating";
10
9
  EventType["ABORTING"] = "aborting";
11
10
  EventType["EXIT"] = "exit";
11
+ EventType["STATUS_MESSAGE"] = "statusMessage";
12
12
  })(EventType || (EventType = {}));
13
13
  export class EventManager {
14
- config;
15
14
  events = new AsyncEventEmitter();
16
15
  initialized = false;
17
16
  intervals = {};
18
- log = log.child({ prefix: 'Events' });
19
- constructor(config = Configuration.getGlobalConfig()) {
20
- this.config = config;
21
- this.events.setMaxListeners(50);
17
+ log = serviceLocator.getLogger().child({ prefix: 'Events' });
18
+ #persistStateIntervalMillis;
19
+ constructor(options) {
20
+ this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
21
+ // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
22
+ this.events.setMaxListeners(150);
22
23
  }
23
24
  /**
24
- * Initializes the event manager by creating the `persistState` event interval.
25
+ * Initializes the event manager by starting the `persistState` event interval.
25
26
  * This is automatically called at the beginning of `crawler.run()`.
26
27
  */
27
28
  async init() {
28
29
  if (this.initialized) {
29
30
  return;
30
31
  }
31
- const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
32
32
  this.intervals.persistState = betterSetInterval((intervalCallback) => {
33
- this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
33
+ this.emit(EventType.PERSIST_STATE, { isMigrating: false });
34
34
  intervalCallback();
35
- }, persistStateIntervalMillis);
35
+ }, this.#persistStateIntervalMillis);
36
36
  this.initialized = true;
37
37
  }
38
38
  /**
@@ -46,7 +46,7 @@ export class EventManager {
46
46
  betterClearInterval(this.intervals.persistState);
47
47
  this.initialized = false;
48
48
  // Emit final PERSIST_STATE event
49
- this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
49
+ this.emit(EventType.PERSIST_STATE, { isMigrating: false });
50
50
  // Wait for PERSIST_STATE to process
51
51
  await this.waitForAllListenersToComplete();
52
52
  }
@@ -86,4 +86,3 @@ export class EventManager {
86
86
  return this.events.waitForAllListenersToComplete();
87
87
  }
88
88
  }
89
- //# sourceMappingURL=event_manager.js.map
package/events/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.d.ts.map
package/events/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from './event_manager.js';
2
2
  export * from './local_event_manager.js';
3
- //# sourceMappingURL=index.js.map
@@ -1,7 +1,19 @@
1
- import { EventManager } from './event_manager.js';
1
+ import type { Configuration } from '../configuration.js';
2
+ import { EventManager, type EventManagerOptions } from './event_manager.js';
3
+ export interface LocalEventManagerOptions extends EventManagerOptions {
4
+ /** Interval between emitted `systemInfo` events in milliseconds. */
5
+ systemInfoIntervalMillis: number;
6
+ }
2
7
  export declare class LocalEventManager extends EventManager {
8
+ #private;
9
+ constructor(options: LocalEventManagerOptions);
10
+ /**
11
+ * Creates a new `LocalEventManager` based on the provided `Configuration`.
12
+ * Uses the global configuration from the service locator if none is provided.
13
+ */
14
+ static fromConfiguration(configuration?: Configuration): LocalEventManager;
3
15
  /**
4
- * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
16
+ * Initializes the EventManager and sets up periodic `systemInfo` events.
5
17
  * This is automatically called at the beginning of `crawler.run()`.
6
18
  */
7
19
  init(): Promise<void>;
@@ -23,5 +35,5 @@ export declare class LocalEventManager extends EventManager {
23
35
  private createSystemInfo;
24
36
  private createCpuInfo;
25
37
  private createMemoryInfo;
38
+ private getMemoryInfo;
26
39
  }
27
- //# sourceMappingURL=local_event_manager.d.ts.map