@crawlee/core 4.0.0-beta.18 → 4.0.0-beta.181

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/README.md +14 -14
  2. package/byte_utils.d.ts +17 -0
  3. package/byte_utils.js +42 -0
  4. package/configuration.d.ts +90 -226
  5. package/configuration.js +164 -225
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +40 -19
  9. package/errors.js +53 -28
  10. package/events/event_manager.d.ts +35 -9
  11. package/events/event_manager.js +12 -13
  12. package/events/index.d.ts +1 -1
  13. package/events/index.js +0 -1
  14. package/events/local_event_manager.d.ts +15 -3
  15. package/events/local_event_manager.js +39 -13
  16. package/events/system_info.d.ts +38 -0
  17. package/events/system_info.js +1 -0
  18. package/index.d.ts +7 -11
  19. package/index.js +8 -10
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +44 -0
  31. package/memory-storage/memory-storage.js +160 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  35. package/memory-storage/resource-clients/dataset.js +106 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  37. package/memory-storage/resource-clients/key-value-store.js +199 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  39. package/memory-storage/resource-clients/request-queue.js +414 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned_or_injected.d.ts +58 -0
  43. package/owned_or_injected.js +98 -0
  44. package/package.json +18 -19
  45. package/proxy_configuration.d.ts +22 -128
  46. package/proxy_configuration.js +32 -144
  47. package/recoverable_state.d.ts +85 -51
  48. package/recoverable_state.js +178 -72
  49. package/request.d.ts +60 -17
  50. package/request.js +130 -69
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +15 -15
  53. package/service_locator.d.ts +165 -0
  54. package/service_locator.js +254 -0
  55. package/storages/batched_adds.d.ts +37 -0
  56. package/storages/batched_adds.js +73 -0
  57. package/storages/dataset.d.ts +114 -56
  58. package/storages/dataset.js +292 -149
  59. package/storages/index.d.ts +7 -9
  60. package/storages/index.js +5 -9
  61. package/storages/key_value_store.d.ts +187 -50
  62. package/storages/key_value_store.js +453 -172
  63. package/storages/key_value_store_codec.d.ts +32 -0
  64. package/storages/key_value_store_codec.js +113 -0
  65. package/storages/request_dedup_cache.d.ts +22 -0
  66. package/storages/request_dedup_cache.js +48 -0
  67. package/storages/request_list.d.ts +54 -129
  68. package/storages/request_list.js +272 -245
  69. package/storages/request_loader.d.ts +127 -0
  70. package/storages/request_loader.js +36 -0
  71. package/storages/request_manager.d.ts +109 -0
  72. package/storages/request_manager.js +1 -0
  73. package/storages/request_manager_tandem.d.ts +50 -23
  74. package/storages/request_manager_tandem.js +132 -57
  75. package/storages/request_queue.d.ts +286 -48
  76. package/storages/request_queue.js +757 -218
  77. package/storages/storage_instance_manager.d.ts +87 -0
  78. package/storages/storage_instance_manager.js +256 -0
  79. package/storages/storage_stats.d.ts +48 -0
  80. package/storages/storage_stats.js +29 -0
  81. package/storages/transaction.d.ts +260 -0
  82. package/storages/transaction.js +287 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +75 -15
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -195
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -204
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -346
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js.map +0 -1
  173. package/http_clients/base-http-client.d.ts +0 -91
  174. package/http_clients/base-http-client.d.ts.map +0 -1
  175. package/http_clients/base-http-client.js +0 -44
  176. package/http_clients/base-http-client.js.map +0 -1
  177. package/http_clients/form-data-like.d.ts +0 -67
  178. package/http_clients/form-data-like.d.ts.map +0 -1
  179. package/http_clients/form-data-like.js +0 -5
  180. package/http_clients/form-data-like.js.map +0 -1
  181. package/http_clients/got-scraping-http-client.d.ts +0 -20
  182. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  183. package/http_clients/got-scraping-http-client.js +0 -102
  184. package/http_clients/got-scraping-http-client.js.map +0 -1
  185. package/http_clients/index.d.ts +0 -3
  186. package/http_clients/index.d.ts.map +0 -1
  187. package/http_clients/index.js +0 -3
  188. package/http_clients/index.js.map +0 -1
  189. package/index.d.ts.map +0 -1
  190. package/index.js.map +0 -1
  191. package/log.d.ts.map +0 -1
  192. package/log.js.map +0 -1
  193. package/proxy_configuration.d.ts.map +0 -1
  194. package/proxy_configuration.js.map +0 -1
  195. package/recoverable_state.d.ts.map +0 -1
  196. package/recoverable_state.js.map +0 -1
  197. package/request.d.ts.map +0 -1
  198. package/request.js.map +0 -1
  199. package/router.d.ts +0 -134
  200. package/router.d.ts.map +0 -1
  201. package/router.js +0 -164
  202. package/router.js.map +0 -1
  203. package/serialization.d.ts.map +0 -1
  204. package/serialization.js.map +0 -1
  205. package/session_pool/consts.d.ts +0 -4
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js +0 -4
  208. package/session_pool/consts.js.map +0 -1
  209. package/session_pool/errors.d.ts +0 -8
  210. package/session_pool/errors.d.ts.map +0 -1
  211. package/session_pool/errors.js +0 -12
  212. package/session_pool/errors.js.map +0 -1
  213. package/session_pool/events.d.ts +0 -3
  214. package/session_pool/events.d.ts.map +0 -1
  215. package/session_pool/events.js +0 -3
  216. package/session_pool/events.js.map +0 -1
  217. package/session_pool/index.d.ts +0 -6
  218. package/session_pool/index.d.ts.map +0 -1
  219. package/session_pool/index.js +0 -6
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts +0 -203
  222. package/session_pool/session.d.ts.map +0 -1
  223. package/session_pool/session.js +0 -278
  224. package/session_pool/session.js.map +0 -1
  225. package/session_pool/session_pool.d.ts +0 -258
  226. package/session_pool/session_pool.d.ts.map +0 -1
  227. package/session_pool/session_pool.js +0 -387
  228. package/session_pool/session_pool.js.map +0 -1
  229. package/storages/access_checking.d.ts +0 -13
  230. package/storages/access_checking.d.ts.map +0 -1
  231. package/storages/access_checking.js +0 -14
  232. package/storages/access_checking.js.map +0 -1
  233. package/storages/dataset.d.ts.map +0 -1
  234. package/storages/dataset.js.map +0 -1
  235. package/storages/index.d.ts.map +0 -1
  236. package/storages/index.js.map +0 -1
  237. package/storages/key_value_store.d.ts.map +0 -1
  238. package/storages/key_value_store.js.map +0 -1
  239. package/storages/request_list.d.ts.map +0 -1
  240. package/storages/request_list.js.map +0 -1
  241. package/storages/request_list_adapter.d.ts +0 -58
  242. package/storages/request_list_adapter.d.ts.map +0 -1
  243. package/storages/request_list_adapter.js +0 -81
  244. package/storages/request_list_adapter.js.map +0 -1
  245. package/storages/request_manager_tandem.d.ts.map +0 -1
  246. package/storages/request_manager_tandem.js.map +0 -1
  247. package/storages/request_provider.d.ts +0 -371
  248. package/storages/request_provider.d.ts.map +0 -1
  249. package/storages/request_provider.js +0 -585
  250. package/storages/request_provider.js.map +0 -1
  251. package/storages/request_queue.d.ts.map +0 -1
  252. package/storages/request_queue.js.map +0 -1
  253. package/storages/request_queue_v2.d.ts +0 -87
  254. package/storages/request_queue_v2.d.ts.map +0 -1
  255. package/storages/request_queue_v2.js +0 -438
  256. package/storages/request_queue_v2.js.map +0 -1
  257. package/storages/sitemap_request_list.d.ts +0 -245
  258. package/storages/sitemap_request_list.d.ts.map +0 -1
  259. package/storages/sitemap_request_list.js +0 -430
  260. package/storages/sitemap_request_list.js.map +0 -1
  261. package/storages/storage_manager.d.ts +0 -58
  262. package/storages/storage_manager.d.ts.map +0 -1
  263. package/storages/storage_manager.js +0 -105
  264. package/storages/storage_manager.js.map +0 -1
  265. package/storages/utils.d.ts.map +0 -1
  266. package/storages/utils.js.map +0 -1
  267. package/typedefs.d.ts.map +0 -1
  268. package/typedefs.js.map +0 -1
  269. package/validators.d.ts.map +0 -1
  270. package/validators.js.map +0 -1
package/request.d.ts CHANGED
@@ -1,8 +1,9 @@
1
1
  import type { BinaryLike } from 'node:crypto';
2
- import type { Dictionary } from '@crawlee/types';
3
- import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
4
- import type { SkippedRequestReason } from './enqueue_links/shared.js';
5
- import type { AllowedHttpMethods } from './typedefs.js';
2
+ import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
3
+ import type { EnqueueStrategy } from '@crawlee/utils';
4
+ /** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
5
+ export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
6
+ export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
6
7
  export declare enum RequestState {
7
8
  UNPROCESSED = 0,
8
9
  BEFORE_NAV = 1,
@@ -45,6 +46,7 @@ export declare enum RequestState {
45
46
  * @category Sources
46
47
  */
47
48
  declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
49
+ #private;
48
50
  /** Request ID */
49
51
  id?: string;
50
52
  /** URL of the web page to crawl. */
@@ -75,9 +77,12 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
75
77
  errorMessages: string[];
76
78
  /** Object with HTTP headers. Key is header name, value is the value. */
77
79
  headers?: Record<string, string>;
78
- /** Private store for the custom user data assigned to the request. */
79
- private _userData;
80
- /** Custom user data assigned to the request. */
80
+ /**
81
+ * Custom user data assigned to the request.
82
+ *
83
+ * All data stored in `userData` must be JSON-serializable.
84
+ * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
85
+ */
81
86
  userData: UserData;
82
87
  /**
83
88
  * ISO datetime string that indicates the time when the request has been processed.
@@ -93,9 +98,21 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
93
98
  * @returns The native `fetch` API Request object.
94
99
  */
95
100
  intoFetchAPIRequest(): Request;
96
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
101
+ /**
102
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
103
+ *
104
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
105
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
106
+ * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
107
+ */
97
108
  get skipNavigation(): boolean;
98
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
109
+ /**
110
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
111
+ *
112
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
113
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
114
+ * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
115
+ */
99
116
  set skipNavigation(value: boolean);
100
117
  /**
101
118
  * Depth of the request in the current crawl tree.
@@ -107,10 +124,9 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
107
124
  * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
108
125
  */
109
126
  set crawlDepth(value: number);
110
- /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
111
- get sessionRotationCount(): number;
112
- /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
113
- set sessionRotationCount(value: number);
127
+ /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
128
+ get sessionId(): string | undefined;
129
+ set sessionId(value: string | undefined);
114
130
  /** shortcut for getting `request.userData.label` */
115
131
  get label(): string | undefined;
116
132
  /** shortcut for setting `request.userData.label` */
@@ -123,6 +139,14 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
123
139
  get state(): RequestState;
124
140
  /** Describes the request's current lifecycle state. */
125
141
  set state(value: RequestState);
142
+ /**
143
+ * Reason for skipping this request.
144
+ */
145
+ get skippedReason(): SkippedRequestReason | undefined;
146
+ /**
147
+ * Reason for skipping this request.
148
+ */
149
+ set skippedReason(value: SkippedRequestReason | undefined);
126
150
  private get enqueueStrategy();
127
151
  private set enqueueStrategy(value);
128
152
  /**
@@ -140,7 +164,7 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
140
164
  */
141
165
  pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
142
166
  /** @internal */
143
- static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
167
+ static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
144
168
  /** @internal */
145
169
  static hashPayload(payload: BinaryLike): string;
146
170
  }
@@ -161,7 +185,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
161
185
  * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
162
186
  *
163
187
  * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
164
- * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
188
+ * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
165
189
  * when requests point to the same URL, but with different methods and payloads. For example: form submits.
166
190
  *
167
191
  * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -185,6 +209,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
185
209
  /**
186
210
  * Custom user data assigned to the request. Use this to save any request related data to the
187
211
  * request's scope, keeping them accessible on retries, failures etc.
212
+ *
213
+ * All data stored in `userData` must be JSON-serializable.
214
+ * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
188
215
  */
189
216
  userData?: UserData;
190
217
  /**
@@ -206,14 +233,30 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
206
233
  * @default false
207
234
  */
208
235
  useExtendedUniqueKey?: boolean;
236
+ /**
237
+ * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
238
+ * is always enqueued even if a request with the same URL (and method/payload) is already present
239
+ * in the queue. Cannot be used together with a custom `uniqueKey`.
240
+ * @default false
241
+ */
242
+ alwaysEnqueue?: boolean;
209
243
  /**
210
244
  * The `true` value indicates that the request will not be automatically retried on error.
211
245
  * @default false
212
246
  */
213
247
  noRetry?: boolean;
248
+ /**
249
+ * ID of a session from the crawler's `SessionPool` to use for this request.
250
+ * When set, the crawler will fetch this session from the pool instead of creating a new one.
251
+ */
252
+ sessionId?: string;
214
253
  /**
215
254
  * If set to `true` then the crawler processing this request evaluates
216
255
  * the `requestHandler` immediately without prior browser navigation.
256
+ *
257
+ * When enabled, the crawling context will not contain the results of the navigation
258
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
259
+ * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
217
260
  * @default false
218
261
  */
219
262
  skipNavigation?: boolean;
@@ -240,7 +283,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
240
283
  /** @internal */
241
284
  lockExpiresAt?: Date;
242
285
  /** @internal */
243
- enqueueStrategy?: EnqueueLinksOptions['strategy'];
286
+ enqueueStrategy?: EnqueueStrategyOption;
244
287
  }
245
288
  export interface PushErrorMessageOptions {
246
289
  /**
@@ -255,6 +298,7 @@ interface ComputeUniqueKeyOptions {
255
298
  payload?: string | Buffer;
256
299
  keepUrlFragment?: boolean;
257
300
  useExtendedUniqueKey?: boolean;
301
+ alwaysEnqueue?: boolean;
258
302
  }
259
303
  export type Source = (Partial<RequestOptions> & {
260
304
  requestsFromUrl?: string;
@@ -266,4 +310,3 @@ export interface InternalSource {
266
310
  regex?: RegExp;
267
311
  }
268
312
  export { CrawleeRequest as Request };
269
- //# sourceMappingURL=request.d.ts.map
package/request.js CHANGED
@@ -1,32 +1,13 @@
1
1
  import crypto from 'node:crypto';
2
2
  import util from 'node:util';
3
- import ow from 'ow';
4
- import { normalizeUrl } from '@apify/utilities';
5
- import { log as defaultLog } from './log.js';
3
+ import { z } from 'zod';
4
+ import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
5
+ import { serviceLocator } from './service_locator.js';
6
6
  import { keys } from './typedefs.js';
7
- // new properties on the Request object breaks serialization
8
- const log = defaultLog.child({ prefix: 'Request' });
9
- const requestOptionalPredicates = {
10
- id: ow.optional.string,
11
- loadedUrl: ow.optional.string.url,
12
- uniqueKey: ow.optional.string,
13
- method: ow.optional.string,
14
- payload: ow.optional.any(ow.string, ow.uint8Array),
15
- noRetry: ow.optional.boolean,
16
- retryCount: ow.optional.number,
17
- sessionRotationCount: ow.optional.number,
18
- maxRetries: ow.optional.number,
19
- errorMessages: ow.optional.array.ofType(ow.string),
20
- headers: ow.optional.object,
21
- userData: ow.optional.object,
22
- label: ow.optional.string,
23
- handledAt: ow.optional.any(ow.string.date, ow.date),
24
- keepUrlFragment: ow.optional.boolean,
25
- useExtendedUniqueKey: ow.optional.boolean,
26
- skipNavigation: ow.optional.boolean,
27
- crawlDepth: ow.optional.number.greaterThanOrEqual(0),
28
- state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
29
- };
7
+ import { parseArgument, schemas } from './validators.js';
8
+ const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
9
+ message: 'Invalid input: expected a date string',
10
+ });
30
11
  export var RequestState;
31
12
  (function (RequestState) {
32
13
  RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
@@ -38,6 +19,34 @@ export var RequestState;
38
19
  RequestState[RequestState["ERROR"] = 6] = "ERROR";
39
20
  RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
40
21
  })(RequestState || (RequestState = {}));
22
+ const requestUrlSchema = z.object({ url: z.string() });
23
+ // new properties on the Request object breaks serialization
24
+ const requestOptionalSchemaShapes = {
25
+ id: z.string().optional(),
26
+ loadedUrl: z.url().optional(),
27
+ uniqueKey: z.string().optional(),
28
+ method: z.string().optional(),
29
+ payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
30
+ noRetry: z.boolean().optional(),
31
+ retryCount: schemas.anyNumber.optional(),
32
+ sessionId: z.string().optional(),
33
+ maxRetries: schemas.anyNumber.optional(),
34
+ errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
35
+ headers: z.looseObject({}).optional(),
36
+ userData: z.looseObject({}).optional(),
37
+ label: z.string().optional(),
38
+ handledAt: z.union([dateString, z.date()]).optional(),
39
+ keepUrlFragment: z.boolean().optional(),
40
+ useExtendedUniqueKey: z.boolean().optional(),
41
+ alwaysEnqueue: z.boolean().optional(),
42
+ skipNavigation: z.boolean().optional(),
43
+ crawlDepth: schemas.anyNumber
44
+ .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
45
+ .optional(),
46
+ state: z.enum(RequestState).optional(),
47
+ };
48
+ // Each schema is wrapped in a single-key object so validation errors carry the property name.
49
+ const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
41
50
  /**
42
51
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
43
52
  * The `Request` object also stores information about errors that occurred during processing of the request.
@@ -101,8 +110,13 @@ class CrawleeRequest {
101
110
  /** Object with HTTP headers. Key is header name, value is the value. */
102
111
  headers;
103
112
  /** Private store for the custom user data assigned to the request. */
104
- _userData = {};
105
- /** Custom user data assigned to the request. */
113
+ #userData = {};
114
+ /**
115
+ * Custom user data assigned to the request.
116
+ *
117
+ * All data stored in `userData` must be JSON-serializable.
118
+ * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
119
+ */
106
120
  userData = {};
107
121
  /**
108
122
  * ISO datetime string that indicates the time when the request has been processed.
@@ -113,76 +127,89 @@ class CrawleeRequest {
113
127
  * `Request` parameters including the URL, HTTP method and headers, and others.
114
128
  */
115
129
  constructor(options) {
116
- ow(options, 'RequestOptions', ow.object);
117
- ow(options.url, 'RequestOptions.url', ow.string);
118
- // 'ow' validation is slow, because it checks all predicates
130
+ // A bare URL is a common slip — point at the object form instead of a generic type error.
131
+ if (typeof options === 'string') {
132
+ throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
133
+ 'Did you mean `new Request({ url })`?');
134
+ }
135
+ parseArgument(options, schemas.anyObject, 'RequestOptions');
136
+ parseArgument(options, requestUrlSchema, 'RequestOptions');
137
+ // Full-shape validation is slow, because it checks all predicates
119
138
  // even if the validated object has only 1 property.
120
139
  // This custom validation loop iterates only over existing
121
140
  // properties and speeds up the validation cca 3-fold.
122
- // See https://github.com/sindresorhus/ow/issues/193
123
141
  keys(options).forEach((prop) => {
124
142
  // skip url, because it is validated above
125
143
  if (prop === 'url') {
126
144
  return;
127
145
  }
128
- const predicate = requestOptionalPredicates[prop];
146
+ const schema = requestOptionalSchemas[prop];
129
147
  const value = options[prop];
130
- if (predicate) {
131
- ow(value, `RequestOptions.${prop}`, predicate);
148
+ if (schema) {
149
+ parseArgument({ [prop]: value }, schema, 'RequestOptions');
132
150
  }
133
151
  });
134
- const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
152
+ const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
135
153
  let { method = 'GET' } = options;
136
154
  method = method.toUpperCase();
137
155
  if (method === 'GET' && payload)
138
156
  throw new Error('Request with GET method cannot have a payload.');
157
+ if (uniqueKey && alwaysEnqueue) {
158
+ throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
159
+ }
139
160
  this.id = id;
140
161
  this.url = url;
141
162
  this.loadedUrl = loadedUrl;
142
163
  this.uniqueKey =
143
164
  uniqueKey ||
144
- CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
165
+ CrawleeRequest.computeUniqueKey({
166
+ url,
167
+ method,
168
+ payload,
169
+ keepUrlFragment,
170
+ useExtendedUniqueKey,
171
+ alwaysEnqueue,
172
+ });
145
173
  this.method = method;
146
174
  this.payload = payload;
147
175
  this.noRetry = noRetry;
148
176
  this.retryCount = retryCount;
149
- this.sessionRotationCount = sessionRotationCount;
150
177
  this.errorMessages = [...errorMessages];
151
178
  this.headers = { ...headers };
152
179
  this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
153
180
  if (label) {
154
181
  userData.label = label;
155
182
  }
183
+ // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
184
+ // bag is non-enumerable, so the spread alone would silently drop the internal state
185
+ // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
186
+ this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
187
+ // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
156
188
  Object.defineProperties(this, {
157
- _userData: {
158
- value: { __crawlee: {}, ...userData },
159
- enumerable: false,
160
- writable: true,
161
- },
162
189
  userData: {
163
- get: () => this._userData,
190
+ get: () => this.#userData,
164
191
  set: (value) => {
165
192
  Object.defineProperties(value, {
166
193
  __crawlee: {
167
- value: this._userData.__crawlee,
194
+ value: this.#userData.__crawlee,
168
195
  enumerable: false,
169
196
  writable: true,
170
197
  },
171
198
  toJSON: {
172
199
  value: () => {
173
- if (Object.keys(this._userData.__crawlee).length > 0) {
200
+ if (Object.keys(this.#userData.__crawlee).length > 0) {
174
201
  return {
175
- ...this._userData,
176
- __crawlee: this._userData.__crawlee,
202
+ ...this.#userData,
203
+ __crawlee: this.#userData.__crawlee,
177
204
  };
178
205
  }
179
- return this._userData;
206
+ return this.#userData;
180
207
  },
181
208
  enumerable: false,
182
209
  writable: true,
183
210
  },
184
211
  });
185
- this._userData = value;
212
+ this.#userData = value;
186
213
  },
187
214
  enumerable: true,
188
215
  },
@@ -195,6 +222,8 @@ class CrawleeRequest {
195
222
  this.maxRetries = maxRetries;
196
223
  if (crawlDepth != null)
197
224
  this.userData.__crawlee.crawlDepth ??= crawlDepth;
225
+ if (sessionId)
226
+ this.sessionId = sessionId;
198
227
  // If it's already set, don't override it (for instance when fetching from storage)
199
228
  if (enqueueStrategy) {
200
229
  this.enqueueStrategy ??= enqueueStrategy;
@@ -211,11 +240,23 @@ class CrawleeRequest {
211
240
  body: this.payload,
212
241
  });
213
242
  }
214
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
243
+ /**
244
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
245
+ *
246
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
247
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
248
+ * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
249
+ */
215
250
  get skipNavigation() {
216
251
  return this.userData.__crawlee?.skipNavigation ?? false;
217
252
  }
218
- /** Tells the crawler processing this request to skip the navigation and process the request directly. */
253
+ /**
254
+ * Tells the crawler processing this request to skip the navigation and process the request directly.
255
+ *
256
+ * When this is set to `true`, the crawling context will not contain the results of the navigation
257
+ * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
258
+ * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
259
+ */
219
260
  set skipNavigation(value) {
220
261
  if (!this.userData.__crawlee) {
221
262
  this.userData.__crawlee = { skipNavigation: value };
@@ -239,18 +280,13 @@ class CrawleeRequest {
239
280
  this.userData.__crawlee ??= {};
240
281
  this.userData.__crawlee.crawlDepth = value;
241
282
  }
242
- /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
243
- get sessionRotationCount() {
244
- return this.userData.__crawlee?.sessionRotationCount ?? 0;
283
+ /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
284
+ get sessionId() {
285
+ return this.userData.__crawlee?.sessionId;
245
286
  }
246
- /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
247
- set sessionRotationCount(value) {
248
- if (!this.userData.__crawlee) {
249
- this.userData.__crawlee = { sessionRotationCount: value };
250
- }
251
- else {
252
- this.userData.__crawlee.sessionRotationCount = value;
253
- }
287
+ set sessionId(value) {
288
+ this.userData.__crawlee ??= {};
289
+ this.userData.__crawlee.sessionId = value;
254
290
  }
255
291
  /** shortcut for getting `request.userData.label` */
256
292
  get label() {
@@ -286,6 +322,23 @@ class CrawleeRequest {
286
322
  this.userData.__crawlee.state = value;
287
323
  }
288
324
  }
325
+ /**
326
+ * Reason for skipping this request.
327
+ */
328
+ get skippedReason() {
329
+ return this.userData.__crawlee?.skippedReason;
330
+ }
331
+ /**
332
+ * Reason for skipping this request.
333
+ */
334
+ set skippedReason(value) {
335
+ if (!this.userData.__crawlee) {
336
+ this.userData.__crawlee = { skippedReason: value };
337
+ }
338
+ else {
339
+ this.userData.__crawlee.skippedReason = value;
340
+ }
341
+ }
289
342
  get enqueueStrategy() {
290
343
  return this.userData.__crawlee?.enqueueStrategy;
291
344
  }
@@ -348,19 +401,28 @@ class CrawleeRequest {
348
401
  this.errorMessages.push(message);
349
402
  }
350
403
  /** @internal */
351
- static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
404
+ static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
352
405
  const normalizedMethod = method.toUpperCase();
353
406
  const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
407
+ let uniqueKey;
354
408
  if (!useExtendedUniqueKey) {
355
409
  if (normalizedMethod !== 'GET' && payload) {
356
- log.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
410
+ serviceLocator
411
+ .getLogger()
412
+ .warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
357
413
  'This is fine. Just letting you know that if your requests point to the same URL ' +
358
414
  'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
359
415
  }
360
- return normalizedUrl;
416
+ uniqueKey = normalizedUrl;
417
+ }
418
+ else {
419
+ const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
420
+ uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
421
+ }
422
+ if (alwaysEnqueue) {
423
+ uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
361
424
  }
362
- const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
363
- return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
425
+ return uniqueKey;
364
426
  }
365
427
  /** @internal */
366
428
  static hashPayload(payload) {
@@ -368,4 +430,3 @@ class CrawleeRequest {
368
430
  }
369
431
  }
370
432
  export { CrawleeRequest as Request };
371
- //# sourceMappingURL=request.js.map
@@ -29,4 +29,3 @@ export declare function deserializeArray<T extends string | Buffer>(compressedDa
29
29
  * @internal
30
30
  */
31
31
  export declare function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
32
- //# sourceMappingURL=serialization.d.ts.map
package/serialization.js CHANGED
@@ -1,9 +1,11 @@
1
1
  import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import zlib from 'node:zlib';
4
- import ow from 'ow';
5
4
  import StreamArray from 'stream-json/streamers/StreamArray.js';
5
+ import { z } from 'zod';
6
+ import { parseArgument, schemas } from './validators.js';
6
7
  const pipeline = util.promisify(streamPipeline);
8
+ const uint8ArraySchema = z.instanceof(Uint8Array);
7
9
  /**
8
10
  * Transforms an array of items to a JSON in a streaming
9
11
  * fashion to save memory. It operates in batches to speed
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
11
13
  * @internal
12
14
  */
13
15
  class ArrayToJson extends Readable {
14
- data;
15
- offset = 0;
16
- batchSize;
16
+ #offset = 0;
17
+ #batchSize;
18
+ #data;
17
19
  constructor(data, options = {}) {
18
20
  super({
19
21
  ...options,
20
22
  autoDestroy: true,
21
23
  emitClose: true,
22
24
  });
23
- this.data = data;
24
25
  const { batchSize = 10000 } = options;
25
- this.batchSize = batchSize;
26
- this.data = data;
26
+ this.#batchSize = batchSize;
27
+ this.#data = data;
27
28
  this.push('[');
28
29
  }
29
30
  _read() {
30
31
  try {
31
- const items = this.data.slice(this.offset, this.offset + this.batchSize);
32
+ const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
32
33
  if (items.length) {
33
34
  const json = JSON.stringify(items);
34
35
  // Strip brackets to flatten the batch.
35
36
  const itemString = json.substring(1, json.length - 1);
36
- if (this.offset > 0)
37
+ if (this.#offset > 0)
37
38
  this.push(',', 'utf8');
38
39
  this.push(itemString, 'utf8');
39
- this.offset += this.batchSize;
40
+ this.#offset += this.#batchSize;
40
41
  }
41
42
  else {
42
43
  this.push(']');
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
58
59
  * @internal
59
60
  */
60
61
  export async function serializeArray(data) {
61
- ow(data, ow.array);
62
+ parseArgument(data, schemas.anyArray);
62
63
  const { chunks, collector } = createChunkCollector();
63
64
  await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
64
65
  return Buffer.concat(chunks);
@@ -73,7 +74,7 @@ export async function serializeArray(data) {
73
74
  * @internal
74
75
  */
75
76
  export async function deserializeArray(compressedData) {
76
- ow(compressedData, ow.uint8Array);
77
+ parseArgument(compressedData, uint8ArraySchema);
77
78
  const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
78
79
  await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
79
80
  return chunks;
@@ -88,7 +89,7 @@ export async function deserializeArray(compressedData) {
88
89
  * @internal
89
90
  */
90
91
  export function createDeserialize(compressedData) {
91
- ow(compressedData, ow.uint8Array);
92
+ parseArgument(compressedData, uint8ArraySchema);
92
93
  const streamArray = StreamArray.withParser();
93
94
  const destination = pluckValue(streamArray);
94
95
  streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
@@ -116,7 +117,6 @@ function createChunkCollector(options = {}) {
116
117
  }
117
118
  function pluckValue(streamArray) {
118
119
  const realPush = streamArray.push.bind(streamArray);
119
- streamArray.push = (obj) => realPush(obj && obj.value);
120
+ streamArray.push = (obj) => realPush(obj?.value ?? null);
120
121
  return streamArray;
121
122
  }
122
- //# sourceMappingURL=serialization.js.map