@crawlee/core 4.0.0-beta.18 → 4.0.0-beta.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +90 -226
- package/configuration.js +164 -225
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/errors.d.ts +40 -19
- package/errors.js +53 -28
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +12 -13
- package/events/index.d.ts +1 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/events/system_info.d.ts +38 -0
- package/events/system_info.js +1 -0
- package/index.d.ts +7 -11
- package/index.js +8 -10
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +77 -1
- package/log.js +134 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +44 -0
- package/memory-storage/memory-storage.js +160 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +106 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +199 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +414 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +18 -19
- package/proxy_configuration.d.ts +22 -128
- package/proxy_configuration.js +32 -144
- package/recoverable_state.d.ts +85 -51
- package/recoverable_state.js +178 -72
- package/request.d.ts +60 -17
- package/request.js +130 -69
- package/serialization.d.ts +0 -1
- package/serialization.js +15 -15
- package/service_locator.d.ts +165 -0
- package/service_locator.js +254 -0
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +114 -56
- package/storages/dataset.js +292 -149
- package/storages/index.d.ts +7 -9
- package/storages/index.js +5 -9
- package/storages/key_value_store.d.ts +187 -50
- package/storages/key_value_store.js +453 -172
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +54 -129
- package/storages/request_list.js +272 -245
- package/storages/request_loader.d.ts +127 -0
- package/storages/request_loader.js +36 -0
- package/storages/request_manager.d.ts +109 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +50 -23
- package/storages/request_manager_tandem.js +132 -57
- package/storages/request_queue.d.ts +286 -48
- package/storages/request_queue.js +757 -218
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/transaction.d.ts +260 -0
- package/storages/transaction.js +287 -0
- package/storages/utils.d.ts +59 -11
- package/storages/utils.js +75 -15
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -18
- package/validators.js +14 -18
- package/autoscaling/autoscaled_pool.d.ts +0 -300
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js +0 -533
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts +0 -4
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js +0 -4
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts +0 -189
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js +0 -285
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts +0 -181
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js +0 -169
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts +0 -43
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js +0 -111
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts +0 -61
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js +0 -99
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts +0 -195
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js +0 -108
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -60
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js +0 -118
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts +0 -55
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js +0 -309
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts +0 -7
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js +0 -7
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts +0 -8
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js +0 -2
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts +0 -204
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js +0 -346
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts +0 -249
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js +0 -269
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts +0 -3
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js +0 -3
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts +0 -73
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js +0 -212
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -91
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -44
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -20
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -102
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts +0 -134
- package/router.d.ts.map +0 -1
- package/router.js +0 -164
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts +0 -4
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js +0 -4
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts +0 -8
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js +0 -12
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts +0 -6
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js +0 -6
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts +0 -203
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js +0 -278
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts +0 -258
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js +0 -387
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_list_adapter.d.ts +0 -58
- package/storages/request_list_adapter.d.ts.map +0 -1
- package/storages/request_list_adapter.js +0 -81
- package/storages/request_list_adapter.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_provider.d.ts +0 -371
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -585
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts +0 -245
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js +0 -430
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/request.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
|
-
import type { Dictionary } from '@crawlee/types';
|
|
3
|
-
import type {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { EnqueueStrategy } from '@crawlee/utils';
|
|
4
|
+
/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
|
|
5
|
+
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
|
|
6
|
+
export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
|
|
6
7
|
export declare enum RequestState {
|
|
7
8
|
UNPROCESSED = 0,
|
|
8
9
|
BEFORE_NAV = 1,
|
|
@@ -45,6 +46,7 @@ export declare enum RequestState {
|
|
|
45
46
|
* @category Sources
|
|
46
47
|
*/
|
|
47
48
|
declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
49
|
+
#private;
|
|
48
50
|
/** Request ID */
|
|
49
51
|
id?: string;
|
|
50
52
|
/** URL of the web page to crawl. */
|
|
@@ -75,9 +77,12 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
75
77
|
errorMessages: string[];
|
|
76
78
|
/** Object with HTTP headers. Key is header name, value is the value. */
|
|
77
79
|
headers?: Record<string, string>;
|
|
78
|
-
/**
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Custom user data assigned to the request.
|
|
82
|
+
*
|
|
83
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
84
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
85
|
+
*/
|
|
81
86
|
userData: UserData;
|
|
82
87
|
/**
|
|
83
88
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -93,9 +98,21 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
93
98
|
* @returns The native `fetch` API Request object.
|
|
94
99
|
*/
|
|
95
100
|
intoFetchAPIRequest(): Request;
|
|
96
|
-
/**
|
|
101
|
+
/**
|
|
102
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
103
|
+
*
|
|
104
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
105
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
106
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
107
|
+
*/
|
|
97
108
|
get skipNavigation(): boolean;
|
|
98
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
111
|
+
*
|
|
112
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
113
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
114
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
115
|
+
*/
|
|
99
116
|
set skipNavigation(value: boolean);
|
|
100
117
|
/**
|
|
101
118
|
* Depth of the request in the current crawl tree.
|
|
@@ -107,10 +124,9 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
107
124
|
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
108
125
|
*/
|
|
109
126
|
set crawlDepth(value: number);
|
|
110
|
-
/**
|
|
111
|
-
get
|
|
112
|
-
|
|
113
|
-
set sessionRotationCount(value: number);
|
|
127
|
+
/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
|
|
128
|
+
get sessionId(): string | undefined;
|
|
129
|
+
set sessionId(value: string | undefined);
|
|
114
130
|
/** shortcut for getting `request.userData.label` */
|
|
115
131
|
get label(): string | undefined;
|
|
116
132
|
/** shortcut for setting `request.userData.label` */
|
|
@@ -123,6 +139,14 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
123
139
|
get state(): RequestState;
|
|
124
140
|
/** Describes the request's current lifecycle state. */
|
|
125
141
|
set state(value: RequestState);
|
|
142
|
+
/**
|
|
143
|
+
* Reason for skipping this request.
|
|
144
|
+
*/
|
|
145
|
+
get skippedReason(): SkippedRequestReason | undefined;
|
|
146
|
+
/**
|
|
147
|
+
* Reason for skipping this request.
|
|
148
|
+
*/
|
|
149
|
+
set skippedReason(value: SkippedRequestReason | undefined);
|
|
126
150
|
private get enqueueStrategy();
|
|
127
151
|
private set enqueueStrategy(value);
|
|
128
152
|
/**
|
|
@@ -140,7 +164,7 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
140
164
|
*/
|
|
141
165
|
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
|
|
142
166
|
/** @internal */
|
|
143
|
-
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
|
|
167
|
+
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
|
|
144
168
|
/** @internal */
|
|
145
169
|
static hashPayload(payload: BinaryLike): string;
|
|
146
170
|
}
|
|
@@ -161,7 +185,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
161
185
|
* The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
|
|
162
186
|
*
|
|
163
187
|
* The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
|
|
164
|
-
* producing a `uniqueKey` in the following format: `METHOD
|
|
188
|
+
* producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
|
|
165
189
|
* when requests point to the same URL, but with different methods and payloads. For example: form submits.
|
|
166
190
|
*
|
|
167
191
|
* Pass an arbitrary non-empty text value to the `uniqueKey` property
|
|
@@ -185,6 +209,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
185
209
|
/**
|
|
186
210
|
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
187
211
|
* request's scope, keeping them accessible on retries, failures etc.
|
|
212
|
+
*
|
|
213
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
214
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
188
215
|
*/
|
|
189
216
|
userData?: UserData;
|
|
190
217
|
/**
|
|
@@ -206,14 +233,30 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
206
233
|
* @default false
|
|
207
234
|
*/
|
|
208
235
|
useExtendedUniqueKey?: boolean;
|
|
236
|
+
/**
|
|
237
|
+
* If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
|
|
238
|
+
* is always enqueued even if a request with the same URL (and method/payload) is already present
|
|
239
|
+
* in the queue. Cannot be used together with a custom `uniqueKey`.
|
|
240
|
+
* @default false
|
|
241
|
+
*/
|
|
242
|
+
alwaysEnqueue?: boolean;
|
|
209
243
|
/**
|
|
210
244
|
* The `true` value indicates that the request will not be automatically retried on error.
|
|
211
245
|
* @default false
|
|
212
246
|
*/
|
|
213
247
|
noRetry?: boolean;
|
|
248
|
+
/**
|
|
249
|
+
* ID of a session from the crawler's `SessionPool` to use for this request.
|
|
250
|
+
* When set, the crawler will fetch this session from the pool instead of creating a new one.
|
|
251
|
+
*/
|
|
252
|
+
sessionId?: string;
|
|
214
253
|
/**
|
|
215
254
|
* If set to `true` then the crawler processing this request evaluates
|
|
216
255
|
* the `requestHandler` immediately without prior browser navigation.
|
|
256
|
+
*
|
|
257
|
+
* When enabled, the crawling context will not contain the results of the navigation
|
|
258
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
259
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
217
260
|
* @default false
|
|
218
261
|
*/
|
|
219
262
|
skipNavigation?: boolean;
|
|
@@ -240,7 +283,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
240
283
|
/** @internal */
|
|
241
284
|
lockExpiresAt?: Date;
|
|
242
285
|
/** @internal */
|
|
243
|
-
enqueueStrategy?:
|
|
286
|
+
enqueueStrategy?: EnqueueStrategyOption;
|
|
244
287
|
}
|
|
245
288
|
export interface PushErrorMessageOptions {
|
|
246
289
|
/**
|
|
@@ -255,6 +298,7 @@ interface ComputeUniqueKeyOptions {
|
|
|
255
298
|
payload?: string | Buffer;
|
|
256
299
|
keepUrlFragment?: boolean;
|
|
257
300
|
useExtendedUniqueKey?: boolean;
|
|
301
|
+
alwaysEnqueue?: boolean;
|
|
258
302
|
}
|
|
259
303
|
export type Source = (Partial<RequestOptions> & {
|
|
260
304
|
requestsFromUrl?: string;
|
|
@@ -266,4 +310,3 @@ export interface InternalSource {
|
|
|
266
310
|
regex?: RegExp;
|
|
267
311
|
}
|
|
268
312
|
export { CrawleeRequest as Request };
|
|
269
|
-
//# sourceMappingURL=request.d.ts.map
|
package/request.js
CHANGED
|
@@ -1,32 +1,13 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import
|
|
4
|
-
import { normalizeUrl } from '@apify/utilities';
|
|
5
|
-
import {
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
|
|
5
|
+
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
|
-
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
loadedUrl: ow.optional.string.url,
|
|
12
|
-
uniqueKey: ow.optional.string,
|
|
13
|
-
method: ow.optional.string,
|
|
14
|
-
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
15
|
-
noRetry: ow.optional.boolean,
|
|
16
|
-
retryCount: ow.optional.number,
|
|
17
|
-
sessionRotationCount: ow.optional.number,
|
|
18
|
-
maxRetries: ow.optional.number,
|
|
19
|
-
errorMessages: ow.optional.array.ofType(ow.string),
|
|
20
|
-
headers: ow.optional.object,
|
|
21
|
-
userData: ow.optional.object,
|
|
22
|
-
label: ow.optional.string,
|
|
23
|
-
handledAt: ow.optional.any(ow.string.date, ow.date),
|
|
24
|
-
keepUrlFragment: ow.optional.boolean,
|
|
25
|
-
useExtendedUniqueKey: ow.optional.boolean,
|
|
26
|
-
skipNavigation: ow.optional.boolean,
|
|
27
|
-
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
28
|
-
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
29
|
-
};
|
|
7
|
+
import { parseArgument, schemas } from './validators.js';
|
|
8
|
+
const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
|
|
9
|
+
message: 'Invalid input: expected a date string',
|
|
10
|
+
});
|
|
30
11
|
export var RequestState;
|
|
31
12
|
(function (RequestState) {
|
|
32
13
|
RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
|
|
@@ -38,6 +19,34 @@ export var RequestState;
|
|
|
38
19
|
RequestState[RequestState["ERROR"] = 6] = "ERROR";
|
|
39
20
|
RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
|
|
40
21
|
})(RequestState || (RequestState = {}));
|
|
22
|
+
const requestUrlSchema = z.object({ url: z.string() });
|
|
23
|
+
// new properties on the Request object breaks serialization
|
|
24
|
+
const requestOptionalSchemaShapes = {
|
|
25
|
+
id: z.string().optional(),
|
|
26
|
+
loadedUrl: z.url().optional(),
|
|
27
|
+
uniqueKey: z.string().optional(),
|
|
28
|
+
method: z.string().optional(),
|
|
29
|
+
payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
|
|
30
|
+
noRetry: z.boolean().optional(),
|
|
31
|
+
retryCount: schemas.anyNumber.optional(),
|
|
32
|
+
sessionId: z.string().optional(),
|
|
33
|
+
maxRetries: schemas.anyNumber.optional(),
|
|
34
|
+
errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
|
|
35
|
+
headers: z.looseObject({}).optional(),
|
|
36
|
+
userData: z.looseObject({}).optional(),
|
|
37
|
+
label: z.string().optional(),
|
|
38
|
+
handledAt: z.union([dateString, z.date()]).optional(),
|
|
39
|
+
keepUrlFragment: z.boolean().optional(),
|
|
40
|
+
useExtendedUniqueKey: z.boolean().optional(),
|
|
41
|
+
alwaysEnqueue: z.boolean().optional(),
|
|
42
|
+
skipNavigation: z.boolean().optional(),
|
|
43
|
+
crawlDepth: schemas.anyNumber
|
|
44
|
+
.refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
|
|
45
|
+
.optional(),
|
|
46
|
+
state: z.enum(RequestState).optional(),
|
|
47
|
+
};
|
|
48
|
+
// Each schema is wrapped in a single-key object so validation errors carry the property name.
|
|
49
|
+
const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
|
|
41
50
|
/**
|
|
42
51
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
43
52
|
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
@@ -101,8 +110,13 @@ class CrawleeRequest {
|
|
|
101
110
|
/** Object with HTTP headers. Key is header name, value is the value. */
|
|
102
111
|
headers;
|
|
103
112
|
/** Private store for the custom user data assigned to the request. */
|
|
104
|
-
|
|
105
|
-
/**
|
|
113
|
+
#userData = {};
|
|
114
|
+
/**
|
|
115
|
+
* Custom user data assigned to the request.
|
|
116
|
+
*
|
|
117
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
118
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
119
|
+
*/
|
|
106
120
|
userData = {};
|
|
107
121
|
/**
|
|
108
122
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -113,76 +127,89 @@ class CrawleeRequest {
|
|
|
113
127
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
114
128
|
*/
|
|
115
129
|
constructor(options) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
130
|
+
// A bare URL is a common slip — point at the object form instead of a generic type error.
|
|
131
|
+
if (typeof options === 'string') {
|
|
132
|
+
throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
|
|
133
|
+
'Did you mean `new Request({ url })`?');
|
|
134
|
+
}
|
|
135
|
+
parseArgument(options, schemas.anyObject, 'RequestOptions');
|
|
136
|
+
parseArgument(options, requestUrlSchema, 'RequestOptions');
|
|
137
|
+
// Full-shape validation is slow, because it checks all predicates
|
|
119
138
|
// even if the validated object has only 1 property.
|
|
120
139
|
// This custom validation loop iterates only over existing
|
|
121
140
|
// properties and speeds up the validation cca 3-fold.
|
|
122
|
-
// See https://github.com/sindresorhus/ow/issues/193
|
|
123
141
|
keys(options).forEach((prop) => {
|
|
124
142
|
// skip url, because it is validated above
|
|
125
143
|
if (prop === 'url') {
|
|
126
144
|
return;
|
|
127
145
|
}
|
|
128
|
-
const
|
|
146
|
+
const schema = requestOptionalSchemas[prop];
|
|
129
147
|
const value = options[prop];
|
|
130
|
-
if (
|
|
131
|
-
|
|
148
|
+
if (schema) {
|
|
149
|
+
parseArgument({ [prop]: value }, schema, 'RequestOptions');
|
|
132
150
|
}
|
|
133
151
|
});
|
|
134
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0,
|
|
152
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
135
153
|
let { method = 'GET' } = options;
|
|
136
154
|
method = method.toUpperCase();
|
|
137
155
|
if (method === 'GET' && payload)
|
|
138
156
|
throw new Error('Request with GET method cannot have a payload.');
|
|
157
|
+
if (uniqueKey && alwaysEnqueue) {
|
|
158
|
+
throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
|
|
159
|
+
}
|
|
139
160
|
this.id = id;
|
|
140
161
|
this.url = url;
|
|
141
162
|
this.loadedUrl = loadedUrl;
|
|
142
163
|
this.uniqueKey =
|
|
143
164
|
uniqueKey ||
|
|
144
|
-
CrawleeRequest.computeUniqueKey({
|
|
165
|
+
CrawleeRequest.computeUniqueKey({
|
|
166
|
+
url,
|
|
167
|
+
method,
|
|
168
|
+
payload,
|
|
169
|
+
keepUrlFragment,
|
|
170
|
+
useExtendedUniqueKey,
|
|
171
|
+
alwaysEnqueue,
|
|
172
|
+
});
|
|
145
173
|
this.method = method;
|
|
146
174
|
this.payload = payload;
|
|
147
175
|
this.noRetry = noRetry;
|
|
148
176
|
this.retryCount = retryCount;
|
|
149
|
-
this.sessionRotationCount = sessionRotationCount;
|
|
150
177
|
this.errorMessages = [...errorMessages];
|
|
151
178
|
this.headers = { ...headers };
|
|
152
179
|
this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
|
|
153
180
|
if (label) {
|
|
154
181
|
userData.label = label;
|
|
155
182
|
}
|
|
183
|
+
// Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
|
|
184
|
+
// bag is non-enumerable, so the spread alone would silently drop the internal state
|
|
185
|
+
// (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
|
|
186
|
+
this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
|
|
187
|
+
// `userData` must stay an enumerable own accessor — serialization in the storages relies on it
|
|
156
188
|
Object.defineProperties(this, {
|
|
157
|
-
_userData: {
|
|
158
|
-
value: { __crawlee: {}, ...userData },
|
|
159
|
-
enumerable: false,
|
|
160
|
-
writable: true,
|
|
161
|
-
},
|
|
162
189
|
userData: {
|
|
163
|
-
get: () => this
|
|
190
|
+
get: () => this.#userData,
|
|
164
191
|
set: (value) => {
|
|
165
192
|
Object.defineProperties(value, {
|
|
166
193
|
__crawlee: {
|
|
167
|
-
value: this.
|
|
194
|
+
value: this.#userData.__crawlee,
|
|
168
195
|
enumerable: false,
|
|
169
196
|
writable: true,
|
|
170
197
|
},
|
|
171
198
|
toJSON: {
|
|
172
199
|
value: () => {
|
|
173
|
-
if (Object.keys(this.
|
|
200
|
+
if (Object.keys(this.#userData.__crawlee).length > 0) {
|
|
174
201
|
return {
|
|
175
|
-
...this
|
|
176
|
-
__crawlee: this.
|
|
202
|
+
...this.#userData,
|
|
203
|
+
__crawlee: this.#userData.__crawlee,
|
|
177
204
|
};
|
|
178
205
|
}
|
|
179
|
-
return this
|
|
206
|
+
return this.#userData;
|
|
180
207
|
},
|
|
181
208
|
enumerable: false,
|
|
182
209
|
writable: true,
|
|
183
210
|
},
|
|
184
211
|
});
|
|
185
|
-
this
|
|
212
|
+
this.#userData = value;
|
|
186
213
|
},
|
|
187
214
|
enumerable: true,
|
|
188
215
|
},
|
|
@@ -195,6 +222,8 @@ class CrawleeRequest {
|
|
|
195
222
|
this.maxRetries = maxRetries;
|
|
196
223
|
if (crawlDepth != null)
|
|
197
224
|
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
225
|
+
if (sessionId)
|
|
226
|
+
this.sessionId = sessionId;
|
|
198
227
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
199
228
|
if (enqueueStrategy) {
|
|
200
229
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
@@ -211,11 +240,23 @@ class CrawleeRequest {
|
|
|
211
240
|
body: this.payload,
|
|
212
241
|
});
|
|
213
242
|
}
|
|
214
|
-
/**
|
|
243
|
+
/**
|
|
244
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
245
|
+
*
|
|
246
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
247
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
248
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
249
|
+
*/
|
|
215
250
|
get skipNavigation() {
|
|
216
251
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
217
252
|
}
|
|
218
|
-
/**
|
|
253
|
+
/**
|
|
254
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
255
|
+
*
|
|
256
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
257
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
258
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
259
|
+
*/
|
|
219
260
|
set skipNavigation(value) {
|
|
220
261
|
if (!this.userData.__crawlee) {
|
|
221
262
|
this.userData.__crawlee = { skipNavigation: value };
|
|
@@ -239,18 +280,13 @@ class CrawleeRequest {
|
|
|
239
280
|
this.userData.__crawlee ??= {};
|
|
240
281
|
this.userData.__crawlee.crawlDepth = value;
|
|
241
282
|
}
|
|
242
|
-
/**
|
|
243
|
-
get
|
|
244
|
-
return this.userData.__crawlee?.
|
|
283
|
+
/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
|
|
284
|
+
get sessionId() {
|
|
285
|
+
return this.userData.__crawlee?.sessionId;
|
|
245
286
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
this.userData.__crawlee = { sessionRotationCount: value };
|
|
250
|
-
}
|
|
251
|
-
else {
|
|
252
|
-
this.userData.__crawlee.sessionRotationCount = value;
|
|
253
|
-
}
|
|
287
|
+
set sessionId(value) {
|
|
288
|
+
this.userData.__crawlee ??= {};
|
|
289
|
+
this.userData.__crawlee.sessionId = value;
|
|
254
290
|
}
|
|
255
291
|
/** shortcut for getting `request.userData.label` */
|
|
256
292
|
get label() {
|
|
@@ -286,6 +322,23 @@ class CrawleeRequest {
|
|
|
286
322
|
this.userData.__crawlee.state = value;
|
|
287
323
|
}
|
|
288
324
|
}
|
|
325
|
+
/**
|
|
326
|
+
* Reason for skipping this request.
|
|
327
|
+
*/
|
|
328
|
+
get skippedReason() {
|
|
329
|
+
return this.userData.__crawlee?.skippedReason;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Reason for skipping this request.
|
|
333
|
+
*/
|
|
334
|
+
set skippedReason(value) {
|
|
335
|
+
if (!this.userData.__crawlee) {
|
|
336
|
+
this.userData.__crawlee = { skippedReason: value };
|
|
337
|
+
}
|
|
338
|
+
else {
|
|
339
|
+
this.userData.__crawlee.skippedReason = value;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
289
342
|
get enqueueStrategy() {
|
|
290
343
|
return this.userData.__crawlee?.enqueueStrategy;
|
|
291
344
|
}
|
|
@@ -348,19 +401,28 @@ class CrawleeRequest {
|
|
|
348
401
|
this.errorMessages.push(message);
|
|
349
402
|
}
|
|
350
403
|
/** @internal */
|
|
351
|
-
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
|
|
404
|
+
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
|
|
352
405
|
const normalizedMethod = method.toUpperCase();
|
|
353
406
|
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
|
|
407
|
+
let uniqueKey;
|
|
354
408
|
if (!useExtendedUniqueKey) {
|
|
355
409
|
if (normalizedMethod !== 'GET' && payload) {
|
|
356
|
-
|
|
410
|
+
serviceLocator
|
|
411
|
+
.getLogger()
|
|
412
|
+
.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
|
|
357
413
|
'This is fine. Just letting you know that if your requests point to the same URL ' +
|
|
358
414
|
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
|
|
359
415
|
}
|
|
360
|
-
|
|
416
|
+
uniqueKey = normalizedUrl;
|
|
417
|
+
}
|
|
418
|
+
else {
|
|
419
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
420
|
+
uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
|
|
421
|
+
}
|
|
422
|
+
if (alwaysEnqueue) {
|
|
423
|
+
uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
|
|
361
424
|
}
|
|
362
|
-
|
|
363
|
-
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
425
|
+
return uniqueKey;
|
|
364
426
|
}
|
|
365
427
|
/** @internal */
|
|
366
428
|
static hashPayload(payload) {
|
|
@@ -368,4 +430,3 @@ class CrawleeRequest {
|
|
|
368
430
|
}
|
|
369
431
|
}
|
|
370
432
|
export { CrawleeRequest as Request };
|
|
371
|
-
//# sourceMappingURL=request.js.map
|
package/serialization.d.ts
CHANGED
package/serialization.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import zlib from 'node:zlib';
|
|
4
|
-
import ow from 'ow';
|
|
5
4
|
import StreamArray from 'stream-json/streamers/StreamArray.js';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import { parseArgument, schemas } from './validators.js';
|
|
6
7
|
const pipeline = util.promisify(streamPipeline);
|
|
8
|
+
const uint8ArraySchema = z.instanceof(Uint8Array);
|
|
7
9
|
/**
|
|
8
10
|
* Transforms an array of items to a JSON in a streaming
|
|
9
11
|
* fashion to save memory. It operates in batches to speed
|
|
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
|
|
|
11
13
|
* @internal
|
|
12
14
|
*/
|
|
13
15
|
class ArrayToJson extends Readable {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
#offset = 0;
|
|
17
|
+
#batchSize;
|
|
18
|
+
#data;
|
|
17
19
|
constructor(data, options = {}) {
|
|
18
20
|
super({
|
|
19
21
|
...options,
|
|
20
22
|
autoDestroy: true,
|
|
21
23
|
emitClose: true,
|
|
22
24
|
});
|
|
23
|
-
this.data = data;
|
|
24
25
|
const { batchSize = 10000 } = options;
|
|
25
|
-
this
|
|
26
|
-
this
|
|
26
|
+
this.#batchSize = batchSize;
|
|
27
|
+
this.#data = data;
|
|
27
28
|
this.push('[');
|
|
28
29
|
}
|
|
29
30
|
_read() {
|
|
30
31
|
try {
|
|
31
|
-
const items = this
|
|
32
|
+
const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
|
|
32
33
|
if (items.length) {
|
|
33
34
|
const json = JSON.stringify(items);
|
|
34
35
|
// Strip brackets to flatten the batch.
|
|
35
36
|
const itemString = json.substring(1, json.length - 1);
|
|
36
|
-
if (this
|
|
37
|
+
if (this.#offset > 0)
|
|
37
38
|
this.push(',', 'utf8');
|
|
38
39
|
this.push(itemString, 'utf8');
|
|
39
|
-
this
|
|
40
|
+
this.#offset += this.#batchSize;
|
|
40
41
|
}
|
|
41
42
|
else {
|
|
42
43
|
this.push(']');
|
|
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
|
|
|
58
59
|
* @internal
|
|
59
60
|
*/
|
|
60
61
|
export async function serializeArray(data) {
|
|
61
|
-
|
|
62
|
+
parseArgument(data, schemas.anyArray);
|
|
62
63
|
const { chunks, collector } = createChunkCollector();
|
|
63
64
|
await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
|
|
64
65
|
return Buffer.concat(chunks);
|
|
@@ -73,7 +74,7 @@ export async function serializeArray(data) {
|
|
|
73
74
|
* @internal
|
|
74
75
|
*/
|
|
75
76
|
export async function deserializeArray(compressedData) {
|
|
76
|
-
|
|
77
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
77
78
|
const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
|
|
78
79
|
await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
|
|
79
80
|
return chunks;
|
|
@@ -88,7 +89,7 @@ export async function deserializeArray(compressedData) {
|
|
|
88
89
|
* @internal
|
|
89
90
|
*/
|
|
90
91
|
export function createDeserialize(compressedData) {
|
|
91
|
-
|
|
92
|
+
parseArgument(compressedData, uint8ArraySchema);
|
|
92
93
|
const streamArray = StreamArray.withParser();
|
|
93
94
|
const destination = pluckValue(streamArray);
|
|
94
95
|
streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
|
|
@@ -116,7 +117,6 @@ function createChunkCollector(options = {}) {
|
|
|
116
117
|
}
|
|
117
118
|
function pluckValue(streamArray) {
|
|
118
119
|
const realPush = streamArray.push.bind(streamArray);
|
|
119
|
-
streamArray.push = (obj) => realPush(obj
|
|
120
|
+
streamArray.push = (obj) => realPush(obj?.value ?? null);
|
|
120
121
|
return streamArray;
|
|
121
122
|
}
|
|
122
|
-
//# sourceMappingURL=serialization.js.map
|