@crawlee/core 4.0.0-beta.3 → 4.0.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +15 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +5 -78
- package/configuration.d.ts.map +1 -1
- package/configuration.js +6 -102
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +8 -8
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +61 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +99 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +14 -22
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -18
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +30 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +41 -23
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +24 -7
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +66 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +18 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +29 -0
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +27 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +155 -2
- package/log.d.ts.map +1 -1
- package/log.js +111 -0
- package/log.js.map +1 -1
- package/package.json +6 -7
- package/proxy_configuration.d.ts +17 -94
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +18 -54
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +137 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +39 -5
- package/request.d.ts.map +1 -1
- package/request.js +56 -15
- package/request.js.map +1 -1
- package/service_locator.d.ts +130 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +290 -0
- package/service_locator.js.map +1 -0
- package/session_pool/session.d.ts +9 -22
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +17 -5
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +15 -10
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +23 -13
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +9 -2
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +16 -6
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +13 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +17 -12
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +11 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +76 -9
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +92 -54
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -1
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -2
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +8 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +10 -8
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +12 -22
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -3
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/request.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
|
|
4
|
+
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
4
5
|
import type { AllowedHttpMethods } from './typedefs.js';
|
|
5
6
|
export declare enum RequestState {
|
|
6
7
|
UNPROCESSED = 0,
|
|
@@ -43,7 +44,7 @@ export declare enum RequestState {
|
|
|
43
44
|
* ```
|
|
44
45
|
* @category Sources
|
|
45
46
|
*/
|
|
46
|
-
|
|
47
|
+
declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
47
48
|
/** Request ID */
|
|
48
49
|
id?: string;
|
|
49
50
|
/** URL of the web page to crawl. */
|
|
@@ -87,10 +88,25 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
87
88
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
88
89
|
*/
|
|
89
90
|
constructor(options: RequestOptions<UserData>);
|
|
91
|
+
/**
|
|
92
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
93
|
+
* @returns The native `fetch` API Request object.
|
|
94
|
+
*/
|
|
95
|
+
intoFetchAPIRequest(): Request;
|
|
90
96
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
91
97
|
get skipNavigation(): boolean;
|
|
92
98
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
93
99
|
set skipNavigation(value: boolean);
|
|
100
|
+
/**
|
|
101
|
+
* Depth of the request in the current crawl tree.
|
|
102
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
103
|
+
*/
|
|
104
|
+
get crawlDepth(): number;
|
|
105
|
+
/**
|
|
106
|
+
* Depth of the request in the current crawl tree.
|
|
107
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
108
|
+
*/
|
|
109
|
+
set crawlDepth(value: number);
|
|
94
110
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
95
111
|
get sessionRotationCount(): number;
|
|
96
112
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
@@ -107,6 +123,14 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
107
123
|
get state(): RequestState;
|
|
108
124
|
/** Describes the request's current lifecycle state. */
|
|
109
125
|
set state(value: RequestState);
|
|
126
|
+
/**
|
|
127
|
+
* Reason for skipping this request.
|
|
128
|
+
*/
|
|
129
|
+
get skippedReason(): SkippedRequestReason | undefined;
|
|
130
|
+
/**
|
|
131
|
+
* Reason for skipping this request.
|
|
132
|
+
*/
|
|
133
|
+
set skippedReason(value: SkippedRequestReason | undefined);
|
|
110
134
|
private get enqueueStrategy();
|
|
111
135
|
private set enqueueStrategy(value);
|
|
112
136
|
/**
|
|
@@ -123,8 +147,6 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
123
147
|
* @param [options]
|
|
124
148
|
*/
|
|
125
149
|
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
|
|
126
|
-
protected _computeUniqueKey(options: ComputeUniqueKeyOptions): string;
|
|
127
|
-
protected _hashPayload(payload: BinaryLike): string;
|
|
128
150
|
/** @internal */
|
|
129
151
|
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
|
|
130
152
|
/** @internal */
|
|
@@ -203,6 +225,18 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
203
225
|
* @default false
|
|
204
226
|
*/
|
|
205
227
|
skipNavigation?: boolean;
|
|
228
|
+
/**
|
|
229
|
+
* Depth of the request in the current crawl tree.
|
|
230
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
231
|
+
* @default 0
|
|
232
|
+
*/
|
|
233
|
+
crawlDepth?: number;
|
|
234
|
+
/**
|
|
235
|
+
* Reason for skipping this request.
|
|
236
|
+
* This is used to provide more information about why the request was skipped.
|
|
237
|
+
* @internal
|
|
238
|
+
*/
|
|
239
|
+
skippedReason?: SkippedRequestReason;
|
|
206
240
|
/**
|
|
207
241
|
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
|
|
208
242
|
*/
|
|
@@ -233,11 +267,11 @@ interface ComputeUniqueKeyOptions {
|
|
|
233
267
|
export type Source = (Partial<RequestOptions> & {
|
|
234
268
|
requestsFromUrl?: string;
|
|
235
269
|
regex?: RegExp;
|
|
236
|
-
}) |
|
|
270
|
+
}) | CrawleeRequest;
|
|
237
271
|
/** @internal */
|
|
238
272
|
export interface InternalSource {
|
|
239
273
|
requestsFromUrl: string;
|
|
240
274
|
regex?: RegExp;
|
|
241
275
|
}
|
|
242
|
-
export {};
|
|
276
|
+
export { CrawleeRequest as Request };
|
|
243
277
|
//# sourceMappingURL=request.d.ts.map
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA0BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAEpD;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,EAMxD;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAmB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/request.js
CHANGED
|
@@ -2,10 +2,9 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import ow from 'ow';
|
|
4
4
|
import { normalizeUrl } from '@apify/utilities';
|
|
5
|
-
import {
|
|
5
|
+
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
7
|
// new properties on the Request object breaks serialization
|
|
8
|
-
const log = defaultLog.child({ prefix: 'Request' });
|
|
9
8
|
const requestOptionalPredicates = {
|
|
10
9
|
id: ow.optional.string,
|
|
11
10
|
loadedUrl: ow.optional.string.url,
|
|
@@ -24,6 +23,7 @@ const requestOptionalPredicates = {
|
|
|
24
23
|
keepUrlFragment: ow.optional.boolean,
|
|
25
24
|
useExtendedUniqueKey: ow.optional.boolean,
|
|
26
25
|
skipNavigation: ow.optional.boolean,
|
|
26
|
+
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
27
27
|
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
28
28
|
};
|
|
29
29
|
export var RequestState;
|
|
@@ -68,7 +68,7 @@ export var RequestState;
|
|
|
68
68
|
* ```
|
|
69
69
|
* @category Sources
|
|
70
70
|
*/
|
|
71
|
-
|
|
71
|
+
class CrawleeRequest {
|
|
72
72
|
/** Request ID */
|
|
73
73
|
id;
|
|
74
74
|
/** URL of the web page to crawl. */
|
|
@@ -130,7 +130,7 @@ export class Request {
|
|
|
130
130
|
ow(value, `RequestOptions.${prop}`, predicate);
|
|
131
131
|
}
|
|
132
132
|
});
|
|
133
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
|
|
133
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
134
134
|
let { method = 'GET' } = options;
|
|
135
135
|
method = method.toUpperCase();
|
|
136
136
|
if (method === 'GET' && payload)
|
|
@@ -139,7 +139,8 @@ export class Request {
|
|
|
139
139
|
this.url = url;
|
|
140
140
|
this.loadedUrl = loadedUrl;
|
|
141
141
|
this.uniqueKey =
|
|
142
|
-
uniqueKey ||
|
|
142
|
+
uniqueKey ||
|
|
143
|
+
CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
|
|
143
144
|
this.method = method;
|
|
144
145
|
this.payload = payload;
|
|
145
146
|
this.noRetry = noRetry;
|
|
@@ -191,11 +192,24 @@ export class Request {
|
|
|
191
192
|
this.skipNavigation = skipNavigation;
|
|
192
193
|
if (maxRetries != null)
|
|
193
194
|
this.maxRetries = maxRetries;
|
|
195
|
+
if (crawlDepth != null)
|
|
196
|
+
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
194
197
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
195
198
|
if (enqueueStrategy) {
|
|
196
199
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
197
200
|
}
|
|
198
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
204
|
+
* @returns The native `fetch` API Request object.
|
|
205
|
+
*/
|
|
206
|
+
intoFetchAPIRequest() {
|
|
207
|
+
return new Request(this.url, {
|
|
208
|
+
method: this.method,
|
|
209
|
+
headers: this.headers,
|
|
210
|
+
body: this.payload,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
199
213
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
200
214
|
get skipNavigation() {
|
|
201
215
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
@@ -209,6 +223,21 @@ export class Request {
|
|
|
209
223
|
this.userData.__crawlee.skipNavigation = value;
|
|
210
224
|
}
|
|
211
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* Depth of the request in the current crawl tree.
|
|
228
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
229
|
+
*/
|
|
230
|
+
get crawlDepth() {
|
|
231
|
+
return this.userData.__crawlee?.crawlDepth ?? 0;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Depth of the request in the current crawl tree.
|
|
235
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
236
|
+
*/
|
|
237
|
+
set crawlDepth(value) {
|
|
238
|
+
this.userData.__crawlee ??= {};
|
|
239
|
+
this.userData.__crawlee.crawlDepth = value;
|
|
240
|
+
}
|
|
212
241
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
213
242
|
get sessionRotationCount() {
|
|
214
243
|
return this.userData.__crawlee?.sessionRotationCount ?? 0;
|
|
@@ -256,6 +285,23 @@ export class Request {
|
|
|
256
285
|
this.userData.__crawlee.state = value;
|
|
257
286
|
}
|
|
258
287
|
}
|
|
288
|
+
/**
|
|
289
|
+
* Reason for skipping this request.
|
|
290
|
+
*/
|
|
291
|
+
get skippedReason() {
|
|
292
|
+
return this.userData.__crawlee?.skippedReason;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Reason for skipping this request.
|
|
296
|
+
*/
|
|
297
|
+
set skippedReason(value) {
|
|
298
|
+
if (!this.userData.__crawlee) {
|
|
299
|
+
this.userData.__crawlee = { skippedReason: value };
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
this.userData.__crawlee.skippedReason = value;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
259
305
|
get enqueueStrategy() {
|
|
260
306
|
return this.userData.__crawlee?.enqueueStrategy;
|
|
261
307
|
}
|
|
@@ -317,27 +363,21 @@ export class Request {
|
|
|
317
363
|
}
|
|
318
364
|
this.errorMessages.push(message);
|
|
319
365
|
}
|
|
320
|
-
// TODO: only for better BC, remove in v4
|
|
321
|
-
_computeUniqueKey(options) {
|
|
322
|
-
return Request.computeUniqueKey(options);
|
|
323
|
-
}
|
|
324
|
-
// TODO: only for better BC, remove in v4
|
|
325
|
-
_hashPayload(payload) {
|
|
326
|
-
return Request.hashPayload(payload);
|
|
327
|
-
}
|
|
328
366
|
/** @internal */
|
|
329
367
|
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
|
|
330
368
|
const normalizedMethod = method.toUpperCase();
|
|
331
369
|
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
|
|
332
370
|
if (!useExtendedUniqueKey) {
|
|
333
371
|
if (normalizedMethod !== 'GET' && payload) {
|
|
334
|
-
|
|
372
|
+
serviceLocator
|
|
373
|
+
.getLogger()
|
|
374
|
+
.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
|
|
335
375
|
'This is fine. Just letting you know that if your requests point to the same URL ' +
|
|
336
376
|
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
|
|
337
377
|
}
|
|
338
378
|
return normalizedUrl;
|
|
339
379
|
}
|
|
340
|
-
const payloadHash = payload ?
|
|
380
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
341
381
|
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
342
382
|
}
|
|
343
383
|
/** @internal */
|
|
@@ -345,4 +385,5 @@ export class Request {
|
|
|
345
385
|
return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
|
|
346
386
|
}
|
|
347
387
|
}
|
|
388
|
+
export { CrawleeRequest as Request };
|
|
348
389
|
//# sourceMappingURL=request.js.map
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACxC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C,gDAAgD;IAChD,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,oBAAoB,GAAG,CAAC,EACxB,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAE1E,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACzD,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAuC;QACrD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACvE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,GAAG,KAAK,CAAC;QAClD,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GACN;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,cAAc;qBACT,SAAS,EAAE;qBACX,WAAW,CACR,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACV,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AA8ID,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import type { StorageClient } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from './configuration.js';
|
|
3
|
+
import type { EventManager } from './events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from './log.js';
|
|
5
|
+
import type { IStorage, StorageManager } from './storages/storage_manager.js';
|
|
6
|
+
import type { Constructor } from './typedefs.js';
|
|
7
|
+
interface ServiceLocatorInterface {
|
|
8
|
+
/**
|
|
9
|
+
* Get the configuration.
|
|
10
|
+
* Creates a default Configuration instance if none has been set.
|
|
11
|
+
*/
|
|
12
|
+
getConfiguration(): Configuration;
|
|
13
|
+
/**
|
|
14
|
+
* Set the configuration.
|
|
15
|
+
*
|
|
16
|
+
* @param configuration The configuration to set
|
|
17
|
+
* @throws {ServiceConflictError} If a different configuration has already been retrieved
|
|
18
|
+
*/
|
|
19
|
+
setConfiguration(configuration: Configuration): void;
|
|
20
|
+
/**
|
|
21
|
+
* Get the event manager.
|
|
22
|
+
* Creates a default LocalEventManager instance if none has been set.
|
|
23
|
+
*/
|
|
24
|
+
getEventManager(): EventManager;
|
|
25
|
+
/**
|
|
26
|
+
* Set the event manager.
|
|
27
|
+
*
|
|
28
|
+
* @param eventManager The event manager to set
|
|
29
|
+
* @throws {ServiceConflictError} If a different event manager has already been retrieved
|
|
30
|
+
*/
|
|
31
|
+
setEventManager(eventManager: EventManager): void;
|
|
32
|
+
/**
|
|
33
|
+
* Get the storage client.
|
|
34
|
+
* Creates a default MemoryStorage instance if none has been set.
|
|
35
|
+
*/
|
|
36
|
+
getStorageClient(): StorageClient;
|
|
37
|
+
/**
|
|
38
|
+
* Set the storage client.
|
|
39
|
+
*
|
|
40
|
+
* @param storageClient The storage client to set
|
|
41
|
+
* @throws {ServiceConflictError} If a different storage client has already been retrieved
|
|
42
|
+
*/
|
|
43
|
+
setStorageClient(storageClient: StorageClient): void;
|
|
44
|
+
/**
|
|
45
|
+
* Get the logger.
|
|
46
|
+
* Returns the default `@apify/log` logger if none has been set.
|
|
47
|
+
*/
|
|
48
|
+
getLogger(): CrawleeLogger;
|
|
49
|
+
/**
|
|
50
|
+
* Set the logger.
|
|
51
|
+
*
|
|
52
|
+
* @param logger The logger to set
|
|
53
|
+
* @throws {ServiceConflictError} If a different logger has already been retrieved
|
|
54
|
+
*/
|
|
55
|
+
setLogger(logger: CrawleeLogger): void;
|
|
56
|
+
/**
|
|
57
|
+
* Get a child logger with the given prefix.
|
|
58
|
+
* Equivalent to `getLogger().child({ prefix })`.
|
|
59
|
+
*/
|
|
60
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
61
|
+
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
62
|
+
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
63
|
+
/**
|
|
64
|
+
* Clears all storage manager caches.
|
|
65
|
+
* @internal
|
|
66
|
+
*/
|
|
67
|
+
clearStorageManagerCache(): void;
|
|
68
|
+
/**
|
|
69
|
+
* Resets the service locator to its initial state.
|
|
70
|
+
* Used mainly for testing purposes.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
reset(): void;
|
|
74
|
+
}
|
|
75
|
+
export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
76
|
+
private configuration?;
|
|
77
|
+
private eventManager?;
|
|
78
|
+
private storageClient?;
|
|
79
|
+
private logger?;
|
|
80
|
+
/**
|
|
81
|
+
* Storage managers for Dataset, KeyValueStore, and RequestQueue.
|
|
82
|
+
* Manages caching and lifecycle of storage instances.
|
|
83
|
+
*/
|
|
84
|
+
private storageManagers;
|
|
85
|
+
/**
|
|
86
|
+
* Creates a new ServiceLocator instance.
|
|
87
|
+
*
|
|
88
|
+
* @param configuration Optional configuration instance to use
|
|
89
|
+
* @param eventManager Optional event manager instance to use
|
|
90
|
+
* @param storageClient Optional storage client instance to use
|
|
91
|
+
* @param logger Optional logger instance to use
|
|
92
|
+
*/
|
|
93
|
+
constructor(configuration?: Configuration, eventManager?: EventManager, storageClient?: StorageClient, logger?: CrawleeLogger);
|
|
94
|
+
getConfiguration(): Configuration;
|
|
95
|
+
setConfiguration(configuration: Configuration): void;
|
|
96
|
+
getEventManager(): EventManager;
|
|
97
|
+
setEventManager(eventManager: EventManager): void;
|
|
98
|
+
getStorageClient(): StorageClient;
|
|
99
|
+
setStorageClient(storageClient: StorageClient): void;
|
|
100
|
+
getLogger(): CrawleeLogger;
|
|
101
|
+
setLogger(logger: CrawleeLogger): void;
|
|
102
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
103
|
+
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
104
|
+
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
105
|
+
clearStorageManagerCache(): void;
|
|
106
|
+
reset(): void;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Wraps all methods on `target` so that any code they invoke will see the given
|
|
110
|
+
* `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
|
|
111
|
+
*
|
|
112
|
+
* Walks the prototype chain and replaces each method on the *instance* (not the prototype)
|
|
113
|
+
* with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
|
|
114
|
+
*
|
|
115
|
+
* The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
|
|
116
|
+
* wrapped method — including `super` calls, since the prototype methods execute within the
|
|
117
|
+
* context established by the instance-level wrapper.
|
|
118
|
+
*
|
|
119
|
+
* @internal
|
|
120
|
+
* @returns Scope control functions: `run` executes a callback within the scoped context,
|
|
121
|
+
* `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
|
|
122
|
+
*/
|
|
123
|
+
export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
|
|
124
|
+
run: <T>(fn: () => T) => T;
|
|
125
|
+
enterScope: () => void;
|
|
126
|
+
exitScope: () => void;
|
|
127
|
+
};
|
|
128
|
+
export declare const serviceLocator: ServiceLocatorInterface;
|
|
129
|
+
export {};
|
|
130
|
+
//# sourceMappingURL=service_locator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service_locator.d.ts","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,UAAU,uBAAuB;IAC7B;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,eAAe,IAAI,YAAY,CAAC;IAEhC;;;;;OAKG;IACH,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,SAAS,IAAI,aAAa,CAAC;IAE3B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAEvC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC;IAE3C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS,CAAC;IAElF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IAE5F;;;OAGG;IACH,wBAAwB,IAAI,IAAI,CAAC;IAEjC;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACjB;AAsCD,qBAAa,cAAe,YAAW,uBAAuB;IAC1D,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,MAAM,CAAC,CAAgB;IAE/B;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAoD;IAE3E;;;;;;;OAOG;gBAEC,aAAa,CAAC,EAAE,aAAa,EAC7B,YAAY,CAAC,EAAE,YAAY,EAC3B,aAAa,CAAC,EAAE,aAAa,EAC7B,MAAM,CAAC,EAAE,aAAa;IAQ1B,gBAAgB,IAAI,aAAa;IAUjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,eAAe,IAAI,YAAY;IAgB/B,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAcjD,gBAAgB,IAAI,aAAa;IAmBjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,SAAS,IAAI,aAAa;IAO1B,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAYtC,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa;IAI1C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS;IAIjF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI;IAY3F,wBAAwB,IAAI,IAAI;IAchC,KAAK,IAAI,IAAI;CAOhB;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,CACvC,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,EAAE,GACX;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,IAAI,CAAA;CAAE,CA4C/E;AAED,eAAO,MAAM,cAAc,EAAE,uBAqD5B,CAAC"}
|