@crawlee/http 4.0.0-beta.63 → 4.0.0-beta.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -51,9 +51,9 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
|
|
|
51
51
|
*
|
|
52
52
|
* `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
53
53
|
*
|
|
54
|
-
* The source URLs are represented using {@link Request} objects that are fed from {@link
|
|
54
|
+
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
55
55
|
*
|
|
56
|
-
*
|
|
56
|
+
* > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
57
57
|
*
|
|
58
58
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
59
59
|
*
|
|
@@ -71,9 +71,9 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
71
71
|
*
|
|
72
72
|
* `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
73
73
|
*
|
|
74
|
-
* The source URLs are represented using {@link Request} objects that are fed from {@link
|
|
74
|
+
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
75
75
|
*
|
|
76
|
-
*
|
|
76
|
+
* > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
77
77
|
*
|
|
78
78
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
79
79
|
*
|
|
@@ -20,6 +20,9 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
20
20
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
21
21
|
* or browser properties before navigation. The function accepts one parameter `crawlingContext`,
|
|
22
22
|
* which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
|
|
23
|
+
*
|
|
24
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
25
|
+
* allowing the hook to override context members for subsequent hooks and pipeline stages.
|
|
23
26
|
* Example:
|
|
24
27
|
* ```
|
|
25
28
|
* preNavigationHooks: [
|
|
@@ -28,24 +31,26 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
28
31
|
* },
|
|
29
32
|
* ]
|
|
30
33
|
* ```
|
|
31
|
-
*
|
|
32
|
-
* Modyfing `pageOptions` is supported only in Playwright incognito.
|
|
33
|
-
* See {@link PrePageCreateHook}
|
|
34
34
|
*/
|
|
35
35
|
preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
|
|
36
36
|
/**
|
|
37
37
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
38
38
|
* The function accepts `crawlingContext` as the only parameter.
|
|
39
|
+
*
|
|
40
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
41
|
+
* which is useful for overriding the `response` after solving a challenge or re-fetching the resource.
|
|
39
42
|
* Example:
|
|
40
43
|
* ```
|
|
41
44
|
* postNavigationHooks: [
|
|
42
45
|
* async (crawlingContext) => {
|
|
43
|
-
*
|
|
46
|
+
* if (await needsRevalidation(crawlingContext)) {
|
|
47
|
+
* return { response: await refetch(crawlingContext.request) };
|
|
48
|
+
* }
|
|
44
49
|
* },
|
|
45
50
|
* ]
|
|
46
51
|
* ```
|
|
47
52
|
*/
|
|
48
|
-
postNavigationHooks?: ((crawlingContext:
|
|
53
|
+
postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
|
|
49
54
|
/**
|
|
50
55
|
* An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
|
|
51
56
|
* you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
@@ -85,10 +90,10 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
85
90
|
/**
|
|
86
91
|
* @internal
|
|
87
92
|
*/
|
|
88
|
-
export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void
|
|
93
|
+
export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
|
|
89
94
|
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
90
95
|
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
|
|
91
|
-
interface
|
|
96
|
+
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
92
97
|
/**
|
|
93
98
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
94
99
|
*/
|
|
@@ -102,7 +107,7 @@ interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends
|
|
|
102
107
|
* @internal
|
|
103
108
|
*/
|
|
104
109
|
export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
105
|
-
JSONData extends JsonValue = any> extends
|
|
110
|
+
JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
|
|
106
111
|
/**
|
|
107
112
|
* The request body of the web page.
|
|
108
113
|
* The type depends on the `Content-Type` header of the web page:
|
|
@@ -163,13 +168,15 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
163
168
|
*
|
|
164
169
|
* This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
|
|
165
170
|
*
|
|
166
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
167
|
-
* {@link
|
|
168
|
-
*
|
|
171
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
172
|
+
* {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
|
|
173
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
174
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
175
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
176
|
+
* result as `requestManager`.
|
|
169
177
|
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
178
|
+
* > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
|
|
179
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
173
180
|
*
|
|
174
181
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
175
182
|
*
|
|
@@ -221,7 +228,7 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
221
228
|
*/
|
|
222
229
|
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
|
|
223
230
|
protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
|
|
224
|
-
protected postNavigationHooks: ((crawlingContext:
|
|
231
|
+
protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
|
|
225
232
|
protected saveResponseCookies: boolean;
|
|
226
233
|
protected navigationTimeoutMillis: number;
|
|
227
234
|
protected ignoreSslErrors: boolean;
|
|
@@ -321,6 +328,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
321
328
|
*/
|
|
322
329
|
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
|
323
330
|
protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
|
|
331
|
+
private prepareHttpRequest;
|
|
324
332
|
private makeHttpRequest;
|
|
325
333
|
private processHttpResponse;
|
|
326
334
|
private handleBlockedRequestByContent;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,
|
|
1
|
+
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EAEnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EACZ,eAAe,EAKlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,aAAa,EAA0B,MAAM,eAAe,CAAC;AAE3E,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAsB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,CAAC,CACnB,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAEzG,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,2BAA2B,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,CAAC;IAC3C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EACjC,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE5D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC9D,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAC5B,eAAe,EAAE,2BAA2B,KAC3C,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAYpC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa;cAkD7D,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YA4B1F,kBAAkB;YA2BlB,eAAe;YAqBf,mBAAmB;YAyFnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAkBvG;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAmB3G;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IAwC1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;cAW/D,MAAM,GAAG,SAAS;;IAqB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAwCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,EAAE,QAAQ;IAKjD,OAAO,CAAC,oBAAoB;IAe5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CA2BvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,QAAQ,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -36,13 +36,15 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
36
36
|
*
|
|
37
37
|
* This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
|
|
38
38
|
*
|
|
39
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
40
|
-
* {@link
|
|
41
|
-
*
|
|
39
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
40
|
+
* {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
|
|
41
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
42
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
43
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
44
|
+
* result as `requestManager`.
|
|
42
45
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
46
|
+
* > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
|
|
47
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
46
48
|
*
|
|
47
49
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
48
50
|
*
|
|
@@ -144,15 +146,27 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
144
146
|
this.saveResponseCookies = saveResponseCookies;
|
|
145
147
|
}
|
|
146
148
|
buildContextPipeline() {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
// When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
|
|
150
|
+
// the response-derived members, so the guarded action is bypassed and the context left untouched.
|
|
151
|
+
const skipGuard = (action) => ({
|
|
152
|
+
action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
|
|
153
|
+
});
|
|
154
|
+
let pipeline = ContextPipeline.create().compose({
|
|
155
|
+
action: this.prepareHttpRequest.bind(this),
|
|
156
|
+
});
|
|
157
|
+
for (const hook of this.preNavigationHooks) {
|
|
158
|
+
pipeline = pipeline.compose(skipGuard(hook));
|
|
159
|
+
}
|
|
160
|
+
let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
|
|
161
|
+
for (const hook of this.postNavigationHooks) {
|
|
162
|
+
pipelineWithNavigation = pipelineWithNavigation.compose(skipGuard(hook));
|
|
163
|
+
}
|
|
164
|
+
return pipelineWithNavigation
|
|
151
165
|
.compose({ action: this.processHttpResponse.bind(this) })
|
|
152
166
|
.compose({ action: this.handleBlockedRequestByContent.bind(this) });
|
|
153
167
|
}
|
|
154
|
-
async
|
|
155
|
-
const { request
|
|
168
|
+
async prepareHttpRequest(crawlingContext) {
|
|
169
|
+
const { request } = crawlingContext;
|
|
156
170
|
if (request.skipNavigation) {
|
|
157
171
|
return {
|
|
158
172
|
request: new Proxy(request, {
|
|
@@ -169,8 +183,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
169
183
|
};
|
|
170
184
|
}
|
|
171
185
|
request.state = RequestState.BEFORE_NAV;
|
|
172
|
-
|
|
186
|
+
return {};
|
|
187
|
+
}
|
|
188
|
+
async makeHttpRequest(crawlingContext) {
|
|
173
189
|
tryCancel();
|
|
190
|
+
const { request, session } = crawlingContext;
|
|
174
191
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
175
192
|
const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
|
|
176
193
|
tryCancel();
|
|
@@ -198,7 +215,6 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
198
215
|
},
|
|
199
216
|
};
|
|
200
217
|
}
|
|
201
|
-
await this._executeHooks(this.postNavigationHooks, crawlingContext);
|
|
202
218
|
tryCancel();
|
|
203
219
|
const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
|
|
204
220
|
tryCancel();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAa7B,OAAO,EACH,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAsB,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAErF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAsLF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IACpD,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,mBAAmB,CAAU;IAC7B,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IACtB,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAExC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS;QAE5E,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,GAAG,IAAI,EAC1B,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAEkB,oBAAoB;QACnC,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,EAAE,CAAC;QAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EACjE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,sBAAsB,CAC5B,yEAAyE,CAC5E,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,sBAAsB,CAC5B,2EAA2E,CAC9E,CAAC;gBACN,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,sBAAsB,CAC5B,4EAA4E,CAC/E,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,IAAI,CAAC,MAAM;wBAAE,SAAS;oBACtB,IAAI,CAAC;wBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;oBAClG,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;oBACpE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAU,EAAE,qCAAqC,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAA0B;QACnF,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1D,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnE,cAAc,CAAC,OAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;YAC3E,OAAO,cAAc,CAAC,OAAQ,CAAC,MAAM,CAAC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC;YACrC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK;iBACrB,YAAY,CAAC,QAAQ,CAAC;iBACtB,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,qFAAqF;QACrF,qFAAqF;QACrF,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAEjG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,SAAS;YACT,aAAa,EAAE,IAAI,CAAC,OAAO;SAC9B,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AASN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAsB,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAErF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA6LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IACpD,kBAAkB,CAAsC;IACxD,mBAAmB,CAEkC;IACrD,mBAAmB,CAAU;IAC7B,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IACtB,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAExC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS;QAE5E,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,mBAAmB,GAAG,IAAI,EAC1B,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAEkB,oBAAoB;QACnC,8FAA8F;QAC9F,kGAAkG;QAClG,MAAM,SAAS,GAAG,CACd,MAA2C,EAChB,EAAE,CAAC,CAAC;YAC/B,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAQ;SAChG,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAG,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YAC7D,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC7C,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC1C,sBAAsB,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,sBAAsB;aACxB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,eAAgC;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACoC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,SAAS,EAAE,CAAC;QAEZ,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EACjE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA4C;QAI5C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,sBAAsB,CAC5B,yEAAyE,CAC5E,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,sBAAsB,CAC5B,kEAAkE,CACrE,CAAC;gBACN,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,sBAAsB,CAC5B,2EAA2E,CAC9E,CAAC;gBACN,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,sBAAsB,CAC5B,4EAA4E,CAC/E,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,IAAI,CAAC,MAAM;wBAAE,SAAS;oBACtB,IAAI,CAAC;wBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;oBAClG,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;oBACpE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAU,EAAE,qCAAqC,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAA0B;QACnF,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1D,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,uBAAuB;YACrC,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnE,cAAc,CAAC,OAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;YAC3E,OAAO,cAAc,CAAC,OAAQ,CAAC,MAAM,CAAC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC;YACrC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK;iBACrB,YAAY,CAAC,QAAQ,CAAC;iBACtB,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAClE,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,EAAE;QAC9E,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,qFAAqF;QACrF,qFAAqF;QACrF,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAEjG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC9C,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,SAAS;YACT,aAAa,EAAE,IAAI,CAAC,OAAO;SAC9B,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AASN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.65",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,11 +49,11 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
55
|
-
"@crawlee/types": "4.0.0-beta.
|
|
56
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.65",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.65",
|
|
54
|
+
"@crawlee/http-client": "4.0.0-beta.65",
|
|
55
|
+
"@crawlee/types": "4.0.0-beta.65",
|
|
56
|
+
"@crawlee/utils": "4.0.0-beta.65",
|
|
57
57
|
"@types/content-type": "^1.1.8",
|
|
58
58
|
"cheerio": "^1.0.0",
|
|
59
59
|
"content-type": "^1.0.5",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "fcee63d935043be16f7be59ab9784161cf132b11"
|
|
74
74
|
}
|