@crawlee/http 4.0.0-beta.18 → 4.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -55,11 +55,11 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
|
|
|
55
55
|
*
|
|
56
56
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
57
57
|
*
|
|
58
|
-
* We can use the `preNavigationHooks` to adjust
|
|
58
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
59
59
|
*
|
|
60
60
|
* ```
|
|
61
61
|
* preNavigationHooks: [
|
|
62
|
-
* (crawlingContext
|
|
62
|
+
* (crawlingContext) => {
|
|
63
63
|
* // ...
|
|
64
64
|
* },
|
|
65
65
|
* ]
|
|
@@ -76,11 +76,11 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
76
76
|
*
|
|
77
77
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
78
78
|
*
|
|
79
|
-
* We can use the `preNavigationHooks` to adjust
|
|
79
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
80
80
|
*
|
|
81
81
|
* ```
|
|
82
82
|
* preNavigationHooks: [
|
|
83
|
-
* (crawlingContext
|
|
83
|
+
* (crawlingContext) => {
|
|
84
84
|
* // ...
|
|
85
85
|
* },
|
|
86
86
|
* ]
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
import { Readable } from 'node:stream';
|
|
2
1
|
import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
|
|
3
2
|
import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
|
|
4
3
|
import type { LoadedRequest } from '@crawlee/core';
|
|
5
4
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
6
5
|
import { type CheerioRoot } from '@crawlee/utils';
|
|
7
6
|
import type { RequestLike, ResponseLike } from 'content-type';
|
|
8
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
9
|
-
import type { Method, OptionsInit } from 'got-scraping';
|
|
10
7
|
import type { JsonValue } from 'type-fest';
|
|
11
8
|
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
12
9
|
JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
|
|
@@ -21,12 +18,12 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
21
18
|
ignoreSslErrors?: boolean;
|
|
22
19
|
/**
|
|
23
20
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
24
|
-
* or browser properties before navigation. The function accepts
|
|
25
|
-
* which
|
|
21
|
+
* or browser properties before navigation. The function accepts one parameter `crawlingContext`,
|
|
22
|
+
* which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
|
|
26
23
|
* Example:
|
|
27
24
|
* ```
|
|
28
25
|
* preNavigationHooks: [
|
|
29
|
-
* async (crawlingContext
|
|
26
|
+
* async (crawlingContext) => {
|
|
30
27
|
* // ...
|
|
31
28
|
* },
|
|
32
29
|
* ]
|
|
@@ -97,7 +94,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
97
94
|
/**
|
|
98
95
|
* @internal
|
|
99
96
|
*/
|
|
100
|
-
export type InternalHttpHook<Context> = (crawlingContext: Context
|
|
97
|
+
export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
|
|
101
98
|
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
102
99
|
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
|
|
103
100
|
interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
@@ -185,11 +182,11 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
185
182
|
*
|
|
186
183
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
187
184
|
*
|
|
188
|
-
* We can use the `preNavigationHooks` to adjust
|
|
185
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
189
186
|
*
|
|
190
187
|
* ```javascript
|
|
191
188
|
* preNavigationHooks: [
|
|
192
|
-
* (crawlingContext
|
|
189
|
+
* (crawlingContext) => {
|
|
193
190
|
* // ...
|
|
194
191
|
* },
|
|
195
192
|
* ]
|
|
@@ -335,15 +332,16 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
335
332
|
private handleBlockedRequestByContent;
|
|
336
333
|
protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
|
|
337
334
|
/**
|
|
338
|
-
*
|
|
335
|
+
* Returns the `Cookie` header value based on the current context and
|
|
336
|
+
* any changes that occurred in the navigation hooks.
|
|
339
337
|
*/
|
|
340
|
-
protected _applyCookies({ session, request }: CrawlingContext,
|
|
338
|
+
protected _applyCookies({ session, request }: CrawlingContext, preHookCookies: string, postHookCookies: string): string;
|
|
341
339
|
/**
|
|
342
340
|
* Function to make the HTTP request. It performs optimizations
|
|
343
341
|
* on the request such as only downloading the request body if the
|
|
344
342
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
345
343
|
*/
|
|
346
|
-
protected _requestFunction({ request, session, proxyUrl,
|
|
344
|
+
protected _requestFunction({ request, session, proxyUrl, cookieString, }: RequestFunctionOptions): Promise<Response>;
|
|
347
345
|
/**
|
|
348
346
|
* Encodes and parses response according to the provided content type
|
|
349
347
|
*/
|
|
@@ -365,78 +363,22 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
365
363
|
/**
|
|
366
364
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
367
365
|
*/
|
|
368
|
-
protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string
|
|
369
|
-
url
|
|
370
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
371
|
-
headers?: import("got-scraping").Headers | undefined;
|
|
372
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
373
|
-
request?: import("got-scraping").RequestFunction | undefined;
|
|
374
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
375
|
-
agent?: import("got-scraping").Agents | undefined;
|
|
376
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
377
|
-
h2session?: import("http2").ClientHttp2Session | undefined;
|
|
378
|
-
decompress?: boolean | undefined;
|
|
379
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
380
|
-
timeout?: import("got-scraping").Delays | undefined;
|
|
381
|
-
prefixUrl?: string | URL | undefined;
|
|
382
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
383
|
-
body?: string | Buffer | Readable | Generator | AsyncGenerator | import("form-data-encoder").FormDataLike | undefined;
|
|
384
|
-
form?: Record<string, any> | undefined;
|
|
385
|
-
json?: unknown;
|
|
386
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
387
|
-
cookieJar?: import("got-scraping").PromiseCookieJar | import("got-scraping").ToughCookieJar | undefined;
|
|
388
|
-
signal?: AbortSignal | undefined;
|
|
389
|
-
ignoreInvalidCookies?: boolean | undefined;
|
|
390
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
391
|
-
searchParams?: string | import("got-scraping").SearchParameters | URLSearchParams | undefined;
|
|
392
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
393
|
-
dnsLookup?: import("cacheable-lookup").default["lookup"] | undefined;
|
|
394
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
395
|
-
dnsCache?: import("cacheable-lookup").default | boolean | undefined;
|
|
396
|
-
context?: Record<string, unknown> | undefined;
|
|
397
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
398
|
-
followRedirect?: boolean | ((response: import("got-scraping").PlainResponse) => boolean) | undefined;
|
|
399
|
-
maxRedirects?: number | undefined;
|
|
366
|
+
protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string): {
|
|
367
|
+
url: string;
|
|
400
368
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
allowGetBody?: boolean | undefined;
|
|
407
|
-
methodRewriting?: boolean | undefined;
|
|
408
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
409
|
-
dnsLookupIpVersion?: import("got-scraping").DnsLookupIpVersion;
|
|
410
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
411
|
-
parseJson?: import("got-scraping").ParseJsonFunction | undefined;
|
|
412
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
413
|
-
stringifyJson?: import("got-scraping").StringifyJsonFunction | undefined;
|
|
414
|
-
localAddress?: string | undefined;
|
|
415
|
-
method?: Method | undefined;
|
|
416
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
417
|
-
createConnection?: import("got-scraping").CreateConnectionFunction | undefined;
|
|
418
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
419
|
-
cacheOptions?: import("got-scraping").CacheOptions | undefined;
|
|
420
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
421
|
-
https?: import("got-scraping").HttpsOptions | undefined;
|
|
422
|
-
encoding?: BufferEncoding | undefined;
|
|
423
|
-
resolveBodyOnly?: boolean | undefined;
|
|
424
|
-
isStream?: boolean | undefined;
|
|
425
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
426
|
-
responseType?: import("got-scraping").ResponseType | undefined;
|
|
427
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
428
|
-
pagination?: import("got-scraping").PaginationOptions<unknown, unknown> | undefined;
|
|
429
|
-
setHost?: boolean | undefined;
|
|
430
|
-
maxHeaderSize?: number | undefined;
|
|
431
|
-
enableUnixSockets?: boolean | undefined;
|
|
432
|
-
} & {
|
|
433
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
434
|
-
hooks?: Partial<import("got-scraping").Hooks>;
|
|
435
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
436
|
-
retry?: Partial<import("got-scraping").RetryOptions>;
|
|
369
|
+
method: import("@crawlee/types").AllowedHttpMethods;
|
|
370
|
+
proxyUrl: string | undefined;
|
|
371
|
+
timeout: {
|
|
372
|
+
request: number;
|
|
373
|
+
};
|
|
437
374
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
438
|
-
|
|
439
|
-
|
|
375
|
+
cookieJar: import("tough-cookie").CookieJar | undefined;
|
|
376
|
+
sessionToken: Session | undefined;
|
|
377
|
+
headers: Record<string, string> | undefined;
|
|
378
|
+
https: {
|
|
379
|
+
rejectUnauthorized: boolean;
|
|
380
|
+
};
|
|
381
|
+
body: string | undefined;
|
|
440
382
|
};
|
|
441
383
|
protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
|
|
442
384
|
encoding: BufferEncoding;
|
|
@@ -460,7 +402,7 @@ interface RequestFunctionOptions {
|
|
|
460
402
|
request: CrawleeRequest;
|
|
461
403
|
session?: Session;
|
|
462
404
|
proxyUrl?: string;
|
|
463
|
-
|
|
405
|
+
cookieString?: string;
|
|
464
406
|
}
|
|
465
407
|
/**
|
|
466
408
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAOlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAwB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtF,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAmCxC,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAkDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;;OAGG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACxB,MAAM;IAQT;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACf,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2C1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;;;;cAY/D,MAAM,GAAG,SAAS;;IAmB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAsCvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -45,11 +45,11 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
45
45
|
*
|
|
46
46
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
47
47
|
*
|
|
48
|
-
* We can use the `preNavigationHooks` to adjust
|
|
48
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
49
49
|
*
|
|
50
50
|
* ```javascript
|
|
51
51
|
* preNavigationHooks: [
|
|
52
|
-
* (crawlingContext
|
|
52
|
+
* (crawlingContext) => {
|
|
53
53
|
* // ...
|
|
54
54
|
* },
|
|
55
55
|
* ]
|
|
@@ -184,17 +184,16 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
184
184
|
},
|
|
185
185
|
};
|
|
186
186
|
}
|
|
187
|
-
const gotOptions = {};
|
|
188
187
|
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
|
|
189
188
|
request.state = RequestState.BEFORE_NAV;
|
|
190
189
|
// Execute pre navigation hooks before applying session pool cookies,
|
|
191
190
|
// as they may also set cookies in the session
|
|
192
|
-
await this._executeHooks(this.preNavigationHooks, crawlingContext
|
|
191
|
+
await this._executeHooks(this.preNavigationHooks, crawlingContext);
|
|
193
192
|
tryCancel();
|
|
194
193
|
const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
|
|
195
|
-
this._applyCookies(crawlingContext,
|
|
194
|
+
const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
|
|
196
195
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
197
|
-
const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl,
|
|
196
|
+
const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, cookieString }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
|
|
198
197
|
tryCancel();
|
|
199
198
|
request.loadedUrl = httpResponse?.url;
|
|
200
199
|
request.state = RequestState.AFTER_NAV;
|
|
@@ -286,58 +285,27 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
286
285
|
return false;
|
|
287
286
|
}
|
|
288
287
|
/**
|
|
289
|
-
*
|
|
288
|
+
* Returns the `Cookie` header value based on the current context and
|
|
289
|
+
* any changes that occurred in the navigation hooks.
|
|
290
290
|
*/
|
|
291
|
-
_applyCookies({ session, request },
|
|
291
|
+
_applyCookies({ session, request }, preHookCookies, postHookCookies) {
|
|
292
292
|
const sessionCookie = session?.getCookieString(request.url) ?? '';
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
|
|
296
|
-
this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
|
|
297
|
-
const sourceCookies = [];
|
|
298
|
-
if (Array.isArray(lowerCaseHeader)) {
|
|
299
|
-
sourceCookies.push(...lowerCaseHeader);
|
|
300
|
-
}
|
|
301
|
-
else {
|
|
302
|
-
sourceCookies.push(lowerCaseHeader);
|
|
303
|
-
}
|
|
304
|
-
if (Array.isArray(upperCaseHeader)) {
|
|
305
|
-
sourceCookies.push(...upperCaseHeader);
|
|
306
|
-
}
|
|
307
|
-
else {
|
|
308
|
-
sourceCookies.push(upperCaseHeader);
|
|
309
|
-
}
|
|
310
|
-
alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
|
|
311
|
-
}
|
|
312
|
-
const sourceCookies = [sessionCookie, preHookCookies];
|
|
313
|
-
if (Array.isArray(alteredGotOptionsCookies)) {
|
|
314
|
-
sourceCookies.push(...alteredGotOptionsCookies);
|
|
315
|
-
}
|
|
316
|
-
else {
|
|
317
|
-
sourceCookies.push(alteredGotOptionsCookies);
|
|
318
|
-
}
|
|
319
|
-
sourceCookies.push(postHookCookies);
|
|
320
|
-
const mergedCookie = mergeCookies(request.url, sourceCookies);
|
|
321
|
-
gotOptions.headers ??= {};
|
|
322
|
-
Reflect.deleteProperty(gotOptions.headers, 'Cookie');
|
|
323
|
-
Reflect.deleteProperty(gotOptions.headers, 'cookie');
|
|
324
|
-
if (mergedCookie !== '') {
|
|
325
|
-
gotOptions.headers.Cookie = mergedCookie;
|
|
326
|
-
}
|
|
293
|
+
const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
|
|
294
|
+
return mergeCookies(request.url, sourceCookies);
|
|
327
295
|
}
|
|
328
296
|
/**
|
|
329
297
|
* Function to make the HTTP request. It performs optimizations
|
|
330
298
|
* on the request such as only downloading the request body if the
|
|
331
299
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
332
300
|
*/
|
|
333
|
-
async _requestFunction({ request, session, proxyUrl,
|
|
301
|
+
async _requestFunction({ request, session, proxyUrl, cookieString, }) {
|
|
334
302
|
if (!TimeoutError) {
|
|
335
303
|
// @ts-ignore
|
|
336
304
|
({ TimeoutError } = await import('got-scraping'));
|
|
337
305
|
}
|
|
338
|
-
const opts = this._getRequestOptions(request, session, proxyUrl
|
|
306
|
+
const opts = this._getRequestOptions(request, session, proxyUrl);
|
|
339
307
|
try {
|
|
340
|
-
return await this._requestAsBrowser(opts, session);
|
|
308
|
+
return await this._requestAsBrowser(opts, session, cookieString);
|
|
341
309
|
}
|
|
342
310
|
catch (e) {
|
|
343
311
|
if (e instanceof TimeoutError) {
|
|
@@ -397,7 +365,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
397
365
|
/**
|
|
398
366
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
399
367
|
*/
|
|
400
|
-
_getRequestOptions(request, session, proxyUrl
|
|
368
|
+
_getRequestOptions(request, session, proxyUrl) {
|
|
401
369
|
const requestOptions = {
|
|
402
370
|
url: request.url,
|
|
403
371
|
method: request.method,
|
|
@@ -405,13 +373,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
405
373
|
timeout: { request: this.navigationTimeoutMillis },
|
|
406
374
|
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
|
|
407
375
|
sessionToken: session,
|
|
408
|
-
|
|
409
|
-
headers: { ...request.headers, ...gotOptions?.headers },
|
|
376
|
+
headers: request.headers,
|
|
410
377
|
https: {
|
|
411
|
-
...gotOptions?.https,
|
|
412
378
|
rejectUnauthorized: !this.ignoreSslErrors,
|
|
413
379
|
},
|
|
414
|
-
|
|
380
|
+
body: undefined,
|
|
415
381
|
};
|
|
416
382
|
// Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
|
|
417
383
|
Reflect.deleteProperty(requestOptions.headers, 'cookie');
|
|
@@ -498,12 +464,17 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
498
464
|
/**
|
|
499
465
|
* @internal wraps public utility for mocking purposes
|
|
500
466
|
*/
|
|
501
|
-
_requestAsBrowser = async (options, session) => {
|
|
467
|
+
_requestAsBrowser = async (options, session, cookieString) => {
|
|
502
468
|
const opts = processHttpRequestOptions({
|
|
503
469
|
...options,
|
|
504
470
|
cookieJar: options.cookieJar,
|
|
505
471
|
responseType: 'text',
|
|
506
472
|
});
|
|
473
|
+
if (cookieString) {
|
|
474
|
+
opts.headers?.delete('Cookie');
|
|
475
|
+
opts.headers?.delete('cookie');
|
|
476
|
+
opts.headers?.set('Cookie', cookieString);
|
|
477
|
+
}
|
|
507
478
|
const response = await this.httpClient.stream(new Request(opts.url, {
|
|
508
479
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
509
480
|
headers: new Headers(opts.headers),
|
|
@@ -516,9 +487,9 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
516
487
|
onRedirect: (redirectResponse, updatedRequest) => {
|
|
517
488
|
if (this.persistCookiesPerSession) {
|
|
518
489
|
session.setCookiesFromResponse(redirectResponse);
|
|
519
|
-
const
|
|
520
|
-
if (
|
|
521
|
-
updatedRequest.headers.set('Cookie',
|
|
490
|
+
const cookieStringRedirected = session.getCookieString(updatedRequest.url.toString());
|
|
491
|
+
if (cookieStringRedirected !== '') {
|
|
492
|
+
updatedRequest.headers.set('Cookie', cookieStringRedirected);
|
|
522
493
|
}
|
|
523
494
|
}
|
|
524
495
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CACxB,OAAuB,EACvB,OAAiB,EACjB,QAAiB,EACjB,UAAwB;QAExB,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;QACA,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;gBAC7C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;oBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;wBACtB,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;oBACvD,CAAC;gBACL,CAAC;YACL,CAAC;SACJ,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA+LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAwD;IAmCxC;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAAkC;gBACnC,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEhH,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAC/E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAwC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAElE,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAEvE,OAAO,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACS;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAuB,EAAE,QAAkB;QACtE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAuB,EAAE,OAAiB,EAAE,QAAiB;QACtF,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE;gBACH,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,IAAI,EAAE,SAA+B;SACxC,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAuB,EACvB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAuB,EAAE,QAAkB;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAAE,OAAwB,EAAE,OAAiB,EAAE,YAAqB,EAAE,EAAE;QACrG,MAAM,IAAI,GAAG,yBAAyB,CAAC;YACnC,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,YAAY,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YAClB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAS,CAAC,CAAC,CAAC,SAAS;YAChE,OAAO,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,kEAAkE;YAClE,MAAM,EAAE,MAAM;SACF,CAAC,EACjB;YACI,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;gBAC7C,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;oBAElD,MAAM,sBAAsB,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACxF,IAAI,sBAAsB,KAAK,EAAE,EAAE,CAAC;wBAChC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;oBACjE,CAAC;gBACL,CAAC;YACL,CAAC;SACJ,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.19",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,9 +49,9 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.19",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.19",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.19",
|
|
55
55
|
"@types/content-type": "^1.1.8",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"content-type": "^1.0.5",
|
|
@@ -69,5 +69,5 @@
|
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
71
|
},
|
|
72
|
-
"gitHead": "
|
|
72
|
+
"gitHead": "690af03140bea34c9a8871d370607cd52339caa0"
|
|
73
73
|
}
|