@crawlee/http 3.6.3-beta.9 → 3.7.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.mjs
CHANGED
|
@@ -4,6 +4,7 @@ export default mod;
|
|
|
4
4
|
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
|
|
5
5
|
export const AutoscaledPool = mod.AutoscaledPool;
|
|
6
6
|
export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
|
|
7
|
+
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
7
8
|
export const BasicCrawler = mod.BasicCrawler;
|
|
8
9
|
export const Configuration = mod.Configuration;
|
|
9
10
|
export const CookieParseError = mod.CookieParseError;
|
|
@@ -23,9 +24,11 @@ export const LogLevel = mod.LogLevel;
|
|
|
23
24
|
export const Logger = mod.Logger;
|
|
24
25
|
export const LoggerJson = mod.LoggerJson;
|
|
25
26
|
export const LoggerText = mod.LoggerText;
|
|
27
|
+
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
26
28
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
27
29
|
export const MissingRouteError = mod.MissingRouteError;
|
|
28
30
|
export const NonRetryableError = mod.NonRetryableError;
|
|
31
|
+
export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
|
|
29
32
|
export const ProxyConfiguration = mod.ProxyConfiguration;
|
|
30
33
|
export const PseudoUrl = mod.PseudoUrl;
|
|
31
34
|
export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
+
/// <reference types="node" />
|
|
4
|
+
/// <reference types="node" />
|
|
5
|
+
/// <reference types="node" />
|
|
3
6
|
import type { IncomingMessage } from 'node:http';
|
|
4
7
|
import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterRoutes, Session } from '@crawlee/basic';
|
|
5
8
|
import { BasicCrawler, CrawlerExtension, Configuration } from '@crawlee/basic';
|
|
@@ -7,7 +10,7 @@ import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
|
7
10
|
import * as cheerio from 'cheerio';
|
|
8
11
|
import type { RequestLike, ResponseLike } from 'content-type';
|
|
9
12
|
// @ts-ignore optional peer dependency
|
|
10
|
-
import type { OptionsInit, PlainResponse } from 'got-scraping';
|
|
13
|
+
import type { OptionsInit, Method, PlainResponse } from 'got-scraping';
|
|
11
14
|
import type { JsonValue } from 'type-fest';
|
|
12
15
|
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
13
16
|
JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
|
|
@@ -267,6 +270,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
267
270
|
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
268
271
|
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
269
272
|
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
273
|
+
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
270
274
|
};
|
|
271
275
|
/**
|
|
272
276
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
@@ -282,7 +286,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
282
286
|
* Wrapper around requestHandler that opens and closes pages etc.
|
|
283
287
|
*/
|
|
284
288
|
protected _runRequestHandler(crawlingContext: Context): Promise<void>;
|
|
285
|
-
protected isRequestBlocked(crawlingContext: Context): Promise<
|
|
289
|
+
protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
|
|
286
290
|
protected _handleNavigation(crawlingContext: Context): Promise<void>;
|
|
287
291
|
/**
|
|
288
292
|
* Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
|
|
@@ -320,7 +324,61 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
320
324
|
/**
|
|
321
325
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
322
326
|
*/
|
|
323
|
-
protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit):
|
|
327
|
+
protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): {
|
|
328
|
+
request?: import("got-scraping", { with: { "resolution-mode": "import" } }).RequestFunction | undefined;
|
|
329
|
+
agent?: import("got-scraping", { with: { "resolution-mode": "import" } }).Agents | undefined;
|
|
330
|
+
h2session?: import("http2").ClientHttp2Session | undefined;
|
|
331
|
+
decompress?: boolean | undefined;
|
|
332
|
+
timeout?: import("got-scraping", { with: { "resolution-mode": "import" } }).Delays | undefined;
|
|
333
|
+
prefixUrl?: string | URL | undefined;
|
|
334
|
+
body?: string | Buffer | import("stream").Readable | Generator<unknown, any, unknown> | AsyncGenerator<unknown, any, unknown> | import("form-data-encoder", { with: { "resolution-mode": "import" } }).FormDataLike | undefined;
|
|
335
|
+
form?: Record<string, any> | undefined;
|
|
336
|
+
json?: unknown;
|
|
337
|
+
url?: string | URL | undefined;
|
|
338
|
+
cookieJar?: import("got-scraping", { with: { "resolution-mode": "import" } }).PromiseCookieJar | import("got-scraping", { with: { "resolution-mode": "import" } }).ToughCookieJar | undefined;
|
|
339
|
+
signal?: AbortSignal | undefined;
|
|
340
|
+
ignoreInvalidCookies?: boolean | undefined;
|
|
341
|
+
searchParams?: string | import("got-scraping", { with: { "resolution-mode": "import" } }).SearchParameters | URLSearchParams | undefined;
|
|
342
|
+
dnsLookup?: {
|
|
343
|
+
(hostname: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
|
|
344
|
+
(hostname: string, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
|
|
345
|
+
(hostname: string, options: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).LookupOptions & {
|
|
346
|
+
all: true;
|
|
347
|
+
}, callback: (error: NodeJS.ErrnoException | null, result: readonly import("cacheable-lookup", { with: { "resolution-mode": "import" } }).EntryObject[]) => void): void;
|
|
348
|
+
(hostname: string, options: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).LookupOptions, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
|
|
349
|
+
} | undefined;
|
|
350
|
+
dnsCache?: boolean | import("cacheable-lookup", { with: { "resolution-mode": "import" } }).default | undefined;
|
|
351
|
+
context?: Record<string, unknown> | undefined;
|
|
352
|
+
followRedirect?: boolean | undefined;
|
|
353
|
+
maxRedirects?: number | undefined;
|
|
354
|
+
cache?: string | boolean | import("cacheable-request", { with: { "resolution-mode": "import" } }).StorageAdapter | undefined;
|
|
355
|
+
throwHttpErrors?: boolean | undefined;
|
|
356
|
+
username?: string | undefined;
|
|
357
|
+
password?: string | undefined;
|
|
358
|
+
http2?: boolean | undefined;
|
|
359
|
+
allowGetBody?: boolean | undefined;
|
|
360
|
+
headers?: import("got-scraping", { with: { "resolution-mode": "import" } }).Headers | undefined;
|
|
361
|
+
methodRewriting?: boolean | undefined;
|
|
362
|
+
dnsLookupIpVersion?: import("got-scraping", { with: { "resolution-mode": "import" } }).DnsLookupIpVersion;
|
|
363
|
+
parseJson?: import("got-scraping", { with: { "resolution-mode": "import" } }).ParseJsonFunction | undefined;
|
|
364
|
+
stringifyJson?: import("got-scraping", { with: { "resolution-mode": "import" } }).StringifyJsonFunction | undefined;
|
|
365
|
+
localAddress?: string | undefined;
|
|
366
|
+
method?: Method | undefined;
|
|
367
|
+
createConnection?: import("got-scraping", { with: { "resolution-mode": "import" } }).CreateConnectionFunction | undefined;
|
|
368
|
+
cacheOptions?: import("got-scraping", { with: { "resolution-mode": "import" } }).CacheOptions | undefined;
|
|
369
|
+
https?: import("got-scraping", { with: { "resolution-mode": "import" } }).HttpsOptions | undefined;
|
|
370
|
+
encoding?: BufferEncoding | undefined;
|
|
371
|
+
resolveBodyOnly?: boolean | undefined;
|
|
372
|
+
isStream?: boolean | undefined;
|
|
373
|
+
responseType?: import("got-scraping", { with: { "resolution-mode": "import" } }).ResponseType | undefined;
|
|
374
|
+
pagination?: import("got-scraping", { with: { "resolution-mode": "import" } }).PaginationOptions<unknown, unknown> | undefined;
|
|
375
|
+
setHost?: boolean | undefined;
|
|
376
|
+
maxHeaderSize?: number | undefined;
|
|
377
|
+
enableUnixSockets?: boolean | undefined;
|
|
378
|
+
} & {
|
|
379
|
+
hooks?: Partial<import("got-scraping", { with: { "resolution-mode": "import" } }).Hooks> | undefined;
|
|
380
|
+
retry?: Partial<import("got-scraping", { with: { "resolution-mode": "import" } }).RetryOptions> | undefined;
|
|
381
|
+
} & import("got-scraping", { with: { "resolution-mode": "import" } }).Context & {
|
|
324
382
|
isStream: true;
|
|
325
383
|
};
|
|
326
384
|
protected _encodeResponse(request: Request, response: IncomingMessage, encoding: BufferEncoding): {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":";;;;;AAAA,OAAO,KAAK,EAAuB,eAAe,EAAE,MAAM,WAAW,CAAC;AAMtE,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,OAAO,EACP,cAAc,EACd,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,gBAAgB,EAIhB,aAAa,EAGhB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5D,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAkC,aAAa,EAAqC,MAAM,cAAc,CAAC;AAI1I,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAoB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAC5B,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,MAAM,WAAW,kBAAkB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,mBAAmB,CAAC,OAAO,CAAC;IAC/I;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,gBAAgB,CAAC,CAAC;IAEnE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAEjD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAElD;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CACpC,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,WAAW,KACtB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,EAAE,2EAA2E;AAC7G,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CACxB,SAAQ,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC5C;;;;;OAKG;IACH,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IACxD,QAAQ,EAAE,aAAa,CAAC;IAExB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;CAAG;AAEpH,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAC5B,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CAAC,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAE,SAAQ,YAAY,CAAC,OAAO,CAAC;aAyCtD,MAAM;IAxC/E;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,SAAS,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3D,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAiBpC;IAEF;;OAEG;gBACS,OAAO,GAAE,kBAAkB,CAAC,OAAO,CAAM,EAAoB,MAAM,gBAAkC;IAmFjH;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IA8B/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;cAqE3C,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;cAa5E,iBAAiB,CAAC,eAAe,EAAE,OAAO;IA6B1D;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM;IAsDvI;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;IAuB5H;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAE,OAAO;;;;;;;;;;;;;;;;;;;cAyC1F,UAAU,CAAC,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAM5H;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBACzD,IAAI;;IAkCxD,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,GAAG;QAC9F,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;IAqCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAqB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CA4BvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AA2ED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -321,6 +321,12 @@ class HttpCrawler extends basic_1.BasicCrawler {
|
|
|
321
321
|
crawlingContext.session.setCookiesFromResponse(response);
|
|
322
322
|
}
|
|
323
323
|
request.loadedUrl = response.url;
|
|
324
|
+
if (!this.requestMatchesEnqueueStrategy(request)) {
|
|
325
|
+
this.log.debug(`Skipping request ${request.id} (${request.url}) because it does not match the enqueue strategy.`);
|
|
326
|
+
request.noRetry = true;
|
|
327
|
+
request.state = basic_1.RequestState.SKIPPED;
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
324
330
|
Object.assign(crawlingContext, parsed);
|
|
325
331
|
Object.defineProperty(crawlingContext, 'json', {
|
|
326
332
|
get() {
|
|
@@ -331,8 +337,10 @@ class HttpCrawler extends basic_1.BasicCrawler {
|
|
|
331
337
|
},
|
|
332
338
|
});
|
|
333
339
|
}
|
|
334
|
-
if (this.retryOnBlocked
|
|
335
|
-
|
|
340
|
+
if (this.retryOnBlocked) {
|
|
341
|
+
const error = await this.isRequestBlocked(crawlingContext);
|
|
342
|
+
if (error)
|
|
343
|
+
throw new basic_1.SessionError(error);
|
|
336
344
|
}
|
|
337
345
|
request.state = basic_1.RequestState.REQUEST_HANDLER;
|
|
338
346
|
try {
|
|
@@ -347,7 +355,10 @@ class HttpCrawler extends basic_1.BasicCrawler {
|
|
|
347
355
|
async isRequestBlocked(crawlingContext) {
|
|
348
356
|
if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
|
|
349
357
|
const $ = await crawlingContext.parseWithCheerio();
|
|
350
|
-
|
|
358
|
+
const foundSelectors = utils_1.RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
|
|
359
|
+
if (foundSelectors.length > 0) {
|
|
360
|
+
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
361
|
+
}
|
|
351
362
|
}
|
|
352
363
|
return false;
|
|
353
364
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":";;;;AACA,yCAAoC;AACpC,kEAA6B;AAE7B,4CAAgE;AAChE,gDAA4E;AAa5E,0CAUwB;AAExB,0CAAkE;AAClE,yDAAmC;AAEnC,wEAA6C;AAG7C,oEAA+B;AAC/B,oEAA8B;AAC9B,oDAAoB;AAGpB,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAsKF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAa,WAAyF,SAAQ,oBAAqB;IAsC/H;;OAEG;IACH,YAAY,UAAuC,EAAE,EAAoB,SAAS,qBAAa,CAAC,eAAe,EAAE;QAC7G,IAAA,YAAE,EAAC,OAAO,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,cAAc,EACd,kBAAkB,EAElB,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,UAAU;QACV,qBAAqB;QAErB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,EAAE,MAAM,CAAC,CAAC;QAnCwC;;;;mBAAkB,MAAM;WAAkC;QAxCjH;;;WAGG;QACH;;;;;WAAwC;QAE9B;;;;;WAAwC;QACxC;;;;;WAAgD;QAChD;;;;;WAAiD;QACjD;;;;;WAAkC;QAClC;;;;;WAAgC;QAChC;;;;;WAAyB;QACzB;;;;;WAAiC;QACjC;;;;;WAA+B;QAC/B;;;;;WAA4C;QAC5C;;;;;WAAwC;QAC/B;;;;;WAAgC;QAsfnD;;WAEG;QACK;;;;mBAAoB,KAAK,EAAE,OAAyC,EAAE,OAAiB,EAAE,EAAE;gBAC/F,qDAAqD;gBACrD,OAAO,IAAI,OAAO,CAAgB,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;oBACxD,8GAA8G;oBAC9G,6DAA6D;oBAC7D,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC,OAAO,CAAC,CAAC;oBAE1C,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAA+B,EAAE,EAAE;wBAC/E,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;4BAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;4BAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;4BAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;gCACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;4BACjD,CAAC;wBACL,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,yEAAyE;oBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;oBACjB,CAAC;oBAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;wBACvB,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC,CAAC;oBACnD,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;YACP,CAAC;WAAC;QAxdE,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;QAC7H,CAAC;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAExC,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,iCAAiC,SAAS,YAAY,CAAC,CAAC;YAC3H,CAAC;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC,CAAC,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,OAAO,SAAS,IAAI,GAAG,KAAK,YAAY,IAAI,CAC1H,CAAC;YACN,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,mBAAmB,SAAS,IAAI,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE1G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,EAAE,eAAe,CAAC,CAAC;YAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;YACxC,IAAA,mBAAS,GAAE,CAAC;YAEZ,qEAAqE;YACrE,eAAe,CAAC,gBAAgB,KAAhC,eAAe,CAAC,gBAAgB,GAAK,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,EAAC;YAEvF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,oBAAY,EAAE,CAAC;QAC7B,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,IAAA,6BAAmB,EACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,EACjE,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;YACF,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,OAAO,2BAAmB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,SAAS,CAAC;QACvC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAmB,EAAE,UAAuB,EAAE,cAAsB,EAAE,eAAuB;QACnI,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EACF,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,GAC1B,GAAG,UAAU,CAAC,OAAO,CAAC;YAEvB,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAAC,CAAC;YAE/J,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG;YAClB,aAAa;YACb,cAAc;SACjB,CAAC;QAEF,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAlB,UAAU,CAAC,OAAO,GAAK,EAAE,EAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAA0B;QAC/F,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAqC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,oBAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B,EAAE,eAAwB;QACtG,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,IAAI,UAAW,IAAI,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAW,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QAE1E,IAAI,CAAC,UAAW,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,uCAAuC,CAAC,CAAC;YAC1E,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;YACvE,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;QACvD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;QAC1I,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,MAAe,EAAE,gBAAyB;QAC5F,OAAO;YACH,IAAI,EAAE,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC;SACzB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAAqC;YACrD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;YACtE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CAAC,OAAgB,EAAE,QAAyB,EAAE,QAAwB;QAI3F,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;QACjI,CAAC;QAED,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,UAAW,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,UAAW,CAAC,CAAC;QAE9F,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBACjE,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACxG,CAAC;IACL,CAAC;;AArgBL,kCAuiBC;AAphB6B;;;;WAAe;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;EAjBqC,CAiBpC;AA4gBN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAkB;IACrD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;IAElC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;YACpB,mBAAmB;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAA2B,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAED,OAAO,MAAkC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,4BAA4B,CAAC,QAAyB;IAC3D,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAChC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,YAAE,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;QACzE,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GAAG,oBAAI,CAAC,WAAW,CAAC,IAAA,mBAAO,EAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;eACrE,yCAAyC,CAAC,CAAC,0FAA0F;QAC5I,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,cAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC;AALD,4CAKC"}
|
|
1
|
+
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":";;;;AACA,yCAAoC;AACpC,kEAA6B;AAE7B,4CAAgE;AAChE,gDAA4E;AAa5E,0CAUwB;AAExB,0CAAkE;AAClE,yDAAmC;AAEnC,wEAA6C;AAG7C,oEAA+B;AAC/B,oEAA8B;AAC9B,oDAAoB;AAGpB,IAAI,YAAsC,CAAC;AAE3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAsKF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAa,WAAyF,SAAQ,oBAAqB;IAsC/H;;OAEG;IACH,YAAY,UAAuC,EAAE,EAAoB,SAAS,qBAAa,CAAC,eAAe,EAAE;QAC7G,IAAA,YAAE,EAAC,OAAO,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,cAAc,EACd,kBAAkB,EAElB,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,UAAU;QACV,qBAAqB;QAErB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,EAAE,MAAM,CAAC,CAAC;QAnCwC;;;;mBAAkB,MAAM;WAAkC;QAxCjH;;;WAGG;QACH;;;;;WAAwC;QAE9B;;;;;WAAwC;QACxC;;;;;WAAgD;QAChD;;;;;WAAiD;QACjD;;;;;WAAkC;QAClC;;;;;WAAgC;QAChC;;;;;WAAyB;QACzB;;;;;WAAiC;QACjC;;;;;WAA+B;QAC/B;;;;;WAA4C;QAC5C;;;;;WAAwC;QAC/B;;;;;WAAgC;QAqgBnD;;WAEG;QACK;;;;mBAAoB,KAAK,EAAE,OAAyC,EAAE,OAAiB,EAAE,EAAE;gBAC/F,qDAAqD;gBACrD,OAAO,IAAI,OAAO,CAAgB,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;oBACxD,8GAA8G;oBAC9G,6DAA6D;oBAC7D,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC,OAAO,CAAC,CAAC;oBAE1C,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,cAAuB,EAAE,gBAA+B,EAAE,EAAE;wBAC/E,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;4BAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;4BAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;4BAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;gCACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;4BACjD,CAAC;wBACL,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,yEAAyE;oBACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClE,MAAM,CAAC,GAAG,EAAE,CAAC;oBACjB,CAAC;oBAED,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;wBACvB,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC,CAAC;oBACnD,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;YACP,CAAC;WAAC;QAveE,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;QAC7H,CAAC;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAExC,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,iCAAiC,SAAS,YAAY,CAAC,CAAC;YAC3H,CAAC;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC,CAAC,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,OAAO,SAAS,IAAI,GAAG,KAAK,YAAY,IAAI,CAC1H,CAAC;YACN,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,mBAAmB,SAAS,IAAI,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE1G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,EAAE,eAAe,CAAC,CAAC;YAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;YACxC,IAAA,mBAAS,GAAE,CAAC;YAEZ,qEAAqE;YACrE,eAAe,CAAC,gBAAgB,KAAhC,eAAe,CAAC,gBAAgB,GAAK,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,EAAC;YAEvF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,GAAG,mDAAmD,CAAC,CAAC;gBAElH,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,OAAO,CAAC;gBAErC,OAAO;YACX,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,oBAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,IAAA,6BAAmB,EACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,EACjE,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;YACF,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,2BAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,SAAS,CAAC;QACvC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAmB,EAAE,UAAuB,EAAE,cAAsB,EAAE,eAAuB;QACnI,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EACF,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,GAC1B,GAAG,UAAU,CAAC,OAAO,CAAC;YAEvB,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAAC,CAAC;YAE/J,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG;YAClB,aAAa;YACb,cAAc;SACjB,CAAC;QAEF,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAlB,UAAU,CAAC,OAAO,GAAK,EAAE,EAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAA0B;QAC/F,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAqC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,oBAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B,EAAE,eAAwB;QACtG,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,IAAI,UAAW,IAAI,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAW,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QAE1E,IAAI,CAAC,UAAW,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,uCAAuC,CAAC,CAAC;YAC1E,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;YACvE,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;QACvD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;QAC1I,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,MAAe,EAAE,gBAAyB;QAC5F,OAAO;YACH,IAAI,EAAE,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC;SACzB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAAqC;YACrD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;YACtE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CAAC,OAAgB,EAAE,QAAyB,EAAE,QAAwB;QAI3F,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;QACjI,CAAC;QAED,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,UAAW,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,UAAW,CAAC,CAAC;QAE9F,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBACjE,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACxG,CAAC;IACL,CAAC;;AAphBL,kCAsjBC;AAniB6B;;;;WAAe;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;EAjBqC,CAiBpC;AA2hBN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAkB;IACrD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;IAElC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;YACpB,mBAAmB;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAA2B,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAED,OAAO,MAAkC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,4BAA4B,CAAC,QAAyB;IAC3D,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAChC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,YAAE,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;QACzE,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GAAG,oBAAI,CAAC,WAAW,CAAC,IAAA,mBAAO,EAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;eACrE,yCAAyC,CAAC,CAAC,0FAA0F;QAC5I,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,cAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC;AALD,4CAKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.7.1-beta.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,9 +55,9 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.3.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/basic": "3.
|
|
59
|
-
"@crawlee/types": "3.
|
|
60
|
-
"@crawlee/utils": "3.
|
|
58
|
+
"@crawlee/basic": "^3.7.1-beta.0",
|
|
59
|
+
"@crawlee/types": "^3.7.1-beta.0",
|
|
60
|
+
"@crawlee/utils": "^3.7.1-beta.0",
|
|
61
61
|
"@types/content-type": "^1.1.5",
|
|
62
62
|
"cheerio": "^1.0.0-rc.12",
|
|
63
63
|
"content-type": "^1.0.4",
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "e8ee23c7c9ad210070cfb38f670310c39a035612"
|
|
79
79
|
}
|