@crawlee/http 4.0.0-beta.2 → 4.0.0-beta.21

package/internals/http-crawler.d.ts

@@ -1,25 +1,13 @@
-import type { IncomingMessage } from 'node:http';
-import type { Readable } from 'node:stream';
-import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterRoutes, Session } from '@crawlee/basic';
-import { BasicCrawler, Configuration, CrawlerExtension } from '@crawlee/basic';
-import type { HttpResponse } from '@crawlee/core';
+import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
+import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
+import type { LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import type { RequestLike, ResponseLike } from 'content-type';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { Method, OptionsInit } from 'got-scraping';
-import { ObjectPredicate } from 'ow';
 import type { JsonValue } from 'type-fest';
-/**
- * TODO exists for BC within HttpCrawler - replace completely with StreamingHttpResponse in 4.0
- * @internal
- */
-export type PlainResponse = Omit<HttpResponse, 'body'> & IncomingMessage & {
-    body?: unknown;
-};
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
-export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext> extends BasicCrawlerOptions<Context> {
+export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
     /**
      * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
      */
@@ -28,20 +16,14 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * If set to true, SSL certificate errors will be ignored.
      */
     ignoreSslErrors?: boolean;
-    /**
-     * If set, this crawler will be configured for all connections to use
-     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
-     * For more information, see the [documentation](https://docs.apify.com/proxy).
-     */
-    proxyConfiguration?: ProxyConfiguration;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
-     * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
+     * which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
      * Example:
      * ```
      * preNavigationHooks: [
-     *     async (crawlingContext, gotOptions) => {
+     *     async (crawlingContext) => {
      *         // ...
      *     },
      * ]
@@ -50,7 +32,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * Modyfing `pageOptions` is supported only in Playwright incognito.
      * See {@link PrePageCreateHook}
      */
-    preNavigationHooks?: InternalHttpHook<Context>[];
+    preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
@@ -63,7 +45,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * ]
      * ```
      */
-    postNavigationHooks?: InternalHttpHook<Context>[];
+    postNavigationHooks?: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
     /**
      * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
      * you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
@@ -112,15 +94,24 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
 /**
  * @internal
  */
-export type InternalHttpHook<Context> = (crawlingContext: Context, gotOptions: OptionsInit) => Awaitable<void>;
+export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
 export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
+interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
+    /**
+     * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
+     */
+    request: LoadedRequest<CrawleeRequest<UserData>>;
+    /**
+     * The HTTP response object containing status code, headers, and other response metadata.
+     */
+    response: Response;
+}
 /**
  * @internal
  */
 export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
-Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
+JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
     /**
      * The request body of the web page.
      * The type depends on the `Content-Type` header of the web page:
@@ -139,7 +130,6 @@ Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
         type: string;
         encoding: BufferEncoding;
     };
-    response: PlainResponse;
     /**
      * Wait for an element matching the selector to appear. Timeout is ignored.
      *
@@ -167,7 +157,7 @@ Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
 }
-export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData, HttpCrawler<HttpCrawlingContext<UserData, JSONData>>> {
+export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
 }
 export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
@@ -192,11 +182,11 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```javascript
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
@@ -238,16 +228,10 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * ```
  * @category Crawlers
  */
-export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, HttpCrawler<Context>>> extends BasicCrawler<Context> {
+export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
     readonly config: Configuration;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration?: ProxyConfiguration;
-    protected userRequestHandlerTimeoutMillis: number;
-    protected preNavigationHooks: InternalHttpHook<Context>[];
-    protected postNavigationHooks: InternalHttpHook<Context>[];
+    protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
+    protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
     protected persistCookiesPerSession: boolean;
     protected navigationTimeoutMillis: number;
     protected ignoreSslErrors: boolean;
@@ -267,8 +251,6 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -279,6 +261,10 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -299,12 +285,16 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -312,7 +302,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -331,135 +321,68 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options?: HttpCrawlerOptions<Context>, config?: Configuration);
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension: CrawlerExtension): void;
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    protected _runRequestHandler(crawlingContext: Context): Promise<void>;
-    protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
-    protected _handleNavigation(crawlingContext: Context): Promise<void>;
+    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>, config?: Configuration);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
+    private makeHttpRequest;
+    private processHttpResponse;
+    private handleBlockedRequestByContent;
+    protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
     /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
+     * Returns the `Cookie` header value based on the current context and
+     * any changes that occurred in the navigation hooks.
      */
-    protected _applyCookies({ session, request }: CrawlingContext, gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string): void;
+    protected _applyCookies({ session, request }: CrawlingContext, preHookCookies: string, postHookCookies: string): string;
     /**
      * Function to make the HTTP request. It performs optimizations
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    protected _requestFunction({ request, session, proxyUrl, gotOptions, }: RequestFunctionOptions): Promise<PlainResponse>;
+    protected _requestFunction({ request, session, proxyUrl, cookieString, }: RequestFunctionOptions): Promise<Response>;
     /**
      * Encodes and parses response according to the provided content type
      */
-    protected _parseResponse(request: Request, responseStream: IncomingMessage, crawlingContext: Context): Promise<(Partial<Context> & {
-        isXml: boolean;
-        response: IncomingMessage;
+    protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
+        response: Response;
         contentType: {
             type: string;
             encoding: BufferEncoding;
         };
-    }) | {
-        body: Buffer<ArrayBufferLike>;
-        response: IncomingMessage;
+        body: string;
+    } | {
+        body: Buffer<ArrayBuffer>;
+        response: Response;
         contentType: {
             type: string;
             encoding: BufferEncoding;
         };
-        enqueueLinks: () => Promise<{
-            processedRequests: never[];
-            unprocessedRequests: never[];
-        }>;
     }>;
-    protected _parseHTML(response: IncomingMessage, _isXml: boolean, _crawlingContext: Context): Promise<Partial<Context>>;
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        body?: string | Buffer | Readable | Generator | AsyncGenerator | import("form-data-encoder").FormDataLike | undefined;
-        json?: unknown;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        request?: import("got-scraping").RequestFunction | undefined;
-        url?: string | URL | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        headers?: import("got-scraping").Headers | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        agent?: import("got-scraping").Agents | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        h2session?: import("http2").ClientHttp2Session | undefined;
-        decompress?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        timeout?: import("got-scraping").Delays | undefined;
-        prefixUrl?: string | URL | undefined;
-        form?: Record<string, any> | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cookieJar?: import("got-scraping").PromiseCookieJar | import("got-scraping").ToughCookieJar | undefined;
-        signal?: AbortSignal | undefined;
-        ignoreInvalidCookies?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        searchParams?: string | import("got-scraping").SearchParameters | URLSearchParams | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsLookup?: import("cacheable-lookup").default["lookup"] | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsCache?: import("cacheable-lookup").default | boolean | undefined;
-        context?: Record<string, unknown> | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        followRedirect?: boolean | ((response: import("got-scraping").PlainResponse) => boolean) | undefined;
-        maxRedirects?: number | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cache?: string | import("cacheable-request").StorageAdapter | boolean | undefined;
-        throwHttpErrors?: boolean | undefined;
-        username?: string | undefined;
-        password?: string | undefined;
-        http2?: boolean | undefined;
-        allowGetBody?: boolean | undefined;
-        methodRewriting?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsLookupIpVersion?: import("got-scraping").DnsLookupIpVersion;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        parseJson?: import("got-scraping").ParseJsonFunction | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        stringifyJson?: import("got-scraping").StringifyJsonFunction | undefined;
-        localAddress?: string | undefined;
-        method?: Method | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        createConnection?: import("got-scraping").CreateConnectionFunction | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cacheOptions?: import("got-scraping").CacheOptions | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        https?: import("got-scraping").HttpsOptions | undefined;
-        encoding?: BufferEncoding | undefined;
-        resolveBodyOnly?: boolean | undefined;
-        isStream?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        responseType?: import("got-scraping").ResponseType | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        pagination?: import("got-scraping").PaginationOptions<unknown, unknown> | undefined;
-        setHost?: boolean | undefined;
-        maxHeaderSize?: number | undefined;
-        enableUnixSockets?: boolean | undefined;
-    } & {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        hooks?: Partial<import("got-scraping").Hooks>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retry?: Partial<import("got-scraping").RetryOptions>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-    } & import("got-scraping").Context & Required<Pick<OptionsInit, "url">> & {
-        isStream: true;
+    protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string): {
+        url: string;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        method: import("@crawlee/types").AllowedHttpMethods;
+        proxyUrl: string | undefined;
+        timeout: number;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        cookieJar: import("tough-cookie").CookieJar | undefined;
+        sessionToken: Session | undefined;
+        headers: Record<string, string> | undefined;
+        https: {
+            rejectUnauthorized: boolean;
+        };
+        body: string | undefined;
     };
-    protected _encodeResponse(request: Request, response: IncomingMessage, encoding: BufferEncoding): {
+    protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
         encoding: BufferEncoding;
-        response: IncomingMessage;
+        response: Response;
     };
     /**
      * Checks and extends supported mime types
@@ -476,10 +399,10 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
     private _requestAsBrowser;
 }
 interface RequestFunctionOptions {
-    request: Request;
+    request: CrawleeRequest;
     session?: Session;
     proxyUrl?: string;
-    gotOptions: OptionsInit;
+    cookieString?: string;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/http-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAuB,eAAe,EAAE,MAAM,WAAW,CAAC;AAEtE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAG5C,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EAEtB,kBAAkB,EAClB,OAAO,EACP,cAAc,EACd,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEH,YAAY,EACZ,aAAa,EACb,gBAAgB,EAOnB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAyB,MAAM,eAAe,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAqC,MAAM,cAAc,CAAC;AAG3F,OAAW,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC;AACzC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAO3C;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,GAClD,eAAe,GAAG;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAkBN,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CACzG,SAAQ,mBAAmB,CAAC,OAAO,CAAC;IACpC;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAEjD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAElD;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE/G,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,EAAE,2EAA2E;AAC7G,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAC5B,SAAQ,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC;IACxC;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IACxD,QAAQ,EAAE,aAAa,CAAC;IAExB;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;CAAG;AAEpH,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAC7E,SAAQ,YAAY,CAAC,OAAO,CAAC;aA0CL,MAAM;IAzC5B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,SAAS,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3D,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAgBpC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,CAAM,EACvB,MAAM,gBAAkC;IA2E9D;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IAkC/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;cAuF3C,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;cAa5E,iBAAiB,CAAC,eAAe,EAAE,OAAO;IA6B1D;;OAEG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,UAAU,EAAE,WAAW,EACvB,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM;IAkD3B;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACb,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;IAwBlD;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,eAAe,EAAE,OAAO;;;;;;;;;;;;;;;;;;;cA8C1F,UAAU,CACtB,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,OAAO,EACf,gBAAgB,EAAE,OAAO,GAC1B,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAM5B;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBACpB,IAAI;;IAkC7F,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,eAAe,EACzB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;IAqCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAuBvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AAkFD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
+{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,IAAI,cAAc,EACzB,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAOlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAwB3C,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,mBAAmB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrE;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtF,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEjD;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAmCxC,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC,GACnE,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAkDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;;OAGG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACxB,MAAM;IAQT;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,YAAY,GACf,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2C1E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM;;;;;;;;;;;cAY/D,MAAM,GAAG,SAAS;;IAmB7C,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAsCvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}