@crawlee/basic 3.0.0-beta.7 → 3.0.0-beta.72

package/internals/basic-crawler.d.ts

@@ -1,65 +1,48 @@
-import { Log } from '@apify/log';
-import { AutoscaledPool, AutoscaledPoolOptions, EnqueueLinksOptions, CrawlerHandleFailedRequestInput, ProxyInfo, QueueOperationInfo, Request, RequestList, RequestQueue, Session, SessionPool, SessionPoolOptions, Statistics, type CrawlingContext, RequestOptions, RequestQueueOperationOptions, Configuration, EventManager, FinalStatistics } from '@crawlee/core';
-import { Response as GotResponse } from 'got-scraping';
-import { ProcessedRequest } from '@crawlee/types';
-import { Awaitable } from '@crawlee/utils';
-export interface BasicCrawlerCrawlingContext extends CrawlingContext {
-    crawler: BasicCrawler;
-    enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<QueueOperationInfo[]>;
-    sendRequest: (request?: Request) => Promise<GotResponse<string>>;
-}
-export interface BasicCrawlerHandleFailedRequestInput extends CrawlerHandleFailedRequestInput {
+import type { Log } from '@apify/log';
+import type { AutoscaledPoolOptions, EnqueueLinksOptions, EventManager, FinalStatistics, ProxyInfo, Request, RequestList, RequestOptions, RequestQueueOperationOptions, RouterHandler, Session, SessionPoolOptions } from '@crawlee/core';
+import { AutoscaledPool, Configuration, type CrawlingContext, RequestQueue, SessionPool, Statistics } from '@crawlee/core';
+import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';
+import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';
+export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
     crawler: BasicCrawler;
+    enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+    sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
 }
 /** @internal */
 export declare type BasicCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue'>;
-export declare type RequestHandler<Context extends CrawlingContext = BasicCrawlerCrawlingContext> = (inputs: Context) => Awaitable<void>;
-export declare type FailedRequestHandler<Inputs extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> = (inputs: Inputs) => Awaitable<void>;
-export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlerCrawlingContext, ErrorContext extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> {
+export declare type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context) => Awaitable<void>;
+export declare type FailedRequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
+export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     session: Session,
-     *     crawler: BasicCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance represents the URL to crawl.
+     * The function receives the {@link BasicCrawlingContext} as an argument,
+     * where the {@link BasicCrawlingContext.request} represents the URL to crawl.
      *
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
      * request later, up to `option.maxRequestRetries` times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
+     * provided to the `failedRequestHandler` parameter.
      * To make this work, you should **always**
      * let your function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage} function.
      */
-    requestHandler: RequestHandler<Context>;
+    requestHandler?: RequestHandler<Context>;
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     session: Session,
-     *     crawler: BasicCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance represents the URL to crawl.
+     * The function receives the {@link BasicCrawlingContext} as an argument,
+     * where the {@link BasicCrawlingContext.request} represents the URL to crawl.
      *
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
      * request later, up to `option.maxRequestRetries` times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
+     * provided to the `failedRequestHandler` parameter.
      * To make this work, you should **always**
      * let your function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
@@ -90,49 +73,37 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     handleRequestTimeoutSecs?: number;
     /**
-     * A function to handle requests that failed more than `option.maxRequestRetries` times.
+     * User-provided function that allows modifying the request object before it gets retried by the crawler.
+     * It's executed before each retry for the requests that failed less than `option.maxRequestRetries` times.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     error: Error,
-     *     session: Session,
-     *     crawler: BasicCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * The function receives the {@link BasicCrawlingContext} as the first argument,
+     * where the {@link BasicCrawlingContext.request} corresponds to the request to be retried.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
-     *
-     * See
-     * [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
-     * for the default implementation of this function.
      */
-    failedRequestHandler?: FailedRequestHandler<ErrorContext>;
+    errorHandler?: FailedRequestHandler<Context>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     error: Error,
-     *     session: Session,
-     *     crawler: BasicCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * The function receives the {@link BasicCrawlingContext} as the first argument,
+     * where the {@link BasicCrawlingContext.request} corresponds to the failed request.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
+     */
+    failedRequestHandler?: FailedRequestHandler<Context>;
+    /**
+     * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
-     * See
-     * [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
-     * for the default implementation of this function.
+     * The function receives the {@link BasicCrawlingContext} as the first argument,
+     * where the {@link BasicCrawlingContext.request} corresponds to the failed request.
+     * Second argument is the `Error` instance that
+     * represents the last error thrown during processing of the request.
      *
      * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
      */
-    handleFailedRequestFunction?: FailedRequestHandler<ErrorContext>;
+    handleFailedRequestFunction?: FailedRequestHandler<Context>;
     /**
-     * Indicates how many times the request is retried if {@link requestHandler} or {@link handlePageFunction} fails.
+     * Indicates how many times the request is retried if {@link requestHandler} fails.
      * @default 3
      */
     maxRequestRetries?: number;
@@ -161,7 +132,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     maxConcurrency?: number;
     /**
-     * Basic crawler will initialize the  {@link SessionPool} with the corresponding `sessionPoolOptions`.
+     * The maximum number of requests per minute the crawler should run.
+     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
+     * Shortcut for the {@link AutoscaledPoolOptions.maxTasksPerMinute} option.
+     */
+    maxRequestsPerMinute?: number;
+    /**
+     * Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
      * The session instance will be than available in the `requestHandler`.
      */
     useSessionPool?: boolean;
@@ -203,16 +180,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  * **Example usage:**
  *
  * ```javascript
- * const { gotScraping } = require('got-scraping');
+ * import { gotScraping } from 'got-scraping';
  *
  * // Prepare a list of URLs to crawl
- * const requestList = new RequestList({
- *   sources: [
- *       { url: 'http://www.example.com/page-1' },
- *       { url: 'http://www.example.com/page-2' },
- *   ],
- * });
- * await requestList.initialize();
+ * const requestList = await RequestList.open(null, [
+ *     { url: 'http://www.example.com/page-1' },
+ *     { url: 'http://www.example.com/page-2' },
+ * ]);
  *
  * // Crawl the URLs
  * const crawler = new BasicCrawler({
@@ -227,7 +201,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  *             headers: request.headers,
  *         });
  *
- *         await Actor.pushData({
+ *         await Dataset.pushData({
  *             url: request.url,
  *             html: body,
  *         })
@@ -238,8 +212,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  * ```
  * @category Crawlers
  */
-export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlerCrawlingContext, ErrorContext extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> {
+export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext> {
     readonly config: Configuration;
+    private static readonly CRAWLEE_STATE_KEY;
     /**
      * Static list of URLs to be processed.
      */
@@ -270,9 +245,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
      * or to abort it by calling {@link AutoscaledPool.abort}.
      */
     autoscaledPool?: AutoscaledPool;
+    /**
+     * Default router instance that will be used if we don't specify any {@link requestHandler}.
+     * See {@link Router.addHandler} and {@link Router.addDefaultHandler.}
+     */
+    readonly router: RouterHandler<Context>;
     protected log: Log;
     protected requestHandler: RequestHandler<Context>;
-    protected failedRequestHandler?: FailedRequestHandler<ErrorContext>;
+    protected errorHandler?: FailedRequestHandler<Context>;
+    protected failedRequestHandler?: FailedRequestHandler<Context>;
     protected requestHandlerTimeoutMillis: number;
     protected internalTimeoutMillis: number;
     protected maxRequestRetries: number;
@@ -282,6 +263,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
     protected crawlingContexts: Map<string, Context>;
     protected autoscaledPoolOptions: AutoscaledPoolOptions;
     protected events: EventManager;
+    private _closeEvents?;
     protected static optionsShape: {
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
@@ -289,6 +271,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
         handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -298,17 +281,24 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
         useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options: BasicCrawlerOptions<Context, ErrorContext>, config?: Configuration);
+    constructor(options?: BasicCrawlerOptions<Context>, config?: Configuration);
     /**
      * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
+     * We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
+     * running `crawler.addRequests()` before the `crawler.run()`.
+     *
+     * @param [requests] The requests to add
+     * @param [options] Options for the request queue
      */
-    run(): Promise<FinalStatistics>;
+    run(requests?: (string | Request | RequestOptions)[], options?: CrawlerAddRequestsOptions): Promise<FinalStatistics>;
     getRequestQueue(): Promise<RequestQueue>;
+    useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
     /**
      * Adds requests to be processed by the crawler
      * @param requests The requests to add
@@ -326,7 +316,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
      * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
      * and RequestQueue is present then enqueues it to the queue first.
      */
-    protected _fetchNextRequest(): Promise<Request | null>;
+    protected _fetchNextRequest(): Promise<Request<Dictionary<any>> | null>;
     /**
      * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
      * then retries them in a case of an error, etc.
@@ -349,7 +339,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
      * Handles errors thrown by user provided requestHandler()
      */
     protected _requestFunctionErrorHandler(error: Error, crawlingContext: Context, source: RequestList | RequestQueue): Promise<void>;
-    protected _handleFailedRequestHandler(crawlingContext: ErrorContext): Promise<void>;
+    protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
+    protected _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void>;
     /**
      * Updates handledRequestsCount from possibly stored counts,
      * usually after worker migration. Since one of the stores
@@ -406,5 +397,30 @@ interface HandlePropertyNameChangeData<New, Old> {
     propertyKey: string;
     allowUndefined?: boolean;
 }
+/**
+ * Creates new {@link Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@link BasicCrawler}.
+ * Defaults to the {@link BasicCrawlingContext}.
+ *
+ * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
+ *
+ * ```ts
+ * import { BasicCrawler, createBasicRouter } from 'crawlee';
+ *
+ * const router = createBasicRouter();
+ * router.addHandler('label-a', async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ * router.addDefaultHandler(async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: router,
+ * });
+ * await crawler.run();
+ * ```
+ */
+export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext>(): RouterHandler<Context>;
 export {};
 //# sourceMappingURL=basic-crawler.d.ts.map

package/internals/basic-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAmB,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAG7C,OAAO,EACH,cAAc,EACd,qBAAqB,EACrB,mBAAmB,EACnB,+BAA+B,EAE/B,SAAS,EACT,kBAAkB,EAClB,OAAO,EACP,WAAW,EACX,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,UAAU,EAEV,KAAK,eAAe,EACpB,cAAc,EACd,4BAA4B,EAE5B,aAAa,EACb,YAAY,EAEZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAA0C,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC/F,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAgB,MAAM,gBAAgB,CAAC;AAGzD,MAAM,WAAW,2BAA4B,SAAQ,eAAe;IAChE,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC1F,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,oCAAqC,SAAQ,+BAA+B;IACzF,OAAO,EAAE,YAAY,CAAC;CACzB;AAED,gBAAgB;AAChB,oBAAY,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAA;AAavF,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,2BAA2B,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEjI,oBAAY,oBAAoB,CAAC,MAAM,SAAS,+BAA+B,GAAG,oCAAoC,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9J,MAAM,WAAW,mBAAmB,CAChC,OAAO,SAAS,eAAe,GAAG,2BAA2B,EAC7D,YAAY,SAAS,+BAA+B,GAAG,oCAAoC;IAE3F;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAE1D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,2BAA2B,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAEjE;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,qBAAa,YAAY,CACrB,OAAO,SAAS,eAAe,GAAG,2BAA2B,EAC7D,YAAY,SAAS,+BAA+B,GAAG,oCAAoC;IAmF1B,QAAQ,CAAC,MAAM;IAjFhF;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IACpE,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;MA4B3B;IAEF;;OAEG;gBACS,OAAO,EAAE,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,EAAW,MAAM,gBAAkC;IAsJlH;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IAuB/B,eAAe;IAMrB;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAetB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IA+GhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cA+BA,2BAA2B,CAAC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAezF;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAM/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAGzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;OAYG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B"}
+{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,EAGb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,cAAc,EAA6B,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAEvG,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAItG,MAAM,WAAW,oBAAoB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC7G,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC5F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,gBAAgB;AAChB,oBAAY,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAA;AAavF,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE1H,oBAAY,oBAAoB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9I,MAAM,WAAW,mBAAmB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IACvF;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;;;;;;;;;;;;;OAkBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD;;;;;;;;;OASG;IACH,2BAA2B,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE5D;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IA6FpB,QAAQ,CAAC,MAAM;IA5FvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAE5D;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAA4B;IAEnE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,YAAY,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACvD,SAAS,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC/D,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAU;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;;;MA6B3B;IAEF;;OAEG;gBACS,OAAO,GAAE,mBAAmB,CAAC,OAAO,CAAM,EAAW,MAAM,gBAAkC;IAiKzG;;;;;;;OAOG;IACG,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,eAAe,CAAC;IA6BpH,eAAe;IAMf,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,QAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAKjG;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAoBtB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IAoHhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cAoCA,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;cAStD,2BAA2B,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAclG;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAY/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAGzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;OAYG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,4BAE5F"}