@crawlee/basic 3.0.3-beta.8 → 3.0.4-beta.0

package/README.md

@@ -63,7 +63,7 @@ Additionally, the package provides various helper functions to simplify running
 
 ## Quick Start
 
-This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/guides/getting-started) tutorial that will take you step by step through creating your first scraper.
+This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/introduction) tutorial that will take you step by step through creating your first scraper.
 
 ### Local stand-alone usage
 

package/index.mjs

@@ -33,6 +33,7 @@ export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
 export const Request = mod.Request;
 export const RequestList = mod.RequestList;
 export const RequestQueue = mod.RequestQueue;
+export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
 export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;

package/internals/basic-crawler.d.ts

@@ -17,37 +17,37 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
-     * The function receives the {@link BasicCrawlingContext} as an argument,
-     * where the {@link BasicCrawlingContext.request|`request`} represents the URL to crawl.
+     * The function receives the {@apilink BasicCrawlingContext} as an argument,
+     * where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl.
      *
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
+     * request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      * If all the retries fail, the crawler calls the function
-     * provided to the {@link BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
+     * provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
     requestHandler?: RequestHandler<Context>;
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
-     * The function receives the {@link BasicCrawlingContext} as an argument,
-     * where the {@link BasicCrawlingContext.request|`request`} represents the URL to crawl.
+     * The function receives the {@apilink BasicCrawlingContext} as an argument,
+     * where the {@apilink BasicCrawlingContext.request|`request`} represents the URL to crawl.
      *
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
+     * request later, up to the {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      * If all the retries fail, the crawler calls the function
-     * provided to the {@link BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
+     * provided to the {@apilink BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      *
      * @deprecated `handleRequestFunction` has been renamed to `requestHandler` and will be removed in a future version.
      * @ignore
@@ -55,25 +55,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
     handleRequestFunction?: RequestHandler<Context>;
     /**
      * Static list of URLs to be processed.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
+     * If not provided, the crawler will open the default request queue when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
+     * > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
      * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
      */
     requestList?: RequestList;
     /**
      * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
+     * If not provided, the crawler will open the default request queue when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
+     * > Alternatively, `requests` parameter of {@apilink BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
      * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
      */
     requestQueue?: RequestQueue;
     /**
-     * Timeout in which the function passed as {@link BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
+     * Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
      * @default 60
      */
     requestHandlerTimeoutSecs?: number;
     /**
-     * Timeout in which the function passed as {@link BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
+     * Timeout in which the function passed as {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
      * @default 60
      * @deprecated `handleRequestTimeoutSecs` has been renamed to `requestHandlerTimeoutSecs` and will be removed in a future version.
      * @ignore
@@ -81,28 +81,28 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
     handleRequestTimeoutSecs?: number;
     /**
      * User-provided function that allows modifying the request object before it gets retried by the crawler.
-     * It's executed before each retry for the requests that failed less than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
+     * It's executed before each retry for the requests that failed less than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      *
-     * The function receives the {@link BasicCrawlingContext} as the first argument,
-     * where the {@link BasicCrawlingContext.request|`request`} corresponds to the request to be retried.
+     * The function receives the {@apilink BasicCrawlingContext} as the first argument,
+     * where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the request to be retried.
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
     errorHandler?: ErrorHandler<Context>;
     /**
-     * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
+     * A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      *
-     * The function receives the {@link BasicCrawlingContext} as the first argument,
-     * where the {@link BasicCrawlingContext.request|`request`} corresponds to the failed request.
+     * The function receives the {@apilink BasicCrawlingContext} as the first argument,
+     * where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request.
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
     failedRequestHandler?: ErrorHandler<Context>;
     /**
-     * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
+     * A function to handle requests that failed more than {@apilink BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      *
-     * The function receives the {@link BasicCrawlingContext} as the first argument,
-     * where the {@link BasicCrawlingContext.request|`request`} corresponds to the failed request.
+     * The function receives the {@apilink BasicCrawlingContext} as the first argument,
+     * where the {@apilink BasicCrawlingContext.request|`request`} corresponds to the failed request.
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      *
@@ -111,7 +111,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     handleFailedRequestFunction?: ErrorHandler<Context>;
     /**
-     * Indicates how many times the request is retried if {@link BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
+     * Indicates how many times the request is retried if {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
      * @default 3
      */
     maxRequestRetries?: number;
@@ -122,38 +122,44 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     maxRequestsPerCrawl?: number;
     /**
-     * Custom options passed to the underlying {@link AutoscaledPool} constructor.
-     * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
-     * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`} options
+     * Custom options passed to the underlying {@apilink AutoscaledPool} constructor.
+     * > *NOTE:* The {@apilink AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
+     * and {@apilink AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`} options
      * are provided by the crawler and cannot be overridden.
-     * However, we can provide a custom implementation of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}.
+     * However, we can provide a custom implementation of {@apilink AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}.
      */
     autoscaledPoolOptions?: AutoscaledPoolOptions;
     /**
      * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
+     * AutoscaledPool {@apilink AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
      * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
      * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
      */
     minConcurrency?: number;
     /**
      * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
+     * AutoscaledPool {@apilink AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
      */
     maxConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
-     * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
+     * Shortcut for the AutoscaledPool {@apilink AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
      */
     maxRequestsPerMinute?: number;
     /**
-     * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
-     * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
+     * Allows to keep the crawler alive even if the {@apilink RequestQueue} gets empty.
+     * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
+     * waiting for more requests to come. Use `crawler.teardown()` to exit the crawler.
+     */
+    keepAlive?: boolean;
+    /**
+     * Basic crawler will initialize the {@apilink SessionPool} with the corresponding {@apilink SessionPoolOptions|`sessionPoolOptions`}.
+     * The session instance will be than available in the {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
      */
     useSessionPool?: boolean;
     /**
-     * The configuration options for {@link SessionPool} to use.
+     * The configuration options for {@apilink SessionPool} to use.
      */
     sessionPoolOptions?: SessionPoolOptions;
     /** @internal */
@@ -167,29 +173,29 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
  * `BasicCrawler` is a low-level tool that requires the user to implement the page
  * download and data extraction functionality themselves.
  * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ * we should consider using {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
  *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
+ * `BasicCrawler` invokes the user-provided {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}
+ * for each {@apilink Request} object, which represents a single URL to crawl.
+ * The {@apilink Request} objects are fed from the {@apilink RequestList} or {@apilink RequestQueue}
+ * instances provided by the {@apilink BasicCrawlerOptions.requestList|`requestList`} or {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`}
  * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
+ * the crawler will open the default request queue either when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
+ * or if `requests` parameter (representing the initial requests) of the {@apilink BasicCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * If both {@apilink BasicCrawlerOptions.requestList|`requestList`} and {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
+ * to the {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
+ * The crawler finishes if there are no more {@apilink Request} objects to crawl.
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
+ * using the functionality provided by the {@apilink AutoscaledPool} class.
+ * All {@apilink AutoscaledPool} configuration options can be passed to the {@apilink BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
  * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
+ * For user convenience, the {@apilink AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
+ * {@apilink AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
+ * underlying {@apilink AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
  *
  * **Example usage:**
  *
@@ -227,36 +233,36 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     readonly config: Configuration;
     private static readonly CRAWLEE_STATE_KEY;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.
      */
     readonly stats: Statistics;
     /**
-     * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
+     * A reference to the underlying {@apilink RequestList} class that manages the crawler's {@apilink Request|requests}.
      * Only available if used by the crawler.
      */
     requestList?: RequestList;
     /**
      * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
+     * A reference to the underlying {@apilink RequestQueue} class that manages the crawler's {@apilink Request|requests}.
      * Only available if used by the crawler.
      */
     requestQueue?: RequestQueue;
     /**
-     * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
+     * A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
      * Only available if used by the crawler.
      */
     sessionPool?: SessionPool;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
-     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
+     * A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
+     * > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
      * We can use it to change the concurrency settings on the fly,
-     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     * to pause the crawler by calling {@apilink AutoscaledPool.pause|`autoscaledPool.pause()`}
+     * or to abort it by calling {@apilink AutoscaledPool.abort|`autoscaledPool.abort()`}.
      */
     autoscaledPool?: AutoscaledPool;
     /**
-     * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
-     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
+     * Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
+     * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context>;
     protected log: Log;
@@ -291,6 +297,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
         log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
     /**
@@ -300,7 +307,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     /**
      * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
      * We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
-     * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before the {@link BasicCrawler.run|`crawler.run()`}.
+     * running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before the {@apilink BasicCrawler.run|`crawler.run()`}.
      *
      * @param [requests] The requests to add
      * @param [options] Options for the request queue
@@ -350,6 +357,14 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     protected _requestFunctionErrorHandler(error: Error, crawlingContext: Context, source: RequestList | RequestQueue): Promise<void>;
     protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
     protected _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void>;
+    /**
+     * Resolves the most verbose error message from a thrown error
+     * @param error The error received
+     * @returns The message to be logged
+     */
+    protected _getMessageFromError(error: Error, forceStack?: boolean): string | Error | undefined;
+    protected _canRequestBeRetried(request: Request, error: Error): boolean;
+    protected _augmentContextWithDeprecatedError(context: Context, error: Error): Context;
     /**
      * Updates handledRequestsCount from possibly stored counts,
      * usually after worker migration. Since one of the stores
@@ -385,8 +400,8 @@ export interface CrawlerAddRequestsResult {
     /**
      * A promise which will resolve with the rest of the requests that were added to the queue.
      *
-     * Alternatively, we can set {@link CrawlerAddRequestsOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} to `true`
-     * in the {@link BasicCrawler.addRequests|`crawler.addRequests()`} options.
+     * Alternatively, we can set {@apilink CrawlerAddRequestsOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} to `true`
+     * in the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} options.
      *
      * **Example:**
      *
@@ -409,9 +424,9 @@ interface HandlePropertyNameChangeData<New, Old> {
     allowUndefined?: boolean;
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
- * Defaults to the {@link BasicCrawlingContext}.
+ * Creates new {@apilink Router} instance that works based on request labels.
+ * This instance can then serve as a {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@apilink BasicCrawler}.
+ * Defaults to the {@apilink BasicCrawlingContext}.
  *
  * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
  *

package/internals/basic-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,EAGb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAU,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAEjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAItG,MAAM,WAAW,oBAAoB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC7G,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC5F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CACzF;AAED,gBAAgB;AAChB,MAAM,WAAW,+BAAgC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC;CAAG;AAarG,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE1H,oBAAY,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtI,MAAM,WAAW,mBAAmB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IACvF;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAErC;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;;;;;OAUG;IACH,2BAA2B,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAEpD;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IA2FpB,QAAQ,CAAC,MAAM;IA1FvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAE5D;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAA4B;IAEnE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC/C,SAAS,CAAC,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACvD,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAU;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;;;MA6B3B;IAEF;;OAEG;gBACS,OAAO,GAAE,mBAAmB,CAAC,OAAO,CAAM,EAAW,MAAM,gBAAkC;IAiKzG;;;;;;;OAOG;IACG,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,eAAe,CAAC;IA2CpH,eAAe;IAMf,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,QAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAKjG;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAoBtB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IAsHhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cAoCA,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;cAStD,2BAA2B,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAclG;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAY/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAQzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,4BAE5F"}
+{"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,EAIb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAU,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAEjF,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAItG,MAAM,WAAW,oBAAoB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC7G,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC5F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CACzF;AAED,gBAAgB;AAChB,MAAM,WAAW,+BAAgC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC;CAAG;AAarG,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE1H,oBAAY,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtI,MAAM,WAAW,mBAAmB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IACvF;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAErC;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;;;;;OAUG;IACH,2BAA2B,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAEpD;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IA4FpB,QAAQ,CAAC,MAAM;IA3FvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAE5D;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAA4B;IAEnE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC/C,SAAS,CAAC,oBAAoB,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACvD,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAU;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;;;;MA8B3B;IAEF;;OAEG;gBACS,OAAO,GAAE,mBAAmB,CAAC,OAAO,CAAM,EAAW,MAAM,gBAAkC;IAuKzG;;;;;;;OAOG;IACG,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,eAAe,CAAC;IA2CpH,eAAe;IAMf,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,QAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAKjG;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAoBtB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IAsHhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cAsCA,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;cAStD,2BAA2B,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAelG;;;;OAIG;IACH,SAAS,CAAC,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,UAAQ;IAW/D,SAAS,CAAC,oBAAoB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAe7D,SAAS,CAAC,kCAAkC,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAa3E;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAQzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,4BAE5F"}

package/internals/basic-crawler.js

@@ -27,29 +27,29 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
  * `BasicCrawler` is a low-level tool that requires the user to implement the page
  * download and data extraction functionality themselves.
  * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ * we should consider using {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler}.
  *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
+ * `BasicCrawler` invokes the user-provided {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}
+ * for each {@apilink Request} object, which represents a single URL to crawl.
+ * The {@apilink Request} objects are fed from the {@apilink RequestList} or {@apilink RequestQueue}
+ * instances provided by the {@apilink BasicCrawlerOptions.requestList|`requestList`} or {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`}
  * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
+ * the crawler will open the default request queue either when the {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
+ * or if `requests` parameter (representing the initial requests) of the {@apilink BasicCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * If both {@apilink BasicCrawlerOptions.requestList|`requestList`} and {@apilink BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
+ * to the {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
+ * The crawler finishes if there are no more {@apilink Request} objects to crawl.
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
+ * using the functionality provided by the {@apilink AutoscaledPool} class.
+ * All {@apilink AutoscaledPool} configuration options can be passed to the {@apilink BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
  * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
+ * For user convenience, the {@apilink AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
+ * {@apilink AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
+ * underlying {@apilink AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
  *
  * **Example usage:**
  *
@@ -95,7 +95,7 @@ class BasicCrawler {
             value: config
         });
         /**
-         * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+         * A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.
          */
         Object.defineProperty(this, "stats", {
             enumerable: true,
@@ -104,7 +104,7 @@ class BasicCrawler {
             value: void 0
         });
         /**
-         * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
+         * A reference to the underlying {@apilink RequestList} class that manages the crawler's {@apilink Request|requests}.
          * Only available if used by the crawler.
          */
         Object.defineProperty(this, "requestList", {
@@ -115,7 +115,7 @@ class BasicCrawler {
         });
         /**
          * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-         * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
+         * A reference to the underlying {@apilink RequestQueue} class that manages the crawler's {@apilink Request|requests}.
          * Only available if used by the crawler.
          */
         Object.defineProperty(this, "requestQueue", {
@@ -125,7 +125,7 @@ class BasicCrawler {
             value: void 0
         });
         /**
-         * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
+         * A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
          * Only available if used by the crawler.
          */
         Object.defineProperty(this, "sessionPool", {
@@ -135,11 +135,11 @@ class BasicCrawler {
             value: void 0
         });
         /**
-         * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
-         * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
+         * A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
+         * > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
          * We can use it to change the concurrency settings on the fly,
-         * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-         * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+         * to pause the crawler by calling {@apilink AutoscaledPool.pause|`autoscaledPool.pause()`}
+         * or to abort it by calling {@apilink AutoscaledPool.abort|`autoscaledPool.abort()`}.
          */
         Object.defineProperty(this, "autoscaledPool", {
             enumerable: true,
@@ -148,8 +148,8 @@ class BasicCrawler {
             value: void 0
         });
         /**
-         * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
-         * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
+         * Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
+         * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
          */
         Object.defineProperty(this, "router", {
             enumerable: true,
@@ -242,7 +242,7 @@ class BasicCrawler {
             value: void 0
         });
         (0, ow_1.default)(options, 'BasicCrawlerOptions', ow_1.default.object.exactShape(BasicCrawler.optionsShape));
-        const { requestList, requestQueue, maxRequestRetries = 3, maxRequestsPerCrawl, autoscaledPoolOptions = {}, sessionPoolOptions = {}, useSessionPool = true, 
+        const { requestList, requestQueue, maxRequestRetries = 3, maxRequestsPerCrawl, autoscaledPoolOptions = {}, keepAlive, sessionPoolOptions = {}, useSessionPool = true, 
         // AutoscaledPool shorthands
         minConcurrency, maxConcurrency, maxRequestsPerMinute, 
         // internal
@@ -316,7 +316,11 @@ class BasicCrawler {
         }
         let shouldLogMaxPagesExceeded = true;
         const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
-        const { isFinishedFunction } = autoscaledPoolOptions;
+        let { isFinishedFunction } = autoscaledPoolOptions;
+        // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
+        if (keepAlive) {
+            isFinishedFunction = async () => false;
+        }
         const basicCrawlerAutoscaledPoolConfiguration = {
             minConcurrency,
             maxConcurrency,
@@ -361,7 +365,7 @@ class BasicCrawler {
     /**
      * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
      * We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
-     * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before the {@link BasicCrawler.run|`crawler.run()`}.
+     * running {@apilink BasicCrawler.addRequests|`crawler.addRequests()`} before the {@apilink BasicCrawler.run|`crawler.run()`}.
      *
      * @param [requests] The requests to add
      * @param [options] Options for the request queue
@@ -693,14 +697,14 @@ class BasicCrawler {
         if (error instanceof core_1.CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = !request.noRetry && request.retryCount < this.maxRequestRetries && !(error instanceof core_1.NonRetryableError);
+        const shouldRetryRequest = this._canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             request.retryCount++;
-            await this._tagUserHandlerError(() => this.errorHandler?.(crawlingContext, error));
+            await this._tagUserHandlerError(() => this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
             const { url, retryCount, id } = request;
             // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
             // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-            const message = process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error;
+            const message = this._getMessageFromError(error);
             this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, { id, url, retryCount });
             await source.reclaimRequest(request);
         }
@@ -724,13 +728,49 @@ class BasicCrawler {
         }
     }
     async _handleFailedRequestHandler(crawlingContext, error) {
-        if (this.failedRequestHandler) {
-            await this._tagUserHandlerError(() => this.failedRequestHandler?.(crawlingContext, error));
-            return;
-        }
+        // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = error instanceof timeout_1.TimeoutError && !process.env.CRAWLEE_VERBOSE_LOG ? error.message : error.stack;
+        const message = this._getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
+        if (this.failedRequestHandler) {
+            await this._tagUserHandlerError(() => this.failedRequestHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
+        }
+    }
+    /**
+     * Resolves the most verbose error message from a thrown error
+     * @param error The error received
+     * @returns The message to be logged
+     */
+    _getMessageFromError(error, forceStack = false) {
+        // For timeout errors we want to show the stack just in case the env variable is set
+        if (error instanceof timeout_1.TimeoutError) {
+            return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error;
+        }
+        return (process.env.CRAWLEE_VERBOSE_LOG || forceStack)
+            ? error.stack ?? (error.message || error)
+            : error.message || error;
+    }
+    _canRequestBeRetried(request, error) {
+        // User requested retry (we ignore retry count here as its explicitly told by the user to retry)
+        if (error instanceof core_1.RetryRequestError) {
+            return true;
+        }
+        // Request should never be retried, or the error encountered makes it not able to be retried
+        if (request.noRetry || (error instanceof core_1.NonRetryableError)) {
+            return false;
+        }
+        // Ensure there are more retries available for the request
+        return request.retryCount < this.maxRequestRetries;
+    }
+    _augmentContextWithDeprecatedError(context, error) {
+        Object.defineProperty(context, 'error', {
+            get: () => {
+                // eslint-disable-next-line max-len
+                this.log.deprecated("The 'error' property of the crawling context is deprecated, and it is now passed as the second parameter in 'errorHandler' and 'failedRequestHandler'. Please update your code, as this property will be removed in a future version.");
+                return error;
+            },
+        });
+        return context;
     }
     /**
      * Updates handledRequestsCount from possibly stored counts,
@@ -767,6 +807,7 @@ class BasicCrawler {
         if (this._closeEvents) {
             await this.events.close();
         }
+        await this.autoscaledPool?.abort();
     }
     _handlePropertyNameChange({ newProperty, newName, oldProperty, oldName, propertyKey, allowUndefined = false, }) {
         if (newProperty && oldProperty) {
@@ -838,14 +879,15 @@ Object.defineProperty(BasicCrawler, "optionsShape", {
         minConcurrency: ow_1.default.optional.number,
         maxConcurrency: ow_1.default.optional.number,
         maxRequestsPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
+        keepAlive: ow_1.default.optional.boolean,
         // internal
         log: ow_1.default.optional.object,
     }
 });
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
- * Defaults to the {@link BasicCrawlingContext}.
+ * Creates new {@apilink Router} instance that works based on request labels.
+ * This instance can then serve as a {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@apilink BasicCrawler}.
+ * Defaults to the {@apilink BasicCrawlingContext}.
  *
  * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
  *