@crawlee/browser 3.0.0-beta.9 → 3.0.0

package/internals/browser-crawler.d.ts

@@ -1,171 +1,141 @@
-import { EnqueueLinksOptions, CrawlerHandleFailedRequestInput, CrawlingContext, ProxyConfiguration, RequestQueue } from '@crawlee/core';
-import { BasicCrawler, BasicCrawlerOptions } from '@crawlee/basic';
-import { Awaitable, Dictionary } from '@crawlee/utils';
-import { BrowserController, BrowserPlugin, BrowserPool, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
-import { BatchAddRequestsResult } from '@crawlee/types';
-import { BrowserLaunchContext } from './browser-launcher';
-export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController> extends CrawlingContext {
+import type { EnqueueLinksOptions, CrawlingContext, ProxyConfiguration, RequestQueue } from '@crawlee/core';
+import { Configuration } from '@crawlee/core';
+import type { BasicCrawlerOptions, Awaitable, Dictionary } from '@crawlee/basic';
+import { BasicCrawler } from '@crawlee/basic';
+import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
+import { BrowserPool } from '@crawlee/browser-pool';
+import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';
+import type { BatchAddRequestsResult } from '@crawlee/types';
+import type { BrowserLaunchContext } from './browser-launcher';
+export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
     browserController: ProvidedController;
     page: Page;
     response?: Response;
     crawler: BrowserCrawler;
     enqueueLinks: (options?: BrowserCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+    sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
 }
-export interface BrowserCrawlerHandleFailedRequestInput extends CrawlerHandleFailedRequestInput {
-    crawler: BrowserCrawler;
-}
-export declare type BrowserCrawlerHandleFailedRequest = (inputs: BrowserCrawlerHandleFailedRequestInput) => Awaitable<void>;
 export declare type BrowserCrawlerHandleRequest<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = (inputs: Context) => Awaitable<void>;
+export declare type BrowserCrawlerHandleFailedRequest<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
 export declare type BrowserCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue' | 'urls'>;
 export declare type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
-export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction'> {
+export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction' | 'errorHandler'> {
     launchContext?: BrowserLaunchContext<any, any>;
     /**
      * Function that is called to process each request.
-     * It is passed an object with the following fields:
      *
-     * ```
-     * {
-     *   request: Request,
-     *   response: Response,
-     *   page: Page,
-     *   session: Session,
-     *   browserController: BrowserController,
-     *   proxyInfo: ProxyInfo,
-     *   crawler: BrowserCrawler,
-     * }
-     * ```
+     * The function receives the {@link BrowserCrawlingContext}
+     * (actual context will be enhanced with the crawler specific properties) as an argument, where:
+     * - {@link BrowserCrawlingContext.request|`request`} is an instance of the {@link Request} object
+     * with details about the URL to open, HTTP method etc;
+     * - {@link BrowserCrawlingContext.page|`page`} is an instance of the
+     * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
+     * Playwright [Page](https://playwright.dev/docs/api/class-page);
+     * - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
+     * - {@link BrowserCrawlingContext.response|`response`} is an instance of the
+     * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
+     * Playwright [Response](https://playwright.dev/docs/api/class-response),
+     * which is the main resource response as returned by the respective `page.goto()` function.
      *
-     * `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
-     * `page` is an instance of the `Puppeteer`
-     * [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright`
-     * [`Page`](https://playwright.dev/docs/api/class-page)
-     * `browserPool` is an instance of the
-     * [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
-     * `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
-     * `response` is an instance of the `Puppeteer`
-     * [`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
-     * [`Response`](https://playwright.dev/docs/api/class-response),
-     * which is the main resource response as returned by `page.goto(request.url)`.
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
+     * request later, up to the {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
+     * provided to the {@link BrowserCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
+     * To make this work, we should **always**
+     * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
+     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler: BrowserCrawlerHandleRequest<Context>;
+    requestHandler?: BrowserCrawlerHandleRequest<Context>;
     /**
      * Function that is called to process each request.
-     * It is passed an object with the following fields:
      *
-     * ```
-     * {
-     *   request: Request,
-     *   response: Response,
-     *   page: Page,
-     *   session: Session,
-     *   browserController: BrowserController,
-     *   proxyInfo: ProxyInfo,
-     *   crawler: BrowserCrawler,
-     * }
-     * ```
+     * The function receives the {@link BrowserCrawlingContext}
+     * (actual context will be enhanced with the crawler specific properties) as an argument, where:
+     * - {@link BrowserCrawlingContext.request|`request`} is an instance of the {@link Request} object
+     * with details about the URL to open, HTTP method etc;
+     * - {@link BrowserCrawlingContext.page|`page`} is an instance of the
+     * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
+     * Playwright [Page](https://playwright.dev/docs/api/class-page);
+     * - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
+     * - {@link BrowserCrawlingContext.response|`response`} is an instance of the
+     * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
+     * Playwright [Response](https://playwright.dev/docs/api/class-response),
+     * which is the main resource response as returned by the respective `page.goto()` function.
      *
-     * `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
-     * `page` is an instance of the `Puppeteer`
-     * [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright`
-     * [`Page`](https://playwright.dev/docs/api/class-page)
-     * `browserPool` is an instance of the
-     * [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
-     * `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
-     * `response` is an instance of the `Puppeteer`
-     * [`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
-     * [`Response`](https://playwright.dev/docs/api/class-response),
-     * which is the main resource response as returned by `page.goto(request.url)`.
      * The function must return a promise, which is then awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
+     * request later, up to the {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
+     * provided to the {@link BrowserCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
+     * To make this work, we should **always**
+     * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
+     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      *
      * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
      */
     handlePageFunction?: BrowserCrawlerHandleRequest<Context>;
     /**
-     * A function to handle requests that failed more than `option.maxRequestRetries` times.
+     * User-provided function that allows modifying the request object before it gets retried by the crawler.
+     * It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     response: Response,
-     *     page: Page,
-     *     browserPool: BrowserPool,
-     *     autoscaledPool: AutoscaledPool,
-     *     session: Session,
-     *     browserController: BrowserController,
-     *     proxyInfo: ProxyInfo,
-     * }
-     * ```
-     * Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * The function receives the {@link BrowserCrawlingContext}
+     * (actual context will be enhanced with the crawler specific properties) as the first argument,
+     * where the {@link BrowserCrawlingContext.request|`request`} corresponds to the request to be retried.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
+     */
+    errorHandler?: BrowserCrawlerHandleFailedRequest<Context>;
+    /**
+     * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
+     * The function receives the {@link BrowserCrawlingContext}
+     * (actual context will be enhanced with the crawler specific properties) as the first argument,
+     * where the {@link BrowserCrawlingContext.request|`request`} corresponds to the failed request.
+     * Second argument is the `Error` instance that
+     * represents the last error thrown during processing of the request.
      */
-    failedRequestHandler?: BrowserCrawlerHandleFailedRequest;
+    failedRequestHandler?: BrowserCrawlerHandleFailedRequest<Context>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     request: Request,
-     *     response: Response,
-     *     page: Page,
-     *     browserPool: BrowserPool,
-     *     autoscaledPool: AutoscaledPool,
-     *     session: Session,
-     *     browserController: BrowserController,
-     *     proxyInfo: ProxyInfo,
-     * }
-     * ```
-     * Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * The function receives the {@link BrowserCrawlingContext}
+     * (actual context will be enhanced with the crawler specific properties) as the first argument,
+     * where the {@link BrowserCrawlingContext.request|`request`} corresponds to the failed request.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      *
      * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
      */
-    handleFailedRequestFunction?: BrowserCrawlerHandleFailedRequest;
+    handleFailedRequestFunction?: BrowserCrawlerHandleFailedRequest<Context>;
     /**
-     * Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
-     * You can tweak those to fine-tune browser management.
+     * Custom options passed to the underlying {@link BrowserPool} constructor.
+     * We can tweak those to fine-tune browser management.
      */
     browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
     /**
      * If set, the crawler will be configured for all connections to use
-     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
-     * For more information, see the [documentation](https://docs.apify.com/proxy).
+     * the Proxy URLs provided and rotated according to the configuration.
      */
     proxyConfiguration?: ProxyConfiguration;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
      * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
      * which are passed to the `page.goto()` function the crawler calls to navigate.
-     * Example:
-     * ```
+     *
+     * **Example:**
+     *
+     * ```js
      * preNavigationHooks: [
      *     async (crawlingContext, gotoOptions) => {
      *         const { page } = crawlingContext;
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *         gotoOptions.timeout = 60_000;
+     *         gotoOptions.waitUntil = 'domcontentloaded';
      *     },
      * ]
      * ```
@@ -174,13 +144,15 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
-     * Example:
-     * ```
+     *
+     * **Example:**
+     *
+     * ```js
      * postNavigationHooks: [
      *     async (crawlingContext) => {
      *         const { page } = crawlingContext;
      *         if (hasCaptcha(page)) {
-     *             await solveCaptcha (page);
+     *             await solveCaptcha(page);
      *         }
      *     },
      * ]
@@ -192,7 +164,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      */
     navigationTimeoutSecs?: number;
     /**
-     * If cookies should be persisted between sessions.
+     * Defines whether the cookies should be persisted for sessions.
      * This can only be used when `useSessionPool` is set to `true`.
      */
     persistCookiesPerSession?: boolean;
@@ -204,46 +176,47 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
  * The URLs to crawl are fed either from a static list of URLs
  * or from a dynamic queue of URLs enabling recursive crawling of websites.
  *
- * Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data,
+ * Since `BrowserCrawler` uses headless (or even headful) browsers to download web pages and extract data,
  * it is useful for crawling of websites that require to execute JavaScript.
- * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BrowserCrawlerOptions.requestList}
- * or {@link BrowserCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
+ * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
+ * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList} and {@link BrowserCrawlerOptions.requestQueue} are used,
+ * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
  * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
- * and then calls the function provided by user as the {@link BrowserCrawlerOptions.handlePageFunction} option.
+ * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
  *
  * New pages are only opened when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `BrowserCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `BrowserCrawler` constructor.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
+ * parameter of the `BrowserCrawler` constructor.
+ * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
+ * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
+ * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
+ *
+ * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
  *
- * Note that the pool of browser instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
- * ```js
- * await crawler.run();
- * ```
  * @category Crawlers
  */
 export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions = Dictionary, Context extends BrowserCrawlingContext = BrowserCrawlingContext, GoToOptions extends Record<PropertyKey, any> = Dictionary> extends BasicCrawler<Context> {
+    readonly config: Configuration;
     /**
      * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
      * Only available if used by the crawler.
      */
     proxyConfiguration?: ProxyConfiguration;
     /**
-     * A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
-     * For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
-     * @todo the type is almost unusable with so many generic arguments, what should go there? we need inference
+     * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
      */
     browserPool: BrowserPool<InternalBrowserPoolOptions>;
     launchContext?: BrowserLaunchContext<LaunchOptions, unknown>;
@@ -269,6 +242,7 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
         handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -276,14 +250,15 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    protected constructor(options: BrowserCrawlerOptions<Context>);
+    protected constructor(options?: BrowserCrawlerOptions<Context>, config?: Configuration);
     /**
-     * Wrapper around handlePageFunction that opens and closes pages etc.
+     * Wrapper around requestHandler that opens and closes pages etc.
      */
     protected _runRequestHandler(crawlingContext: Context): Promise<void>;
     protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, useIncognitoPages?: boolean): void;
@@ -292,17 +267,16 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     /**
      * Marks session bad in case of navigation timeout.
      */
-    protected _handleNavigationTimeout(crawlingContext: Context, error: Error): void;
+    protected _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void>;
     protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
     /**
      * Should be overridden in case of different automation library that does not support this response API.
-     * @todo: This can be also done as a postNavigation hook except the loadedUrl marking.
      */
     protected _responseHandler(crawlingContext: Context): Promise<void>;
     protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
     protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
     /**
-     * Function for cleaning up after all request are processed.
+     * Function for cleaning up after all requests are processed.
      * @ignore
      */
     teardown(): Promise<void>;

package/internals/browser-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EACH,mBAAmB,EACnB,+BAA+B,EAC/B,eAAe,EAIf,kBAAkB,EAElB,YAAY,EAIf,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,YAAY,EACZ,mBAAmB,EACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAEH,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EAEV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,WAAW,sBAAsB,CACnC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,GAAG,UAAU,EACrB,kBAAkB,GAAG,iBAAiB,CACxC,SAAQ,eAAe;IACrB,iBAAiB,EAAE,kBAAkB,CAAC;IACtC,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAClG;AAED,MAAM,WAAW,sCAAuC,SAAQ,+BAA+B;IAC3F,OAAO,EAAE,cAAc,CAAC;CAC3B;AAED,oBAAY,iCAAiC,GAAG,CAAC,MAAM,EAAE,sCAAsC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEpH,oBAAY,2BAA2B,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEhJ,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,CAAA;AAElG,oBAAY,WAAW,CACnB,OAAO,GAAG,sBAAsB,EAChC,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,UAAU,IACrE,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5E,MAAM,WAAW,qBAAqB,CAClC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACV,mBAAmB,EAEjB,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,CAClC;IACG,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsCG;IACH,cAAc,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,kBAAkB,CAAC,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAE1D;;;;;;;;;;;;;;;;;;;OAmBG;IACH,oBAAoB,CAAC,EAAE,iCAAiC,CAAC;IAEzD;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,2BAA2B,CAAC,EAAE,iCAAiC,CAAC;IAEhE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAE/H;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,8BAAsB,cAAc,CAChC,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,GAAG,UAAU,EAC1B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,UAAU,CAC3D,SAAQ,YAAY,CAAC,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,CAAC,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE7D,SAAS,CAAC,0BAA0B,EAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAC5E,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAE5C,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;IACH,SAAS,aAAa,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC;IAkF7D;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;IAkEpE,SAAS,CAAC,mCAAmC,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI;cA6B5G,iBAAiB,CAAC,eAAe,EAAE,OAAO;cAwB1C,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAevI;;OAEG;IACH,SAAS,CAAC,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAQhF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAE1I;;;OAGG;cACa,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;cAczD,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBlG,SAAS,CAAC,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI;IAkBjH;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,mCAgB7B"}
+{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACR,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAElB,YAAY,EAEf,MAAM,eAAe,CAAC;AACvB,OAAO,EAMH,aAAa,EAChB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,mBAAmB,EACnB,SAAS,EACT,UAAU,EACb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEH,YAAY,EACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACR,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAEH,WAAW,EACd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAG5E,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,gBAAgB,CAAC;AACrF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE/D,MAAM,WAAW,sBAAsB,CACnC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,GAAG,UAAU,EACrB,kBAAkB,GAAG,iBAAiB,EACtC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC1C,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B,iBAAiB,EAAE,kBAAkB,CAAC;IACtC,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC/F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,oBAAY,2BAA2B,CACnC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5G,oBAAY,iCAAiC,CACzC,OAAO,SAAS,sBAAsB,GAAE,sBAAsB,IAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAExH,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,CAAA;AAElG,oBAAY,WAAW,CACnB,OAAO,GAAG,sBAAsB,EAChC,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,UAAU,IACrE,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5E,MAAM,WAAW,qBAAqB,CAClC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACV,mBAAmB,EAEjB,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,GAE7B,cAAc,CACnB;IACG,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,kBAAkB,CAAC,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAE1D;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,iCAAiC,CAAC,OAAO,CAAC,CAAC;IAE1D;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,iCAAiC,CAAC,OAAO,CAAC,CAAC;IAElE;;;;;;;;;;OAUG;IACH,2BAA2B,CAAC,EAAE,iCAAiC,CAAC,OAAO,CAAC,CAAC;IAEzE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAE/H;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;;;OAiBG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,8BAAsB,cAAc,CAChC,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,GAAG,UAAU,EAC1B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,UAAU,CAC3D,SAAQ,YAAY,CAAC,OAAO,CAAC;aAuC2D,MAAM;IAtC5F;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,CAAC,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE7D,SAAS,CAAC,0BAA0B,EAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAC5E,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAE5C,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;IACH,SAAS,aAAa,OAAO,GAAE,qBAAqB,CAAC,OAAO,CAAM,EAAoB,MAAM,gBAAkC;IAuF9H;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;IAkEpE,SAAS,CAAC,mCAAmC,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI;cA6B5G,iBAAiB,CAAC,eAAe,EAAE,OAAO;cAwB1C,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAevI;;OAEG;cACa,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/F,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAE1I;;OAEG;cACa,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;cAczD,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBlG,SAAS,CAAC,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI;IAkBjH;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,mCAgB7B"}

package/internals/browser-crawler.js

@@ -15,41 +15,43 @@ const tough_cookie_1 = require("tough-cookie");
  * The URLs to crawl are fed either from a static list of URLs
  * or from a dynamic queue of URLs enabling recursive crawling of websites.
  *
- * Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data,
+ * Since `BrowserCrawler` uses headless (or even headful) browsers to download web pages and extract data,
  * it is useful for crawling of websites that require to execute JavaScript.
- * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BrowserCrawlerOptions.requestList}
- * or {@link BrowserCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
+ * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
+ * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList} and {@link BrowserCrawlerOptions.requestQueue} are used,
+ * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
  * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
- * and then calls the function provided by user as the {@link BrowserCrawlerOptions.handlePageFunction} option.
+ * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
  *
  * New pages are only opened when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `BrowserCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `BrowserCrawler` constructor.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
+ * parameter of the `BrowserCrawler` constructor.
+ * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
+ * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
+ * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
+ *
+ * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
  *
- * Note that the pool of browser instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
- * ```js
- * await crawler.run();
- * ```
  * @category Crawlers
  */
 class BrowserCrawler extends basic_1.BasicCrawler {
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    constructor(options) {
+    constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
         (0, ow_1.default)(options, 'BrowserCrawlerOptions', ow_1.default.object.exactShape(BrowserCrawler.optionsShape));
         const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], 
         // Ignored
@@ -58,6 +60,12 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             ...basicCrawlerOptions,
             requestHandler: (...args) => this._runRequestHandler(...args),
             requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+        }, config);
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
         });
         /**
          * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
@@ -70,9 +78,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             value: void 0
         });
         /**
-         * A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
-         * For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
-         * @todo the type is almost unusable with so many generic arguments, what should go there? we need inference
+         * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
          */
         Object.defineProperty(this, "browserPool", {
             enumerable: true,
@@ -122,7 +128,11 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             propertyKey: 'userProvidedRequestHandler',
             newProperty: userProvidedRequestHandler,
             oldProperty: handlePageFunction,
+            allowUndefined: true, // fallback to the default router
         });
+        if (!this.userProvidedRequestHandler) {
+            this.userProvidedRequestHandler = this.router;
+        }
         this._handlePropertyNameChange({
             newName: 'failedRequestHandler',
             oldName: 'handleFailedRequestFunction',
@@ -147,8 +157,9 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             this.persistCookiesPerSession = false;
         }
         if (launchContext?.userAgent) {
+            if (browserPoolOptions.useFingerprints)
+                this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
             browserPoolOptions.useFingerprints = false;
-            this.log.info('Disabling automatic fingerprint injection because custom user agent has been provided.');
         }
         const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
         this.browserPool = new browser_pool_1.BrowserPool({
@@ -164,7 +175,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         });
     }
     /**
-     * Wrapper around handlePageFunction that opens and closes pages etc.
+     * Wrapper around requestHandler that opens and closes pages etc.
      */
     async _runRequestHandler(crawlingContext) {
         const newPageOptions = {
@@ -205,7 +216,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
                 if (this.persistCookiesPerSession) {
                     const cookies = await crawlingContext.browserController.getCookies(page);
                     (0, timeout_1.tryCancel)();
-                    session?.setPuppeteerCookies(cookies, request.loadedUrl);
+                    session?.setCookies(cookies, request.loadedUrl);
                 }
             }
             await (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
@@ -214,7 +225,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
                 session.markGood();
         }
         finally {
-            page.close().catch((error) => this.log.debug('Error while closing page', { error }));
+            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
         }
     }
     _enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages) {
@@ -252,14 +263,14 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions) ?? undefined;
         }
         catch (error) {
-            this._handleNavigationTimeout(crawlingContext, error);
+            await this._handleNavigationTimeout(crawlingContext, error);
             throw error;
         }
         (0, timeout_1.tryCancel)();
         await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
     }
     async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.getPuppeteerCookies(request.url) ?? [];
+        const sessionCookie = session?.getCookies(request.url) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => tough_cookie_1.Cookie.parse(c)?.toJSON());
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => tough_cookie_1.Cookie.parse(c)?.toJSON());
         await browserController.setCookies(page, [
@@ -271,15 +282,15 @@ class BrowserCrawler extends basic_1.BasicCrawler {
     /**
      * Marks session bad in case of navigation timeout.
      */
-    _handleNavigationTimeout(crawlingContext, error) {
+    async _handleNavigationTimeout(crawlingContext, error) {
         const { session } = crawlingContext;
         if (error && error.constructor.name === 'TimeoutError') {
             (0, core_1.handleRequestTimeout)({ session, errorMessage: error.message });
         }
+        await crawlingContext.page.close();
     }
     /**
      * Should be overridden in case of different automation library that does not support this response API.
-     * @todo: This can be also done as a postNavigation hook except the loadedUrl marking.
      */
     async _responseHandler(crawlingContext) {
         const { response, session, request, page } = crawlingContext;
@@ -328,7 +339,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         }
     }
     /**
-     * Function for cleaning up after all request are processed.
+     * Function for cleaning up after all requests are processed.
      * @ignore
      */
     async teardown() {