apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/crawlers/browser_crawler.js

@@ -1,540 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const timeout_1 = require("@apify/timeout");
-const browser_pool_1 = require("browser-pool"); // eslint-disable-line import/no-duplicates,no-unused-vars
-const constants_1 = require("../constants");
-const events_1 = (0, tslib_1.__importDefault)(require("../session_pool/events")); // eslint-disable-line import/no-duplicates
-const validators_1 = require("../validators");
-const crawler_utils_1 = require("./crawler_utils");
-/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
-const basic_crawler_1 = require("./basic_crawler");
-/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
-/**
- * @typedef BrowserCrawlingContext
- * @property {BrowserController} browserController
- */
-/**
- * @callback Hook
- * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
- * @param {Object<string,*>} gotoOptions
- * @returns {Promise<void>}
- */
-/**
- * @callback BrowserHandlePageFunction
- * @param {BrowserCrawlingContext & CrawlingContext} context
- * @returns {Promise<void>}
- */
-/**
- * @callback GotoFunction
- * @param {BrowserCrawlingContext & CrawlingContext} context
- * @param {Object<string,*>} gotoOptions
- * @returns {Promise<*>}
- */
-/**
- * @typedef BrowserCrawlerOptions
- * @property {BrowserHandlePageFunction} handlePageFunction
- *   Function that is called to process each request.
- *   It is passed an object with the following fields:
- *
- * ```
- * {
- *   request: Request,
- *   response: Response,
- *   page: Page,
- *   session: Session,
- *   browserController: BrowserController,
- *   proxyInfo: ProxyInfo,
- *   crawler: BrowserCrawler,
- * }
- * ```
- *
- *   `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
- *   `page` is an instance of the `Puppeteer`
- *   [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) or `Playwright`
- *   [`Page`](https://playwright.dev/docs/api/class-page)
- *   `browserPool` is an instance of the
- *   [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
- *   `browserController` is an instance of the
- *   [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
- *   `response` is an instance of the `Puppeteer`
- *   [`Response`](https://pptr.dev/#?product=Puppeteer&show=api-class-response) or `Playwright`
- *   [`Response`](https://playwright.dev/docs/api/class-response),
- *   which is the main resource response as returned by `page.goto(request.url)`.
- *   The function must return a promise, which is then awaited by the crawler.
- *
- *   If the function throws an exception, the crawler will try to re-crawl the
- *   request later, up to `option.maxRequestRetries` times.
- *   If all the retries fail, the crawler calls the function
- *   provided to the `handleFailedRequestFunction` parameter.
- *   To make this work, you should **always**
- *   let your function throw exceptions rather than catch them.
- *   The exceptions are logged to the request using the
- *   {@link Request#pushErrorMessage} function.
- * @property {number} [handlePageTimeoutSecs=60]
- *   Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
- * @property {GotoFunction} [gotoFunction]
- *   Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
- * @property {HandleFailedRequest} [handleFailedRequestFunction]
- *   A function to handle requests that failed more than `option.maxRequestRetries` times.
- *
- *   The function receives the following object as an argument:
- * ```
- * {
- *   request: Request,
- *   response: Response,
- *   page: Page,
- *   browserPool: BrowserPool,
- *   autoscaledPool: AutoscaledPool,
- *   session: Session,
- *   browserController: BrowserController,
- *   proxyInfo: ProxyInfo,
- * }
- * ```
- *   Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
- *   represents the last error thrown during processing of the request.
- * @property {BrowserPoolOptions} [browserPoolOptions]
- *   Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
- *   You can tweak those to fine-tune browser management.
- * @property {boolean} [persistCookiesPerSession=true]
- *   Automatically saves cookies to Session. Works only if Session Pool is used.
- * @property {ProxyConfiguration} [proxyConfiguration]
- *   If set, `PuppeteerCrawler` will be configured for all connections to use
- *   [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
- *   For more information, see the [documentation](https://docs.apify.com/proxy).
- * @property {Array<Hook>} [preNavigationHooks]
- *   Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- *   or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
- *   which are passed to the `page.goto()` function the crawler calls to navigate.
- *   Example:
- * ```
- * preNavigationHooks: [
- *     async (crawlingContext, gotoOptions) => {
- *         const { page } = crawlingContext;
- *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
- *     },
- * ]
- * ```
- * @property {Array<Hook>} [postNavigationHooks]
- *   Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
- *   The function accepts `crawlingContext` as the only parameter.
- *   Example:
- * ```
- * postNavigationHooks: [
- *     async (crawlingContext) => {
- *         const { page } = crawlingContext;
- *         if (hasCaptcha(page)) {
- *             await solveCaptcha (page);
- *         }
- *     },
- * ]
- * ```
- * @property {RequestList} [requestList]
- *   Static list of URLs to be processed.
- *   Either `requestList` or `requestQueue` option must be provided (or both).
- * @property {RequestQueue} [requestQueue]
- *   Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
- *   Either `requestList` or `requestQueue` option must be provided (or both).
- * @property {number} [handleRequestTimeoutSecs=60]
- *   Timeout in which the function passed as `handleRequestFunction` needs to finish, in seconds.
- * @property {HandleFailedRequest} [handleFailedRequestFunction]
- *   A function to handle requests that failed more than `option.maxRequestRetries` times.
- *
- *   The function receives the following object as an argument:
- * ```
- * {
- *   request: Request,
- *   error: Error,
- *   session: Session,
- *   crawler: BrowserCrawler,
- * }
- * ```
- *   where the {@link Request} instance corresponds to the failed request, and the `Error` instance
- *   represents the last error thrown during processing of the request.
- *
- *   See
- *   [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
- *   for the default implementation of this function.
- * @property {number} [maxRequestRetries=3]
- *   Indicates how many times the request is retried if {@link BasicCrawlerOptions.handleRequestFunction} fails.
- * @property {number} [maxRequestsPerCrawl]
- *   Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
- *   Always set this value in order to prevent infinite loops in misconfigured crawlers.
- *   Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
- * @property {AutoscaledPoolOptions} [autoscaledPoolOptions]
- *   Custom options passed to the underlying {@link AutoscaledPool} constructor.
- *   Note that the `runTaskFunction` and `isTaskReadyFunction` options
- *   are provided by `BasicCrawler` and cannot be overridden.
- *   However, you can provide a custom implementation of `isFinishedFunction`.
- * @property {number} [minConcurrency=1]
- *   Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
- *
- *   *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
- *   If you're not sure, just keep the default value and the concurrency will scale up automatically.
- * @property {number} [maxConcurrency=1000]
- *   Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
- * @property {boolean} [useSessionPool=true]
- *   Browser crawler will initialize the  {@link SessionPool} with the corresponding `sessionPoolOptions`.
- *   The session instance will be than available in the `handleRequestFunction`.
- * @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {@link SessionPool} to use.
- * @ignore
- */
-/**
- * Provides a simple framework for parallel crawling of web pages
- * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
- * and [Playwright](https://github.com/microsoft/playwright).
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data,
- * it is useful for crawling of websites that require to execute JavaScript.
- * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
- * which downloads the pages using raw HTTP requests and is about 10x faster. @TODO: more?
- *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BrowserCrawlerOptions.requestList}
- * or {@link BrowserCrawlerOptions.requestQueue} constructor options, respectively.
- *
- * If both {@link BrowserCrawlerOptions.requestList} and {@link BrowserCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
- *
- * The crawler finishes when there are no more {@link Request} objects to crawl.
- *
- * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
- * and then calls the function provided by user as the {@link BrowserCrawlerOptions.handlePageFunction} option.
- *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `BrowserCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `BrowserCrawler` constructor.
- *
- * Note that the pool of browser instances is internally managed by the {@link BrowserPool} class.
- *
- * await crawler.run();
- * ```
- * @property {Statistics} stats
- *  Contains statistics about the current run.
- * @property {RequestList} [requestList]
- *  A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
- *  Only available if used by the crawler.
- * @property {RequestQueue} [requestQueue]
- *  A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
- *  Only available if used by the crawler.
- * @property {SessionPool} [sessionPool]
- *  A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
- *  Only available if used by the crawler.
- * @property {ProxyConfiguration} [proxyConfiguration]
- *  A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
- *  Only available if used by the crawler.
- * @property {BrowserPool} browserPool
- *  A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
- *  For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
- * @property {AutoscaledPool} autoscaledPool
- *  A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
- *  Note that this property is only initialized after calling the {@link CheerioCrawler#run} function.
- *  You can use it to change the concurrency settings on the fly,
- *  to pause the crawler by calling {@link AutoscaledPool#pause}
- *  or to abort it by calling {@link AutoscaledPool#abort}.
- * @ignore
- */
-class BrowserCrawler extends basic_crawler_1.BasicCrawler {
-    /**
-     * @param {BrowserCrawlerOptions} options
-     * All `BrowserCrawler` parameters are passed via an options object.
-     */
-    constructor(options) {
-        (0, ow_1.default)(options, 'BrowserCrawlerOptions', ow_1.default.object.exactShape(BrowserCrawler.optionsShape));
-        const { handlePageFunction, handlePageTimeoutSecs = 60, navigationTimeoutSecs = 60, gotoFunction, // deprecated
-        gotoTimeoutSecs, // deprecated
-        persistCookiesPerSession, proxyConfiguration, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], ...basicCrawlerOptions } = options;
-        super({
-            ...basicCrawlerOptions,
-            handleRequestFunction: (...args) => this._handleRequestFunction(...args),
-            handleRequestTimeoutSecs: navigationTimeoutSecs + handlePageTimeoutSecs + constants_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        });
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
-        if (gotoTimeoutSecs) {
-            this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
-        }
-        this.handlePageFunction = handlePageFunction;
-        this.handlePageTimeoutSecs = handlePageTimeoutSecs;
-        this.handlePageTimeoutMillis = this.handlePageTimeoutSecs * 1000;
-        this.navigationTimeoutMillis = (gotoTimeoutSecs || navigationTimeoutSecs) * 1000;
-        this.gotoFunction = gotoFunction;
-        this.defaultGotoOptions = {
-            timeout: this.navigationTimeoutMillis,
-        };
-        this.proxyConfiguration = proxyConfiguration;
-        /** @type {Array<Hook>} */
-        this.preNavigationHooks = preNavigationHooks;
-        /** @type {Array<Hook>} */
-        this.postNavigationHooks = postNavigationHooks;
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
-        const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
-        this.browserPool = new browser_pool_1.BrowserPool({
-            ...rest,
-            preLaunchHooks: [
-                this._extendLaunchContext.bind(this),
-                ...preLaunchHooks,
-            ],
-            postLaunchHooks: [
-                this._maybeAddSessionRetiredListener.bind(this),
-                ...postLaunchHooks,
-            ],
-        });
-    }
-    /**
-     * Wrapper around handlePageFunction that opens and closes pages etc.
-     *
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _handleRequestFunction(crawlingContext) {
-        const newPageOptions = {
-            id: crawlingContext.id,
-        };
-        const useIncognitoPages = this.launchContext && this.launchContext.useIncognitoPages;
-        if (this.proxyConfiguration && useIncognitoPages) {
-            const { session } = crawlingContext;
-            const proxyInfo = this.proxyConfiguration.newProxyInfo(session && session.id);
-            crawlingContext.session = session;
-            crawlingContext.proxyInfo = proxyInfo;
-            newPageOptions.proxyUrl = proxyInfo.url;
-            // Disable SSL verification for MITM proxies
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                newPageOptions.pageOptions = {
-                    ignoreHTTPSErrors: true,
-                };
-            }
-        }
-        const page = await this.browserPool.newPage(newPageOptions);
-        (0, timeout_1.tryCancel)();
-        this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
-        // DO NOT MOVE THIS LINE ABOVE!
-        // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
-        // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
-        // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
-        const { request, session } = crawlingContext;
-        if (this.useSessionPool) {
-            const sessionCookies = session.getPuppeteerCookies(request.url);
-            if (sessionCookies.length) {
-                await crawlingContext.browserController.setCookies(page, sessionCookies);
-                (0, timeout_1.tryCancel)();
-            }
-        }
-        try {
-            await this._handleNavigation(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            await this._responseHandler(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            // save cookies
-            // @TODO: Should we save the cookies also after/only the handle page?
-            if (this.persistCookiesPerSession) {
-                const cookies = await crawlingContext.browserController.getCookies(page);
-                (0, timeout_1.tryCancel)();
-                session.setPuppeteerCookies(cookies, request.loadedUrl);
-            }
-            await (0, timeout_1.addTimeoutToPromise)(() => this.handlePageFunction(crawlingContext), this.handlePageTimeoutMillis, `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
-            (0, timeout_1.tryCancel)();
-            if (session)
-                session.markGood();
-        }
-        finally {
-            page.close().catch((error) => this.log.debug('Error while closing page', { error }));
-        }
-    }
-    /**
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @param {*} page
-     * @param {boolean} useIncognitoPages
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages) {
-        crawlingContext.page = page;
-        // This switch is because the crawlingContexts are created on per request basis.
-        // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
-        // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
-        // which configures proxy and session for every new request
-        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
-        crawlingContext.browserController = browserControllerInstance;
-        if (!useIncognitoPages) {
-            crawlingContext.session = browserControllerInstance.launchContext.session;
-        }
-        if (!crawlingContext.proxyInfo) {
-            crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
-        }
-    }
-    /**
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _handleNavigation(crawlingContext) {
-        /** @type {*} */
-        const gotoOptions = { ...this.defaultGotoOptions };
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
-        (0, timeout_1.tryCancel)();
-        try {
-            crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
-        }
-        catch (error) {
-            this._handleNavigationTimeout(crawlingContext, error);
-            throw error;
-        }
-        (0, timeout_1.tryCancel)();
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
-    }
-    /**
-     * Marks session bad in case of navigation timeout.
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @param {Error} error
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _handleNavigationTimeout(crawlingContext, error) {
-        const { session } = crawlingContext;
-        if (error && error.constructor.name === 'TimeoutError') {
-            (0, crawler_utils_1.handleRequestTimeout)(session, error.message);
-        }
-    }
-    /**
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @param {Object<string,*>} gotoOptions
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _navigationHandler(crawlingContext, gotoOptions) {
-        if (!this.gotoFunction) {
-            // @TODO: although it is optional in the validation,
-            //  because when you make automation library specific you can override this handler.
-            throw new Error('BrowserCrawler: You must specify a gotoFunction!');
-        }
-        return this.gotoFunction(crawlingContext, gotoOptions);
-    }
-    /**
-     * Should be overriden in case of different automation library that does not support this response API.
-     * // @TODO: This can be also done as a postNavigation hook except the loadedUrl marking.
-     * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
-     * @return {Promise<void>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _responseHandler(crawlingContext) {
-        const { response, session, request, page } = crawlingContext;
-        if (this.sessionPool && response) {
-            if (typeof response === 'object' && typeof response.status === 'function') {
-                (0, crawler_utils_1.throwOnBlockedRequest)(session, response.status());
-            }
-            else {
-                this.log.debug('Got a malformed Browser response.', { request, response });
-            }
-        }
-        request.loadedUrl = await page.url();
-    }
-    /**
-     * @param {string} pageId
-     * @param {*} launchContext
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _extendLaunchContext(pageId, launchContext) {
-        const launchContextExtends = {};
-        if (this.sessionPool) {
-            launchContextExtends.session = await this.sessionPool.getSession();
-        }
-        if (this.proxyConfiguration) {
-            const proxyInfo = this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
-            launchContext.proxyUrl = proxyInfo.url;
-            launchContextExtends.proxyInfo = proxyInfo;
-            // Disable SSL verification for MITM proxies
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                launchContext.launchOptions.ignoreHTTPSErrors = true;
-            }
-        }
-        launchContext.extend(launchContextExtends);
-    }
-    /**
-     *
-     * @param {string} pageId
-     * @param {BrowserController} browserController
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _maybeAddSessionRetiredListener(pageId, browserController) {
-        if (this.sessionPool) {
-            const listener = (session) => {
-                const { launchContext } = browserController;
-                if (session.id === launchContext.session.id) {
-                    this.browserPool.retireBrowserController(browserController);
-                }
-            };
-            this.sessionPool.on(events_1.default.SESSION_RETIRED, listener);
-            browserController.on('browserClosed', () => this.sessionPool.removeListener(events_1.default.SESSION_RETIRED, listener));
-        }
-    }
-    /**
-     * Function for cleaning up after all request are processed.
-     * @ignore
-     */
-    async teardown() {
-        await this.browserPool.destroy();
-        await super.teardown();
-    }
-}
-exports.default = BrowserCrawler;
-/**
- * @internal
- * @type any
- */
-Object.defineProperty(BrowserCrawler, "optionsShape", {
-    enumerable: true,
-    configurable: true,
-    writable: true,
-    value: {
-        ...basic_crawler_1.BasicCrawler.optionsShape,
-        // TODO temporary until the API is unified in V2
-        handleRequestFunction: ow_1.default.undefined,
-        handlePageFunction: ow_1.default.function,
-        gotoFunction: ow_1.default.optional.function,
-        gotoTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
-        navigationTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
-        handlePageTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
-        preNavigationHooks: ow_1.default.optional.array,
-        postNavigationHooks: ow_1.default.optional.array,
-        browserPoolOptions: ow_1.default.object,
-        sessionPoolOptions: ow_1.default.optional.object,
-        persistCookiesPerSession: ow_1.default.optional.boolean,
-        useSessionPool: ow_1.default.optional.boolean,
-        proxyConfiguration: ow_1.default.optional.object.validate(validators_1.validators.proxyConfiguration),
-    }
-});
-//# sourceMappingURL=browser_crawler.js.map

package/build/crawlers/browser_crawler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,4CAAgE;AAChE,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,8CAA2C;AAC3C,mDAGyB;AAEzB,kFAAkF;AAClF,mDAAgE;AAUhE,iFAAiF;AAEjF;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAqB,cAAe,SAAQ,4BAAY;IA0BpD;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,kBAAkB,EAClB,qBAAqB,GAAG,EAAE,EAC1B,qBAAqB,GAAG,EAAE,EAC1B,YAAY,EAAE,aAAa;QAC3B,eAAe,EAAE,aAAa;QAC9B,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,IAAI,CAAC;YACxE,wBAAwB,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,6CAAiC;SAC9G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,8EAA8E,CAAC,CAAC;SACvG;QAED,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QACjE,IAAI,CAAC,uBAAuB,GAAG,CAAC,eAAe,IAAI,qBAAqB,CAAC,GAAG,IAAI,CAAC;QAEjF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACxC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,0BAA0B;QAC1B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,0BAA0B;QAC1B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAAC;YAC/B,GAAG,IAAI;YACP,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,cAAc,GAAG;YACnB,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC;QACrF,IAAI,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9E,eAAe,CAAC,OAAO,GAAG,OAAO,CAAC;YAClC,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YAExC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;iBAC1B,CAAC;aACL;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,+BAA+B;QAC/B,iEAAiE;QACjE,6EAA6E;QAC7E,+GAA+G;QAC/G,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,cAAc,CAAC,MAAM,EAAE;gBACvB,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;aACf;SACJ;QAED,IAAI;YACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,IAAA,mBAAS,GAAE,CAAC;YAEZ,eAAe;YACf,qEAAqE;YACrE,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;gBACZ,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aAC3D;YAED,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EAC9C,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SACxF;IACL,CAAC;IAED;;;;;;;OAOG;IACH,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB;QACxE,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC;QACpF,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,iBAAiB,EAAE;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAO,CAAC;SAC7E;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC5B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAS,CAAC;SACjF;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,gBAAgB;QAChB,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1F;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,MAAM,KAAK,CAAC;SACf;QAED,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;OAOG;IACH,wBAAwB,CAAC,eAAe,EAAE,KAAK;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,oCAAoB,EAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;SAChD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,oDAAoD;YACpD,oFAAoF;YACpF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;SACvE;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,eAAe;QAClC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE;YAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAA,qCAAqB,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACrD;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAM,EAAE,aAAa;QAC5C,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxH,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,aAAa,CAAC,aAAa,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxD;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;OAOG;IACH,+BAA+B,CAAC,MAAM,EAAE,iBAAiB;QACrD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAO,EAAE,EAAE;gBACzB,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAK,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;oBACzC,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;iBAC/D;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YACtD,iBAAiB,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AA1VL,iCA2VC;AA1VG;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAElC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC"}