@crawlee/browser 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/internals/browser-crawler.js

@@ -0,0 +1,405 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.browserCrawlerEnqueueLinks = exports.BrowserCrawler = void 0;
+const tslib_1 = require("tslib");
+const timeout_1 = require("@apify/timeout");
+const core_1 = require("@crawlee/core");
+const basic_1 = require("@crawlee/basic");
+const browser_pool_1 = require("@crawlee/browser-pool");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+/**
+ * Provides a simple framework for parallel crawling of web pages
+ * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
+ * and [Playwright](https://github.com/microsoft/playwright).
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `BrowserCrawler` uses headless or even headfull browsers to download web pages and extract data,
+ * it is useful for crawling of websites that require to execute JavaScript.
+ * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * which downloads the pages using raw HTTP requests and is about 10x faster.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BrowserCrawlerOptions.requestList}
+ * or {@link BrowserCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link BrowserCrawlerOptions.requestList} and {@link BrowserCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the {@link BrowserCrawlerOptions.handlePageFunction} option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `BrowserCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `BrowserCrawler` constructor.
+ *
+ * Note that the pool of browser instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
+ * ```js
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+class BrowserCrawler extends basic_1.BasicCrawler {
+    /**
+     * All `BrowserCrawler` parameters are passed via an options object.
+     */
+    constructor(options) {
+        (0, ow_1.default)(options, 'BrowserCrawlerOptions', ow_1.default.object.exactShape(BrowserCrawler.optionsShape));
+        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, gotoFunction, // deprecated
+        gotoTimeoutSecs, // deprecated
+        persistCookiesPerSession, proxyConfiguration, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], 
+        // Ignored
+        handleRequestFunction, requestHandler: userProvidedRequestHandler, handlePageFunction, failedRequestHandler, handleFailedRequestFunction, ...basicCrawlerOptions } = options;
+        super({
+            ...basicCrawlerOptions,
+            requestHandler: (...args) => this._runRequestHandler(...args),
+            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+        });
+        /**
+         * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "proxyConfiguration", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A reference to the underlying `BrowserPool` class that manages the crawler's browsers.
+         * For more information about it, see the [`browser-pool` module](https://github.com/apify/browser-pool).
+         * @todo the type is almost unusable with so many generic arguments, what should go there? we need inference
+         */
+        Object.defineProperty(this, "browserPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "launchContext", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userProvidedRequestHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "navigationTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "gotoFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "defaultGotoOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "postNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "persistCookiesPerSession", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this._handlePropertyNameChange({
+            newName: 'requestHandler',
+            oldName: 'handlePageFunction',
+            propertyKey: 'userProvidedRequestHandler',
+            newProperty: userProvidedRequestHandler,
+            oldProperty: handlePageFunction,
+        });
+        this._handlePropertyNameChange({
+            newName: 'failedRequestHandler',
+            oldName: 'handleFailedRequestFunction',
+            propertyKey: 'failedRequestHandler',
+            newProperty: failedRequestHandler,
+            oldProperty: handleFailedRequestFunction,
+            allowUndefined: true,
+        });
+        // Cookies should be persisted per session only if session pool is used
+        if (!this.useSessionPool && persistCookiesPerSession) {
+            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
+        }
+        if (gotoTimeoutSecs) {
+            this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
+        }
+        this.navigationTimeoutMillis = (gotoTimeoutSecs || navigationTimeoutSecs) * 1000;
+        this.gotoFunction = gotoFunction;
+        this.defaultGotoOptions = {
+            timeout: this.navigationTimeoutMillis,
+        };
+        this.proxyConfiguration = proxyConfiguration;
+        this.preNavigationHooks = preNavigationHooks;
+        this.postNavigationHooks = postNavigationHooks;
+        if (this.useSessionPool) {
+            this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
+        }
+        else {
+            this.persistCookiesPerSession = false;
+        }
+        const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
+        this.browserPool = new browser_pool_1.BrowserPool({
+            ...rest,
+            preLaunchHooks: [
+                this._extendLaunchContext.bind(this),
+                ...preLaunchHooks,
+            ],
+            postLaunchHooks: [
+                this._maybeAddSessionRetiredListener.bind(this),
+                ...postLaunchHooks,
+            ],
+        });
+    }
+    /**
+     * Wrapper around handlePageFunction that opens and closes pages etc.
+     */
+    async _runRequestHandler(crawlingContext) {
+        const newPageOptions = {
+            id: crawlingContext.id,
+        };
+        const useIncognitoPages = this.launchContext?.useIncognitoPages;
+        if (this.proxyConfiguration && useIncognitoPages) {
+            const { session } = crawlingContext;
+            const proxyInfo = this.proxyConfiguration.newProxyInfo(session?.id);
+            crawlingContext.proxyInfo = proxyInfo;
+            newPageOptions.proxyUrl = proxyInfo.url;
+            if (this.proxyConfiguration.isManInTheMiddle) {
+                /**
+                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
+                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
+                 */
+                newPageOptions.pageOptions = {
+                    ignoreHTTPSErrors: true,
+                };
+            }
+        }
+        const page = await this.browserPool.newPage(newPageOptions);
+        (0, timeout_1.tryCancel)();
+        this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
+        // DO NOT MOVE THIS LINE ABOVE!
+        // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
+        // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
+        // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
+        const { request, session } = crawlingContext;
+        if (this.useSessionPool) {
+            const sessionCookies = session.getPuppeteerCookies(request.url);
+            if (sessionCookies.length) {
+                await crawlingContext.browserController.setCookies(page, sessionCookies);
+                (0, timeout_1.tryCancel)();
+            }
+        }
+        try {
+            await this._handleNavigation(crawlingContext);
+            (0, timeout_1.tryCancel)();
+            await this._responseHandler(crawlingContext);
+            (0, timeout_1.tryCancel)();
+            // save cookies
+            // TODO: Should we save the cookies also after/only the handle page?
+            if (this.persistCookiesPerSession) {
+                const cookies = await crawlingContext.browserController.getCookies(page);
+                (0, timeout_1.tryCancel)();
+                session?.setPuppeteerCookies(cookies, request.loadedUrl);
+            }
+            await (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
+            (0, timeout_1.tryCancel)();
+            if (session)
+                session.markGood();
+        }
+        finally {
+            page.close().catch((error) => this.log.debug('Error while closing page', { error }));
+        }
+    }
+    _enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages) {
+        crawlingContext.page = page;
+        // This switch is because the crawlingContexts are created on per request basis.
+        // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
+        // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
+        // which configures proxy and session for every new request
+        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
+        crawlingContext.browserController = browserControllerInstance;
+        if (!useIncognitoPages) {
+            crawlingContext.session = browserControllerInstance.launchContext.session;
+        }
+        if (!crawlingContext.proxyInfo) {
+            crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
+        }
+        crawlingContext.enqueueLinks = async (enqueueOptions) => {
+            return browserCrawlerEnqueueLinks({
+                options: enqueueOptions,
+                page,
+                requestQueue: await this.getRequestQueue(),
+                defaultBaseUrl: new URL(crawlingContext.request.loadedUrl ?? crawlingContext.request.url).origin,
+            });
+        };
+    }
+    async _handleNavigation(crawlingContext) {
+        const gotoOptions = { ...this.defaultGotoOptions };
+        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+        (0, timeout_1.tryCancel)();
+        try {
+            crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
+        }
+        catch (error) {
+            this._handleNavigationTimeout(crawlingContext, error);
+            throw error;
+        }
+        (0, timeout_1.tryCancel)();
+        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+    }
+    /**
+     * Marks session bad in case of navigation timeout.
+     */
+    _handleNavigationTimeout(crawlingContext, error) {
+        const { session } = crawlingContext;
+        if (error && error.constructor.name === 'TimeoutError') {
+            (0, core_1.handleRequestTimeout)({ session, errorMessage: error.message });
+        }
+    }
+    async _navigationHandler(crawlingContext, gotoOptions) {
+        if (!this.gotoFunction) {
+            // @TODO: although it is optional in the validation,
+            //   because when you make automation library specific you can override this handler.
+            throw new Error('BrowserCrawler: You must specify a gotoFunction!');
+        }
+        return this.gotoFunction(crawlingContext, gotoOptions);
+    }
+    /**
+     * Should be overridden in case of different automation library that does not support this response API.
+     * @todo: This can be also done as a postNavigation hook except the loadedUrl marking.
+     */
+    async _responseHandler(crawlingContext) {
+        const { response, session, request, page } = crawlingContext;
+        if (this.sessionPool && response && session) {
+            if (typeof response === 'object' && typeof response.status === 'function') {
+                (0, core_1.throwOnBlockedRequest)(session, response.status());
+            }
+            else {
+                this.log.debug('Got a malformed Browser response.', { request, response });
+            }
+        }
+        request.loadedUrl = await page.url();
+    }
+    async _extendLaunchContext(_pageId, launchContext) {
+        const launchContextExtends = {};
+        if (this.sessionPool) {
+            launchContextExtends.session = await this.sessionPool.getSession();
+        }
+        if (this.proxyConfiguration) {
+            const proxyInfo = this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
+            launchContext.proxyUrl = proxyInfo.url;
+            launchContextExtends.proxyInfo = proxyInfo;
+            // Disable SSL verification for MITM proxies
+            if (this.proxyConfiguration.isManInTheMiddle) {
+                /**
+                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
+                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
+                 */
+                launchContext.launchOptions.ignoreHTTPSErrors = true;
+            }
+        }
+        launchContext.extend(launchContextExtends);
+    }
+    _maybeAddSessionRetiredListener(_pageId, browserController) {
+        if (this.sessionPool) {
+            const listener = (session) => {
+                const { launchContext } = browserController;
+                if (session.id === launchContext.session.id) {
+                    this.browserPool.retireBrowserController(browserController);
+                }
+            };
+            this.sessionPool.on(core_1.EVENT_SESSION_RETIRED, listener);
+            browserController.on("browserClosed" /* BROWSER_CLOSED */, () => {
+                return this.sessionPool.removeListener(core_1.EVENT_SESSION_RETIRED, listener);
+            });
+        }
+    }
+    /**
+     * Function for cleaning up after all request are processed.
+     * @ignore
+     */
+    async teardown() {
+        await this.browserPool.destroy();
+        await super.teardown();
+    }
+}
+exports.BrowserCrawler = BrowserCrawler;
+Object.defineProperty(BrowserCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...basic_1.BasicCrawler.optionsShape,
+        handlePageFunction: ow_1.default.optional.function,
+        gotoFunction: ow_1.default.optional.function,
+        gotoTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
+        navigationTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
+        preNavigationHooks: ow_1.default.optional.array,
+        postNavigationHooks: ow_1.default.optional.array,
+        browserPoolOptions: ow_1.default.object,
+        sessionPoolOptions: ow_1.default.optional.object,
+        persistCookiesPerSession: ow_1.default.optional.boolean,
+        useSessionPool: ow_1.default.optional.boolean,
+        proxyConfiguration: ow_1.default.optional.object.validate(core_1.validators.proxyConfiguration),
+    }
+});
+/** @internal */
+async function browserCrawlerEnqueueLinks({ options, page, requestQueue, defaultBaseUrl }) {
+    const baseUrl = options?.baseUrl ?? defaultBaseUrl;
+    const urls = await extractUrlsFromPage(page, options?.selector ?? 'a', baseUrl);
+    return (0, core_1.enqueueLinks)({
+        // FIXME this does not make much sense, as the instance would be ignored by any crawler - the argument needs to be required
+        requestQueue: requestQueue ?? await core_1.RequestQueue.open(),
+        urls,
+        baseUrl,
+        ...options,
+    });
+}
+exports.browserCrawlerEnqueueLinks = browserCrawlerEnqueueLinks;
+/**
+ * Extracts URLs from a given page.
+ * @ignore
+ */
+// eslint-disable-next-line @typescript-eslint/ban-types
+async function extractUrlsFromPage(page, selector, baseUrl) {
+    const urls = await page.$$eval(selector, (linkEls) => linkEls.map((link) => link.getAttribute('href')).filter((href) => !!href));
+    return urls.map((href) => {
+        // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
+        const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
+        if (!isHrefAbsolute && !baseUrl) {
+            throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+                + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
+        }
+        return baseUrl
+            ? (new URL(href, baseUrl)).href
+            : href;
+    });
+}
+//# sourceMappingURL=browser-crawler.js.map

package/dist/internals/browser-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":";;;;AAAA,4CAAgE;AAChE,wCAcuB;AACvB,0CAIwB;AAExB,wDAU+B;AAC/B,oDAAoB;AAwPpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAsB,cAKpB,SAAQ,oBAAqB;IA0C3B;;OAEG;IACH,YAAsB,OAAoD;QACtE,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,yBAAyB,GAAG,EAAE,EAC9B,YAAY,EAAE,aAAa;QAC3B,eAAe,EAAE,aAAa;QAC9B,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QACxB,UAAU;QACV,qBAAqB,EAErB,cAAc,EAAE,0BAA0B,EAC1C,kBAAkB,EAElB,oBAAoB,EACpB,2BAA2B,EAC3B,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC;YAC7D,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,CAAC,CAAC;QAvEP;;;WAGG;QACH;;;;;WAAwC;QAExC;;;;WAIG;QACH;;;;;WAAqD;QAErD;;;;;WAA6D;QAE7D;;;;;WAA4E;QAC5E;;;;;WAA0C;QAC1C;;;;;WAA4D;QAC5D;;;;;WAA0C;QAC1C;;;;;WAAqD;QACrD;;;;;WAAsD;QACtD;;;;;WAA4C;QAoDxC,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,4BAA4B;YACzC,WAAW,EAAE,0BAA0B;YACvC,WAAW,EAAE,kBAAkB;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,sBAAsB;YAC/B,OAAO,EAAE,6BAA6B;YACtC,WAAW,EAAE,sBAAsB;YACnC,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,2BAA2B;YACxC,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,8EAA8E,CAAC,CAAC;SACvG;QAED,IAAI,CAAC,uBAAuB,GAAG,CAAC,eAAe,IAAI,qBAAqB,CAAC,GAAG,IAAI,CAAC;QAEjF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACd,CAAC;QAE5B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAElF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAA6B;YAC3D,GAAG,IAAW;YACd,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAEhE,IAAI,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACpE,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YAExC,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;iBAC1B,CAAC;aACL;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAe,CAAC;QAC1E,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,+BAA+B;QAC/B,iEAAiE;QACjE,6EAA6E;QAC7E,+GAA+G;QAC/G,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,cAAc,GAAG,OAAQ,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjE,IAAI,cAAc,CAAC,MAAM,EAAE;gBACvB,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;aACf;SACJ;QAED,IAAI;YACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,IAAA,mBAAS,GAAE,CAAC;YAEZ,eAAe;YACf,oEAAoE;YACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;gBACZ,OAAO,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAU,CAAC,CAAC;aAC7D;YAED,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,eAAe,CAAC,CAAC,EACvE,IAAI,CAAC,2BAA2B,EAChC,kCAAkC,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CACvF,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SAC/F;IACL,CAAC;IAES,mCAAmC,CAAC,eAAwB,EAAE,IAAgB,EAAE,iBAA2B;QACjH,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAW,CAAiC,CAAC;QAC3H,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,iBAAiB,EAAE;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAkB,CAAC;SACxF;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC5B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAsB,CAAC;SAC9F;QAED,eAAe,CAAC,YAAY,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;YACpD,OAAO,0BAA0B,CAAC;gBAC9B,OAAO,EAAE,cAAc;gBACvB,IAAI;gBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;gBAC1C,cAAc,EAAE,IAAI,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;aACnG,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1F;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAE/D,MAAM,KAAK,CAAC;SACf;QAED,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAED;;OAEG;IACO,wBAAwB,CAAC,eAAwB,EAAE,KAAY;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,2BAAoB,EAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAClE;IACL,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAAwB,EAAE,WAAwB;QACjF,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,oDAAoD;YACpD,qFAAqF;YACrF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;SACvE;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QACrD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE;YACzC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAA,4BAAqB,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACrD;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAe,EAAE,aAA4B;QAC9E,MAAM,oBAAoB,GAAiD,EAAE,CAAC;QAE9E,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxH,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACF,aAAa,CAAC,aAA4B,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxE;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAES,+BAA+B,CAAC,OAAe,EAAE,iBAA+C;QACtG,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;gBAClC,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAM,aAAa,CAAC,OAAmB,CAAC,EAAE,EAAE;oBACtD,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAAsG,CACzG,CAAC;iBACL;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,4BAAqB,EAAE,QAAQ,CAAC,CAAC;YACrD,iBAAiB,CAAC,EAAE,uCAA2C,GAAG,EAAE;gBAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,4BAAqB,EAAE,QAAQ,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AA1VL,wCA2VC;AA9TG;;;;WAAyC;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAElC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC;AAwTN,gBAAgB;AACT,KAAK,UAAU,0BAA0B,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,cAAc,EAA+B;IACzH,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,cAAc,CAAC;IAEnD,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,IAAW,EAAE,OAAO,EAAE,QAAQ,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;IAEvF,OAAO,IAAA,mBAAY,EAAC;QAChB,2HAA2H;QAC3H,YAAY,EAAE,YAAY,IAAI,MAAM,mBAAY,CAAC,IAAI,EAAE;QACvD,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AAZD,gEAYC;AAED;;;GAGG;AACH,wDAAwD;AACxD,KAAK,UAAU,mBAAmB,CAAC,IAA0B,EAAE,QAAgB,EAAE,OAAgB;IAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpJ,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAC7B,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,+CAA+C;kBAC9E,+EAA+E,CAAC,CAAC;SAC9F;QACD,OAAO,OAAO;YACV,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI;YAC/B,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/internals/browser-launcher.d.ts

@@ -0,0 +1,60 @@
+import { Dictionary, Constructor } from '@crawlee/utils';
+import { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';
+export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
+    /**
+     * If `true` and `executablePath` is not set,
+     * the launcher will launch full Google Chrome browser available on the machine
+     * rather than the bundled Chromium. The path to Chrome executable
+     * is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
+     * or defaults to the typical Google Chrome executable location specific for the operating system.
+     * By default, this option is `false`.
+     * @default false
+     */
+    useChrome?: boolean;
+    launcher?: Launcher;
+}
+/**
+ * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
+ * @ignore
+ */
+export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
+    launcher: Launcher;
+    proxyUrl?: string;
+    useChrome?: boolean;
+    launchOptions: Dictionary;
+    otherLaunchContextProps: Dictionary;
+    Plugin: T;
+    userAgent?: string;
+    protected static optionsShape: {
+        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+    };
+    static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T;
+    /**
+     * All `BrowserLauncher` parameters are passed via an launchContext object.
+     */
+    constructor(launchContext: BrowserLaunchContext<LaunchOptions, Launcher>);
+    /**
+     * @ignore
+     */
+    createBrowserPlugin(): Plugin;
+    /**
+     * Launches a browser instance based on the plugin.
+     * @returns Browser instance.
+     */
+    launch(): LaunchResult;
+    createLaunchOptions(): {
+        args?: string[];
+    } & Dictionary;
+    protected _getDefaultHeadlessOption(): boolean;
+    protected _getChromeExecutablePath(): string;
+    /**
+     * Gets a typical path to Chrome executable, depending on the current operating system.
+     */
+    protected _getTypicalChromeExecutablePath(): string;
+    protected _validateProxyUrlProtocol(proxyUrl?: string): void;
+}
+//# sourceMappingURL=browser-launcher.d.ts.map

package/dist/internals/browser-launcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EACH,UAAU,EACV,WAAW,EACd,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,aAAa,EACb,oBAAoB,EACvB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxD,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhF,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,uBAAuB,EAAE,UAAU,CAAC;IAEpC,MAAM,EAAG,CAAC,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;MAM3B;IAEF,MAAM,CAAC,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC;IAuB7E;;OAEG;gBACS,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,QAAQ,CAAC;IAmBxE;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAQ7B;;;OAGG;IACH,MAAM,IAAI,YAAY;IAOtB,mBAAmB,IAAI;QAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,UAAU;IAgBvD,SAAS,CAAC,yBAAyB,IAAI,OAAO;IAI9C,SAAS,CAAC,wBAAwB,IAAI,MAAM;IAI5C;;OAEG;IACH,SAAS,CAAC,+BAA+B,IAAI,MAAM;IA8BnD,SAAS,CAAC,yBAAyB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;CAe/D"}