@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.5

package/internals/puppeteer-crawler.d.ts

@@ -0,0 +1,149 @@
+import { BrowserCrawler, BrowserCrawlerHandleRequest, BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook } from '@crawlee/browser';
+import { PuppeteerPlugin } from '@crawlee/browser-pool';
+import { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
+import { PuppeteerLaunchContext } from './puppeteer-launcher';
+import { DirectNavigationOptions } from './utils/puppeteer_utils';
+export declare type PuppeteerController = ReturnType<PuppeteerPlugin['_createController']>;
+export declare type PuppeteerCrawlContext = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;
+export declare type PuppeteerHook = BrowserHook<PuppeteerCrawlContext, PuppeteerGoToOptions>;
+export declare type PuppeteerRequestHandlerParam = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;
+export declare type PuppeteerRequestHandler = BrowserCrawlerHandleRequest<PuppeteerRequestHandlerParam>;
+export declare type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
+export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<PuppeteerCrawlContext, PuppeteerGoToOptions, {
+    browserPlugins: [PuppeteerPlugin];
+}> {
+    /**
+     * Options used by {@link launchPuppeteer} to start new Puppeteer instances.
+     */
+    launchContext?: PuppeteerLaunchContext;
+    /**
+     * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
+     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
+     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * Example:
+     * ```
+     * preNavigationHooks: [
+     *     async (crawlingContext, gotoOptions) => {
+     *         const { page } = crawlingContext;
+     *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *     },
+     * ]
+     * ```
+     */
+    preNavigationHooks?: PuppeteerHook[];
+    /**
+     * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
+     * The function accepts `crawlingContext` as the only parameter.
+     * Example:
+     * ```
+     * postNavigationHooks: [
+     *     async (crawlingContext) => {
+     *         const { page } = crawlingContext;
+     *         if (hasCaptcha(page)) {
+     *             await solveCaptcha (page);
+     *         }
+     *     },
+     * ]
+     * ```
+     */
+    postNavigationHooks?: PuppeteerHook[];
+}
+/**
+ * Provides a simple framework for parallel crawling of web pages
+ * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `PuppeteerCrawler` uses headless Chrome to download web pages and extract data,
+ * it is useful for crawling of websites that require to execute JavaScript.
+ * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * which downloads the pages using raw HTTP requests and is about 10x faster.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
+ * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.handlePageFunction} option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
+ *
+ * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ page, request }) => {
+ *         // This function is called to extract data from a single web page
+ *         // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ *         // 'request' is an instance of Request class with information about the page to load
+ *         await Actor.pushData({
+ *             title: await page.title(),
+ *             url: request.url,
+ *             succeeded: true,
+ *         })
+ *     },
+ *     handleFailedRequestFunction: async ({ request }) => {
+ *         // This function is called when the crawling of a request failed too many times
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             succeeded: false,
+ *             errors: request.errorMessages,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+export declare class PuppeteerCrawler extends BrowserCrawler<{
+    browserPlugins: [PuppeteerPlugin];
+}, LaunchOptions, PuppeteerCrawlContext> {
+    protected static optionsShape: {
+        browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        gotoFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        gotoTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+    };
+    /**
+     * All `PuppeteerCrawler` parameters are passed via an options object.
+     */
+    constructor(options: PuppeteerCrawlerOptions);
+    protected _navigationHandler(crawlingContext: PuppeteerCrawlContext, gotoOptions: DirectNavigationOptions): Promise<any>;
+}
+//# sourceMappingURL=puppeteer-crawler.d.ts.map

package/internals/puppeteer-crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAsB,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAqB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,uBAAuB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,oBAAY,mBAAmB,GAAG,UAAU,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAEnF,oBAAY,qBAAqB,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAEnG,oBAAY,aAAa,GAAG,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;AAErF,oBAAY,4BAA4B,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAE1G,oBAAY,uBAAuB,GAAG,2BAA2B,CAAC,4BAA4B,CAAC,CAAC;AAEhG,oBAAY,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAClE,qBAAqB,EACrB,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAAC;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EAAE,aAAa,EAAE,qBAAqB,CAAC;IAC7H,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF;;OAEG;gBACS,OAAO,EAAE,uBAAuB;cAqCnB,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,EAAE,WAAW,EAAE,uBAAuB;CAQ3H"}

package/internals/puppeteer-crawler.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PuppeteerCrawler = void 0;
+const tslib_1 = require("tslib");
+const browser_1 = require("@crawlee/browser");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const puppeteer_launcher_1 = require("./puppeteer-launcher");
+const stealth_1 = require("./stealth");
+const puppeteer_utils_1 = require("./utils/puppeteer_utils");
+/**
+ * Provides a simple framework for parallel crawling of web pages
+ * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `PuppeteerCrawler` uses headless Chrome to download web pages and extract data,
+ * it is useful for crawling of websites that require to execute JavaScript.
+ * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * which downloads the pages using raw HTTP requests and is about 10x faster.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
+ * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.handlePageFunction} option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
+ *
+ * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ page, request }) => {
+ *         // This function is called to extract data from a single web page
+ *         // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ *         // 'request' is an instance of Request class with information about the page to load
+ *         await Actor.pushData({
+ *             title: await page.title(),
+ *             url: request.url,
+ *             succeeded: true,
+ *         })
+ *     },
+ *     handleFailedRequestFunction: async ({ request }) => {
+ *         // This function is called when the crawling of a request failed too many times
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             succeeded: false,
+ *             errors: request.errorMessages,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+class PuppeteerCrawler extends browser_1.BrowserCrawler {
+    /**
+     * All `PuppeteerCrawler` parameters are passed via an options object.
+     */
+    constructor(options) {
+        (0, ow_1.default)(options, 'PuppeteerCrawlerOptions', ow_1.default.object.exactShape(PuppeteerCrawler.optionsShape));
+        const { launchContext = {}, browserPoolOptions = {}, proxyConfiguration, ...browserCrawlerOptions } = options;
+        const { stealth = false } = launchContext;
+        if (launchContext.proxyUrl) {
+            throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
+                + 'Use PuppeteerCrawlerOptions.proxyConfiguration');
+        }
+        const puppeteerLauncher = new puppeteer_launcher_1.PuppeteerLauncher(launchContext);
+        browserPoolOptions.browserPlugins = [
+            puppeteerLauncher.createBrowserPlugin(),
+        ];
+        super({ ...browserCrawlerOptions, proxyConfiguration, browserPoolOptions });
+        if (stealth) {
+            this.browserPool.postLaunchHooks.push(async (_pageId, browserController) => {
+                // TODO: We can do this better now. It is not necessary to override the page.
+                //  we can modify the page in the postPageCreateHook
+                const { hideWebDriver, ...newStealthOptions } = puppeteerLauncher.stealthOptions;
+                (0, stealth_1.applyStealthToBrowser)(browserController.browser, newStealthOptions);
+            });
+        }
+        this.launchContext = launchContext;
+    }
+    async _navigationHandler(crawlingContext, gotoOptions) {
+        if (this.gotoFunction) {
+            this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
+            return this.gotoFunction(crawlingContext, gotoOptions);
+        }
+        return (0, puppeteer_utils_1.gotoExtended)(crawlingContext.page, crawlingContext.request, gotoOptions);
+    }
+}
+exports.PuppeteerCrawler = PuppeteerCrawler;
+Object.defineProperty(PuppeteerCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...browser_1.BrowserCrawler.optionsShape,
+        browserPoolOptions: ow_1.default.optional.object,
+        launchContext: ow_1.default.optional.object,
+    }
+});
+//# sourceMappingURL=puppeteer-crawler.js.map

package/internals/puppeteer-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;AAK1B,oDAAoB;AAEpB,6DAAiF;AACjF,uCAAkD;AAClD,6DAAgF;AA0DhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,MAAa,gBAAiB,SAAQ,wBAA2F;IAO7H;;OAEG;IACH,YAAY,OAAgC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAwB,EAC7C,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,aAAa,CAAC;QAE1C,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QAED,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAE5E,IAAI,OAAO,EAAE;YACT,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE;gBACvE,6EAA6E;gBAC7E,oDAAoD;gBACpD,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,iBAAiB,CAAC,cAAe,CAAC;gBAClF,IAAA,+BAAqB,EAAC,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAsC,EAAE,WAAoC;QACpH,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAyB,CAAC,CAAC;SACxE;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAtDL,4CAuDC;AAtDG;;;;WAAyC;QACrC,GAAG,wBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;GAAC"}

package/internals/puppeteer-launcher.d.ts

@@ -0,0 +1,142 @@
+import { Browser } from 'puppeteer';
+import { PuppeteerPlugin } from '@crawlee/browser-pool';
+import { BrowserLaunchContext, BrowserLauncher } from '@crawlee/browser';
+import { StealthOptions } from './stealth';
+/**
+ * Apify extends the launch options of Puppeteer.
+ * You can use any of the Puppeteer compatible
+ * [`LaunchOptions`](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)
+ * options by providing the `launchOptions` property.
+ *
+ * **Example:**
+ * ```js
+ * // launch a headless Chrome (not Chromium)
+ * const launchContext = {
+ *     // Apify helpers
+ *     useChrome: true,
+ *     proxyUrl: 'http://user:password@some.proxy.com'
+ *     // Native Puppeteer options
+ *     launchOptions: {
+ *         headless: true,
+ *         args: ['--some-flag'],
+ *     }
+ * }
+ * ```
+ */
+export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPlugin['launchOptions'], unknown> {
+    /**
+     *  `puppeteer.launch` [options](https://pptr.dev/#?product=Puppeteer&version=v13.5.1&show=api-puppeteerlaunchoptions)
+     */
+    launchOptions?: PuppeteerPlugin['launchOptions'];
+    /**
+     * URL to a HTTP proxy server. It must define the port number,
+     * and it may also contain proxy username and password.
+     *
+     * Example: `http://bob:pass123@proxy.example.com:1234`.
+     */
+    proxyUrl?: string;
+    /**
+     * The `User-Agent` HTTP header used by the browser.
+     * If not provided, the function sets `User-Agent` to a reasonable default
+     * to reduce the chance of detection of the crawler.
+     */
+    userAgent?: string;
+    /**
+     * If `true` and `executablePath` is not set,
+     * Puppeteer will launch full Google Chrome browser available on the machine
+     * rather than the bundled Chromium. The path to Chrome executable
+     * is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
+     * or defaults to the typical Google Chrome executable location specific for the operating system.
+     * By default, this option is `false`.
+     * @default false
+     */
+    useChrome?: boolean;
+    /**
+     * Already required module (`Object`). This enables usage of various Puppeteer
+     * wrappers such as `puppeteer-extra`.
+     *
+     * Take caution, because it can cause all kinds of unexpected errors and weird behavior.
+     * Apify SDK is not tested with any other library besides `puppeteer` itself.
+     */
+    launcher?: unknown;
+    /**
+     * With this option selected, all pages will be opened in a new incognito browser context.
+     * This means they will not share cookies nor cache and their resources will not be throttled by one another.
+     * @default false
+     */
+    useIncognitoPages?: boolean;
+    /**
+     * This setting hides most of the known properties that identify headless Chrome and makes it nearly undetectable.
+     * It is recommended to use it together with the `useChrome` set to `true`.
+     * @deprecated
+     */
+    stealth?: boolean;
+    /**
+     * Using this configuration, you can disable some of the hiding tricks.
+     * For these settings to take effect `stealth` must be set to true
+     * @deprecated
+     */
+    stealthOptions?: StealthOptions;
+}
+/**
+ * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
+ * @ignore
+ */
+export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
+    protected static optionsShape: {
+        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        stealth: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        stealthOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+    };
+    stealth?: boolean;
+    stealthOptions: StealthOptions;
+    /**
+     * All `PuppeteerLauncher` parameters are passed via an launchContext object.
+     */
+    constructor(launchContext?: PuppeteerLaunchContext);
+    launch(): Promise<import("packages/browser-pool/dist/puppeteer/puppeteer-proxy-per-page").Browser>;
+    createLaunchOptions(): {
+        args?: string[] | undefined;
+    } & import("packages/utils/dist").Dictionary<unknown>;
+}
+/**
+ * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
+ * The function has the same argument and the return value as `puppeteer.launch()`.
+ * See [Puppeteer documentation](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions) for more details.
+ *
+ * The `launchPuppeteer()` function alters the following Puppeteer options:
+ *
+ * - Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
+ *   unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.
+ *   Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.
+ * - Takes the `proxyUrl` option, validates it and adds it to `args` as `--proxy-server=XXX`.
+ *   The proxy URL must define a port number and have one of the following schemes: `http://`,
+ *   `https://`, `socks4://` or `socks5://`.
+ *   If the proxy is HTTP (i.e. has the `http://` scheme) and contains username or password,
+ *   the `launchPuppeteer` functions sets up an anonymous proxy HTTP
+ *   to make the proxy work with headless Chrome. For more information, read the
+ *   [blog post about proxy-chain library](https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212).
+ *
+ * To use this function, you need to have the [puppeteer](https://www.npmjs.com/package/puppeteer)
+ * NPM package installed in your project.
+ * When running on the Apify cloud, you can achieve that simply
+ * by using the `apify/actor-node-chrome` base Docker image for your actor - see
+ * [Apify Actor documentation](https://docs.apify.com/actor/build#base-images)
+ * for details.
+ *
+ * For an example of usage, see the [Puppeteer proxy Example](/docs/examples/puppeteer-with-proxy).
+ *
+ * @param [launchContext]
+ *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
+ *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
+ * @returns
+ *   Promise that resolves to Puppeteer's `Browser` instance.
+ */
+export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext): Promise<Browser>;
+//# sourceMappingURL=puppeteer-launcher.d.ts.map

package/internals/puppeteer-launcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE,OAAO,EAAyB,cAAc,EAAE,MAAM,WAAW,CAAC;AAalE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB,CAAC,eAAe,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC3G;;OAEG;IACH,aAAa,CAAC,EAAE,eAAe,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;CACnC;AAED;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC;IAC5E,iBAA0B,YAAY;;;;;;;;;;MAMpC;IAEF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,EAAE,cAAc,CAAC;IAE/B;;OAEG;gBACS,aAAa,GAAE,sBAA2B;IA0BvC,MAAM;IAiBZ,mBAAmB;;;CAoC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,eAAe,CAAC,aAAa,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAI9F"}

package/internals/puppeteer-launcher.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.launchPuppeteer = exports.PuppeteerLauncher = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const browser_pool_1 = require("@crawlee/browser-pool");
+const browser_1 = require("@crawlee/browser");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const stealth_1 = require("./stealth");
+/**
+ * The default user agent used by `Actor.launchPuppeteer`.
+ * Last updated on 2020-05-22.
+ */
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.67 Safari/537.36';
+const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
+    width: 1366,
+    height: 768,
+};
+/**
+ * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
+ * @ignore
+ */
+class PuppeteerLauncher extends browser_1.BrowserLauncher {
+    /**
+     * All `PuppeteerLauncher` parameters are passed via an launchContext object.
+     */
+    constructor(launchContext = {}) {
+        (0, ow_1.default)(launchContext, 'PuppeteerLauncher', ow_1.default.object.exactShape(PuppeteerLauncher.optionsShape));
+        const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), userAgent, stealth = false, stealthOptions = {}, ...browserLauncherOptions } = launchContext;
+        super({
+            ...browserLauncherOptions,
+            launcher,
+        });
+        Object.defineProperty(this, "stealth", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "stealthOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.userAgent = userAgent;
+        this.stealth = stealth;
+        this.stealthOptions = {
+            hideWebDriver: true,
+            ...stealthOptions,
+        };
+        this.Plugin = browser_pool_1.PuppeteerPlugin;
+    }
+    async launch() {
+        const browser = await super.launch();
+        if (this.stealth) {
+            log_1.default.deprecated('Puppeteer "stealth" and "stealthOptions" are deprecated.'
+                + ' You should use fingerprints instead.'
+                + ' Checkout the fingerprints guide: https://sdk.apify.com/docs/guides/avoid-blocking');
+            const { hideWebDriver, ...newStealthOptions } = this.stealthOptions;
+            (0, stealth_1.applyStealthToBrowser)(browser, newStealthOptions);
+        }
+        return browser;
+    }
+    createLaunchOptions() {
+        const launchOptions = super.createLaunchOptions();
+        launchOptions.args = launchOptions.args || [];
+        try {
+            // eslint-disable-next-line
+            const { isAtHome } = require('apify');
+            if (isAtHome())
+                launchOptions.args.push('--no-sandbox');
+            // eslint-disable-next-line no-empty
+        }
+        catch { }
+        launchOptions.defaultViewport ?? (launchOptions.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT);
+        // When User-Agent is not set and we're using Chromium or headless mode,
+        // it is better to use DEFAULT_USER_AGENT to reduce chance of detection
+        let { userAgent } = this;
+        if (!userAgent && (!launchOptions.executablePath || launchOptions.headless)) {
+            userAgent = DEFAULT_USER_AGENT;
+        }
+        if (userAgent) {
+            launchOptions.args.push(`--user-agent=${userAgent}`);
+        }
+        if (this.stealthOptions?.hideWebDriver) {
+            const idx = launchOptions.args.findIndex((arg) => arg.startsWith('--disable-blink-features='));
+            if (idx !== -1) {
+                const arg = launchOptions.args[idx];
+                launchOptions.args[idx] = `${arg},AutomationControlled`;
+            }
+            else {
+                launchOptions.args.push('--disable-blink-features=AutomationControlled');
+            }
+        }
+        return launchOptions;
+    }
+}
+exports.PuppeteerLauncher = PuppeteerLauncher;
+Object.defineProperty(PuppeteerLauncher, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...browser_1.BrowserLauncher.optionsShape,
+        launcher: ow_1.default.optional.object,
+        userAgent: ow_1.default.optional.string,
+        stealth: ow_1.default.optional.boolean,
+        stealthOptions: ow_1.default.optional.object,
+    }
+});
+/**
+ * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
+ * The function has the same argument and the return value as `puppeteer.launch()`.
+ * See [Puppeteer documentation](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions) for more details.
+ *
+ * The `launchPuppeteer()` function alters the following Puppeteer options:
+ *
+ * - Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
+ *   unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.
+ *   Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running actors.
+ * - Takes the `proxyUrl` option, validates it and adds it to `args` as `--proxy-server=XXX`.
+ *   The proxy URL must define a port number and have one of the following schemes: `http://`,
+ *   `https://`, `socks4://` or `socks5://`.
+ *   If the proxy is HTTP (i.e. has the `http://` scheme) and contains username or password,
+ *   the `launchPuppeteer` functions sets up an anonymous proxy HTTP
+ *   to make the proxy work with headless Chrome. For more information, read the
+ *   [blog post about proxy-chain library](https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212).
+ *
+ * To use this function, you need to have the [puppeteer](https://www.npmjs.com/package/puppeteer)
+ * NPM package installed in your project.
+ * When running on the Apify cloud, you can achieve that simply
+ * by using the `apify/actor-node-chrome` base Docker image for your actor - see
+ * [Apify Actor documentation](https://docs.apify.com/actor/build#base-images)
+ * for details.
+ *
+ * For an example of usage, see the [Puppeteer proxy Example](/docs/examples/puppeteer-with-proxy).
+ *
+ * @param [launchContext]
+ *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
+ *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
+ * @returns
+ *   Promise that resolves to Puppeteer's `Browser` instance.
+ */
+async function launchPuppeteer(launchContext) {
+    const puppeteerLauncher = new PuppeteerLauncher(launchContext);
+    return puppeteerLauncher.launch();
+}
+exports.launchPuppeteer = launchPuppeteer;
+//# sourceMappingURL=puppeteer-launcher.js.map

package/internals/puppeteer-launcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-launcher.js","sourceRoot":"","sources":["../../src/internals/puppeteer-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,wDAAwD;AACxD,8CAAyE;AACzE,6DAA6B;AAC7B,uCAAkE;AAElE;;;GAGG;AACH,MAAM,kBAAkB,GAAG,0HAA0H,CAAC;AAEtJ,MAAM,iCAAiC,GAAG;IACtC,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAsFF;;;GAGG;AACH,MAAa,iBAAkB,SAAQ,yBAAyC;IAY5E;;OAEG;IACH,YAAY,gBAAwC,EAAE;QAClD,IAAA,YAAE,EAAC,aAAa,EAAE,mBAAmB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,yBAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,SAAS,EACT,OAAO,GAAG,KAAK,EACf,cAAc,GAAG,EAAE,EACnB,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CAAC;YACF,GAAG,sBAAsB;YACzB,QAAQ;SACX,CAAC,CAAC;QApBP;;;;;WAAkB;QAClB;;;;;WAA+B;QAqB3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG;YAClB,aAAa,EAAE,IAAI;YACnB,GAAG,cAAc;SACpB,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,8BAAe,CAAC;IAClC,CAAC;IAEQ,KAAK,CAAC,MAAM;QACjB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QAErC,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,aAAG,CAAC,UAAU,CACV,0DAA0D;kBACxD,uCAAuC;kBACvC,oFAAoF,CACzF,CAAC;YACF,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,CAAC,cAAe,CAAC;YAErE,IAAA,+BAAqB,EAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;SACrD;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEQ,mBAAmB;QACxB,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;QAClD,aAAa,CAAC,IAAI,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC;QAE9C,IAAI;YACA,2BAA2B;YAC3B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,QAAQ,EAAE;gBAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC5D,oCAAoC;SACnC;QAAC,MAAM,GAAE;QAEV,aAAa,CAAC,eAAe,KAA7B,aAAa,CAAC,eAAe,GAAK,iCAAiC,EAAC;QAEpE,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,aAAa,CAAC,cAAc,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE;YACzE,SAAS,GAAG,kBAAkB,CAAC;SAClC;QAED,IAAI,SAAS,EAAE;YACX,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,aAAa,EAAE;YACpC,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAC/F,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE;gBACZ,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,uBAAuB,CAAC;aAC3D;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;aAC5E;SACJ;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;;AA7FL,8CA8FC;AA7FG;;;;WAAyC;QACrC,GAAG,yBAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACrC;GAAC;AAyFN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACI,KAAK,UAAU,eAAe,CAAC,aAAsC;IACxE,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAE/D,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC;AAJD,0CAIC"}

package/internals/stealth/hiding_tricks.d.ts

@@ -0,0 +1,10 @@
+export declare const hackPermissions: () => void;
+export declare const addLanguage: () => void;
+export declare const emulateWebGL: () => void;
+export declare const emulateWindowFrame: () => void;
+export declare const addPlugins: () => void;
+export declare const emulateConsoleDebug: () => void;
+export declare const mockChrome: () => void;
+export declare const mockChromeInIframe: () => void;
+export declare const mockDeviceMemory: () => void;
+//# sourceMappingURL=hiding_tricks.d.ts.map

package/internals/stealth/hiding_tricks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hiding_tricks.d.ts","sourceRoot":"","sources":["../../../src/internals/stealth/hiding_tricks.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,eAAe,YAqC3B,CAAC;AAEF,eAAO,MAAM,WAAW,YAIvB,CAAC;AAEF,eAAO,MAAM,YAAY,YAoBxB,CAAC;AAEF,eAAO,MAAM,kBAAkB,YAS9B,CAAC;AAEF,eAAO,MAAM,UAAU,YAmNtB,CAAC;AAEF,eAAO,MAAM,mBAAmB,YAI/B,CAAC;AAGF,eAAO,MAAM,UAAU,YAStB,CAAC;AAIF,eAAO,MAAM,kBAAkB,YA0D9B,CAAC;AAEF,eAAO,MAAM,gBAAgB,YAK5B,CAAC"}