@crawlee/browser-pool 3.0.0-alpha.9 → 3.0.0-beta.2

package/README.md

@@ -18,6 +18,8 @@ running on the [Apify Cloud](https://apify.com/).
 
 **View full documentation, guides and examples on the [Apify SDK project website](https://sdk.apify.com)**
 
+> Would you like to work with us on Apify SDK or similar projects? [We are hiring!](https://apify.com/jobs#senior-node.js-engineer)
+
 ## Motivation
 
 Thanks to tools like [Playwright](https://github.com/microsoft/playwright), [Puppeteer](https://github.com/puppeteer/puppeteer) or

package/abstract-classes/browser-controller.d.ts

@@ -0,0 +1,153 @@
+import { TypedEmitter } from 'tiny-typed-emitter';
+import { BROWSER_CONTROLLER_EVENTS } from '../events';
+import { LaunchContext } from '../launch-context';
+import { UnwrapPromise } from '../utils';
+import { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin';
+export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+    [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
+}
+export interface Cookie {
+    /**
+     * Cookie name.
+     */
+    name: string;
+    /**
+     * Cookie value.
+     */
+    value: string;
+    /**
+     * The request-URI to associate with the setting of the cookie. This value can affect the
+     * default domain, path, source port, and source scheme values of the created cookie.
+     */
+    url?: string;
+    /**
+     * Cookie domain.
+     */
+    domain?: string;
+    /**
+     * Cookie path.
+     */
+    path?: string;
+    /**
+     * True if cookie is secure.
+     */
+    secure?: boolean;
+    /**
+     * True if cookie is http-only.
+     */
+    httpOnly?: boolean;
+    /**
+     * Cookie SameSite type.
+     */
+    sameSite?: 'Strict' | 'Lax' | 'None';
+    /**
+     * Cookie expiration date, session cookie if not set
+     */
+    expires?: number;
+    /**
+     * Cookie Priority.
+     */
+    priority?: 'Low' | 'Medium' | 'High';
+    /**
+     * True if cookie is SameParty.
+     */
+    sameParty?: boolean;
+    /**
+     * Cookie source scheme type.
+     */
+    sourceScheme?: 'Unset' | 'NonSecure' | 'Secure';
+    /**
+     * Cookie source port. Valid values are {-1, [1, 65535]}, -1 indicates an unspecified port.
+     * An unspecified port value allows protocol clients to emulate legacy cookie scope for the port.
+     * This is a temporary ability and it will be removed in the future.
+     */
+    sourcePort?: number;
+}
+/**
+ * The `BrowserController` serves two purposes. First, it is the base class that
+ * specialized controllers like `PuppeteerController` or `PlaywrightController`
+ * extend. Second, it defines the public interface of the specialized classes
+ * which provide only private methods. Therefore, we do not keep documentation
+ * for the specialized classes, because it's the same for all of them.
+ * @hideconstructor
+ */
+export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
+    id: string;
+    /**
+     * The `BrowserPlugin` instance used to launch the browser.
+     */
+    browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+    /**
+     * Browser representation of the underlying automation library.
+     */
+    browser: LaunchResult;
+    /**
+     * The configuration the browser was launched with.
+     */
+    launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+    isActive: boolean;
+    activePages: number;
+    totalPages: number;
+    lastPageOpenedAt: number;
+    private _activate;
+    private isActivePromise;
+    private commitBrowser;
+    private hasBrowserPromise;
+    constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
+    /**
+     * Activates the BrowserController. If you try to open new pages before
+     * activation, the pages will get queued and will only be opened after
+     * activate is called.
+     * @ignore
+     */
+    activate(): void;
+    /**
+     * @ignore
+     */
+    assignBrowser(browser: LaunchResult, launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): void;
+    /**
+     * Gracefully closes the browser and makes sure
+     * there will be no lingering browser processes.
+     *
+     * Emits 'browserClosed' event.
+     */
+    close(): Promise<void>;
+    /**
+     * Immediately kills the browser process.
+     *
+     * Emits 'browserClosed' event.
+     */
+    kill(): Promise<void>;
+    /**
+     * Opens new browser page.
+     * @ignore
+     */
+    newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+    getCookies(page: NewPageResult): Promise<Cookie[]>;
+    /**
+     * @private
+     */
+    protected abstract _close(): Promise<void>;
+    /**
+     * @private
+     */
+    protected abstract _kill(): Promise<void>;
+    /**
+     * @private
+     */
+    protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    /**
+     * @private
+     */
+    protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+    /**
+     * @private
+     */
+    protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
+    /**
+     * @private
+     */
+    abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
+}
+//# sourceMappingURL=browser-controller.d.ts.map

package/abstract-classes/browser-controller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-controller.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAK/E,MAAM,WAAW,uBAAuB,CACpC,OAAO,SAAS,aAAa,EAC7B,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EACjD,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,CAAC,yBAAyB,CAAC,cAAc,CAAC,EACtC,CAAC,UAAU,EAAE,iBAAiB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,KAAK,IAAI,CAAC;CACrH;AAED,MAAM,WAAW,MAAM;IACnB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;IAChD;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB,CACnC,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EACjD,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CACpE,SAAQ,YAAY,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IACjH,EAAE,SAAY;IAEd;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IAEnG;;OAEG;IACH,OAAO,EAAE,YAAY,CAAc;IAEnC;;OAEG;IACH,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,CAAc;IAEhH,QAAQ,UAAS;IAEjB,WAAW,SAAK;IAEhB,UAAU,SAAK;IAEf,gBAAgB,SAAc;IAE9B,OAAO,CAAC,SAAS,CAAc;IAE/B,OAAO,CAAC,eAAe,CAEpB;IAEH,OAAO,CAAC,aAAa,CAAc;IAEnC,OAAO,CAAC,iBAAiB,CAEtB;gBAES,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAK9G;;;;;OAKG;IACH,QAAQ,IAAI,IAAI;IAQhB;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,IAAI;IAS9I;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB5B;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACG,OAAO,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAW7D,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjE,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAIxD;;OAEG;uBAGsB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhD;;OAEG;uBAGsB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/C;;OAEG;uBAEsB,QAAQ,CAAC,WAAW,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAIvF;;OAEG;uBAEsB,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3F;;OAEG;uBAEsB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAI5E;;OAEG;IAEH,QAAQ,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAG1G"}

package/abstract-classes/browser-controller.js

@@ -0,0 +1,230 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BrowserController = void 0;
+const nanoid_1 = require("nanoid");
+const tiny_typed_emitter_1 = require("tiny-typed-emitter");
+const timeout_1 = require("@apify/timeout");
+const logger_1 = require("../logger");
+const utils_1 = require("./utils");
+const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
+/**
+ * The `BrowserController` serves two purposes. First, it is the base class that
+ * specialized controllers like `PuppeteerController` or `PlaywrightController`
+ * extend. Second, it defines the public interface of the specialized classes
+ * which provide only private methods. Therefore, we do not keep documentation
+ * for the specialized classes, because it's the same for all of them.
+ * @hideconstructor
+ */
+class BrowserController extends tiny_typed_emitter_1.TypedEmitter {
+    constructor(browserPlugin) {
+        super();
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (0, nanoid_1.nanoid)()
+        });
+        /**
+         * The `BrowserPlugin` instance used to launch the browser.
+         */
+        Object.defineProperty(this, "browserPlugin", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Browser representation of the underlying automation library.
+         */
+        Object.defineProperty(this, "browser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        /**
+         * The configuration the browser was launched with.
+         */
+        Object.defineProperty(this, "launchContext", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "isActive", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "activePages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "totalPages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "lastPageOpenedAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Date.now()
+        });
+        Object.defineProperty(this, "_activate", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isActivePromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Promise((resolve) => {
+                this._activate = resolve;
+            })
+        });
+        Object.defineProperty(this, "commitBrowser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "hasBrowserPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Promise((resolve) => {
+                this.commitBrowser = resolve;
+            })
+        });
+        this.browserPlugin = browserPlugin;
+    }
+    /**
+     * Activates the BrowserController. If you try to open new pages before
+     * activation, the pages will get queued and will only be opened after
+     * activate is called.
+     * @ignore
+     */
+    activate() {
+        if (!this.browser) {
+            throw new Error('Cannot activate BrowserController without an assigned browser.');
+        }
+        this._activate();
+        this.isActive = true;
+    }
+    /**
+     * @ignore
+     */
+    assignBrowser(browser, launchContext) {
+        if (this.browser) {
+            throw new Error('BrowserController already has a browser instance assigned.');
+        }
+        this.browser = browser;
+        this.launchContext = launchContext;
+        this.commitBrowser();
+    }
+    /**
+     * Gracefully closes the browser and makes sure
+     * there will be no lingering browser processes.
+     *
+     * Emits 'browserClosed' event.
+     */
+    async close() {
+        await this.hasBrowserPromise;
+        try {
+            await this._close();
+            // TODO: shouldn't this go in a finally instead?
+            this.isActive = false;
+        }
+        catch (error) {
+            logger_1.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
+        }
+        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
+        setTimeout(() => {
+            this._kill().catch((err) => {
+                logger_1.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
+            });
+        }, PROCESS_KILL_TIMEOUT_MILLIS);
+    }
+    /**
+     * Immediately kills the browser process.
+     *
+     * Emits 'browserClosed' event.
+     */
+    async kill() {
+        await this.hasBrowserPromise;
+        await this._kill();
+        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
+    }
+    /**
+     * Opens new browser page.
+     * @ignore
+     */
+    async newPage(pageOptions) {
+        this.activePages++;
+        this.totalPages++;
+        await this.isActivePromise;
+        const page = await this._newPage(pageOptions);
+        (0, timeout_1.tryCancel)();
+        this.lastPageOpenedAt = Date.now();
+        return page;
+    }
+    async setCookies(page, cookies) {
+        return this._setCookies(page, cookies);
+    }
+    async getCookies(page) {
+        return this._getCookies(page);
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    // eslint-disable-next-line space-before-function-paren
+    async _close() {
+        (0, utils_1.throwImplementationNeeded)('_close');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    // eslint-disable-next-line space-before-function-paren
+    async _kill() {
+        (0, utils_1.throwImplementationNeeded)('_kill');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    async _newPage(pageOptions) {
+        (0, utils_1.throwImplementationNeeded)('_newPage');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    async _setCookies(page, cookies) {
+        (0, utils_1.throwImplementationNeeded)('_setCookies');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    async _getCookies(page) {
+        (0, utils_1.throwImplementationNeeded)('_getCookies');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    normalizeProxyOptions(proxyUrl, pageOptions) {
+        (0, utils_1.throwImplementationNeeded)('_normalizeProxyOptions');
+    }
+}
+exports.BrowserController = BrowserController;
+//# sourceMappingURL=browser-controller.js.map

package/abstract-classes/browser-controller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-controller.js","sourceRoot":"","sources":["../../src/abstract-classes/browser-controller.ts"],"names":[],"mappings":";;;AAAA,mCAAgC;AAChC,2DAAkD;AAClD,4CAA2C;AAG3C,sCAAgC;AAGhC,mCAAoD;AAEpD,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAuEzC;;;;;;;GAOG;AACH,MAAsB,iBAMpB,SAAQ,iCAA2G;IAsCjH,YAAY,aAAkG;QAC1G,KAAK,EAAE,CAAC;QAtCZ;;;;mBAAK,IAAA,eAAM,GAAE;WAAC;QAEd;;WAEG;QACH;;;;;WAAmG;QAEnG;;WAEG;QACH;;;;mBAAwB,SAAU;WAAC;QAEnC;;WAEG;QACH;;;;mBAAqG,SAAU;WAAC;QAEhH;;;;mBAAW,KAAK;WAAC;QAEjB;;;;mBAAc,CAAC;WAAC;QAEhB;;;;mBAAa,CAAC;WAAC;QAEf;;;;mBAAmB,IAAI,CAAC,GAAG,EAAE;WAAC;QAE9B;;;;;WAA+B;QAE/B;;;;mBAA0B,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACpD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC;YAC7B,CAAC,CAAC;WAAC;QAEH;;;;;WAAmC;QAEnC;;;;mBAA4B,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACtD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;YACjC,CAAC,CAAC;WAAC;QAIC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,QAAQ;QACJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACf,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;SACrF;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,OAAqB,EAAE,aAAkG;QACnI,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;SACjF;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAE7B,IAAI;YACA,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,gDAAgD;YAChD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;SACzB;QAAC,OAAO,KAAK,EAAE;YACZ,YAAG,CAAC,KAAK,CAAC,oCAAqC,KAAe,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;SAC9F;QAED,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;QAE1D,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACvB,YAAG,CAAC,KAAK,CAAC,mCAAmC,GAAG,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC7B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,iEAA2C,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,WAA4B;QACtC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,eAAe,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB,EAAE,OAAiB;QACnD,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAmB;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,8DAA8D;IAC9D,uDAAuD;IACpC,KAAK,CAAC,MAAM;QAC3B,IAAA,iCAAyB,EAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,8DAA8D;IAC9D,uDAAuD;IACpC,KAAK,CAAC,KAAK;QAC1B,IAAA,iCAAyB,EAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,8DAA8D;IAC3C,KAAK,CAAC,QAAQ,CAAC,WAA4B;QAC1D,IAAA,iCAAyB,EAAC,UAAU,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,8DAA8D;IAC3C,KAAK,CAAC,WAAW,CAAC,IAAmB,EAAE,OAAiB;QACvE,IAAA,iCAAyB,EAAC,aAAa,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,8DAA8D;IAC3C,KAAK,CAAC,WAAW,CAAC,IAAmB;QACpD,IAAA,iCAAyB,EAAC,aAAa,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,8DAA8D;IACrD,qBAAqB,CAAC,QAA4B,EAAE,WAAgB;QACzE,IAAA,iCAAyB,EAAC,wBAAwB,CAAC,CAAC;IACxD,CAAC;CACJ;AAxLD,8CAwLC"}

package/abstract-classes/browser-plugin.d.ts

@@ -0,0 +1,102 @@
+import { LaunchContext, LaunchContextOptions } from '../launch-context';
+import { BrowserController } from './browser-controller';
+import { UnwrapPromise } from '../utils';
+/**
+ * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
+ * when Chromium/Chrome browser is launched:
+ *  - in headless mode,
+ *  - without using a fingerprint,
+ *  - without specifying a user agent.
+ * Last updated on 2022-05-05.
+ */
+export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36";
+/**
+ * Each plugin expects an instance of the object with the `.launch()` property.
+ * For Puppeteer, it is the `puppeteer` module itself, whereas for Playwright
+ * it is one of the browser types, such as `puppeteer.chromium`.
+ * `BrowserPlugin` does not include the library. You can choose any version
+ * or fork of the library. It also keeps `browser-pool` installation small.
+ */
+export interface CommonLibrary {
+    product?: string;
+    launch(...args: unknown[]): Promise<CommonBrowser>;
+    name?: () => string;
+}
+/** @internal */
+export interface CommonBrowser {
+    newPage(...args: unknown[]): Promise<CommonPage>;
+}
+/** @internal */
+export interface CommonPage {
+    close(...args: unknown[]): Promise<unknown>;
+    url(): string | Promise<string>;
+}
+export interface BrowserPluginOptions<LibraryOptions> {
+    /**
+     * Options that will be passed down to the automation library. E.g.
+     * `puppeteer.launch(launchOptions);`. This is a good place to set
+     * options that you want to apply as defaults. To dynamically override
+     * those options per-browser, see the `preLaunchHooks` of {@link BrowserPool}.
+     */
+    launchOptions?: LibraryOptions;
+    /**
+     * Automation libraries configure proxies differently. This helper allows you
+     * to set a proxy URL without worrying about specific implementations.
+     * It also allows you use an authenticated proxy without extra code.
+     */
+    proxyUrl?: string;
+    /**
+     * By default pages share the same browser context.
+     * If set to true each page uses its own context that is destroyed once the page is closed or crashes.
+     *
+     * @default false
+     */
+    useIncognitoPages?: boolean;
+    /**
+     * Path to a User Data Directory, which stores browser session data like cookies and local storage.
+     */
+    userDataDir?: string;
+}
+export declare type CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> = Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>>;
+/**
+ * The `BrowserPlugin` serves two purposes. First, it is the base class that
+ * specialized controllers like `PuppeteerPlugin` or `PlaywrightPlugin` extend.
+ * Second, it allows the user to configure the automation libraries and
+ * feed them to {@link BrowserPool} for use.
+ */
+export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+    name: string;
+    library: Library;
+    launchOptions: LibraryOptions;
+    proxyUrl?: string;
+    userDataDir?: string;
+    useIncognitoPages: boolean;
+    constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
+    /**
+     * Creates a `LaunchContext` with all the information needed
+     * to launch a browser. Aside from library specific launch options,
+     * it also includes internal properties used by `BrowserPool` for
+     * management of the pool and extra features.
+     */
+    createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+    createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+    /**
+     * Launches the browser using provided launch context.
+     */
+    launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
+    private _mergeArgsToHideWebdriver;
+    /**
+     * @private
+     */
+    protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
+    protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
+    /**
+     * @private
+     */
+    protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
+    /**
+     * @private
+     */
+    protected abstract _createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+}
+//# sourceMappingURL=browser-plugin.d.ts.map

package/abstract-classes/browser-plugin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-plugin.d.ts","sourceRoot":"","sources":["../../src/abstract-classes/browser-plugin.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,8GAA8G,CAAC;AAE9I;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACnD,IAAI,CAAC,EAAE,MAAM,MAAM,CAAC;CACvB;AAED,gBAAgB;AAChB,MAAM,WAAW,aAAa;IAC1B,OAAO,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACpD;AAED,gBAAgB;AAChB,MAAM,WAAW,UAAU;IACvB,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,GAAG,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB,CAAC,cAAc;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,cAAc,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,oBAAY,0BAA0B,CAClC,OAAO,SAAS,aAAa,EAC7B,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EACjD,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,IAClE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;AAE/H;;;;;GAKG;AACH,8BAAsB,aAAa,CAC/B,OAAO,SAAS,aAAa,GAAG,aAAa,EAC7C,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EACjD,YAAY,SAAS,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EACjF,cAAc,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,aAAa,GAAG,aAAa,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAElE,IAAI,SAAyB;IAE7B,OAAO,EAAE,OAAO,CAAC;IAEjB,aAAa,EAAE,cAAc,CAAC;IAE9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,iBAAiB,EAAE,OAAO,CAAC;gBAEf,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,oBAAoB,CAAC,cAAc,CAAM;IAehF;;;;;OAKG;IACH,mBAAmB,CACf,OAAO,GAAE,0BAA0B,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAM,GAC/G,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAmBtF,gBAAgB,IAAI,iBAAiB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;IAI3G;;OAEG;IACG,MAAM,CACR,aAAa,GAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAA8B,GAChI,OAAO,CAAC,YAAY,CAAC;IAsBxB,OAAO,CAAC,yBAAyB;IAgBjC;;OAEG;IAGH,SAAS,CAAC,QAAQ,CAAC,wBAAwB,CAAC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9J,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,OAAO;IAIvJ;;OAEG;IAEH,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAIrJ;;OAEG;IAGH,SAAS,CAAC,QAAQ,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,CAAC;CAGlI"}

package/abstract-classes/browser-plugin.js

@@ -0,0 +1,154 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BrowserPlugin = exports.DEFAULT_USER_AGENT = void 0;
+const tslib_1 = require("tslib");
+const lodash_merge_1 = tslib_1.__importDefault(require("lodash.merge"));
+const launch_context_1 = require("../launch-context");
+const utils_1 = require("./utils");
+/**
+ * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
+ * when Chromium/Chrome browser is launched:
+ *  - in headless mode,
+ *  - without using a fingerprint,
+ *  - without specifying a user agent.
+ * Last updated on 2022-05-05.
+ */
+exports.DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36';
+;
+/**
+ * The `BrowserPlugin` serves two purposes. First, it is the base class that
+ * specialized controllers like `PuppeteerPlugin` or `PlaywrightPlugin` extend.
+ * Second, it allows the user to configure the automation libraries and
+ * feed them to {@link BrowserPool} for use.
+ */
+class BrowserPlugin {
+    constructor(library, options = {}) {
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+        Object.defineProperty(this, "library", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "launchOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "proxyUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userDataDir", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "useIncognitoPages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, } = options;
+        this.library = library;
+        this.launchOptions = launchOptions;
+        this.proxyUrl = proxyUrl && new URL(proxyUrl).href;
+        this.userDataDir = userDataDir;
+        this.useIncognitoPages = useIncognitoPages;
+    }
+    /**
+     * Creates a `LaunchContext` with all the information needed
+     * to launch a browser. Aside from library specific launch options,
+     * it also includes internal properties used by `BrowserPool` for
+     * management of the pool and extra features.
+     */
+    createLaunchContext(options = {}) {
+        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, } = options;
+        return new launch_context_1.LaunchContext({
+            id,
+            launchOptions: (0, lodash_merge_1.default)({}, this.launchOptions, launchOptions),
+            browserPlugin: this,
+            proxyUrl,
+            useIncognitoPages,
+            userDataDir,
+        });
+    }
+    createController() {
+        return this._createController();
+    }
+    /**
+     * Launches the browser using provided launch context.
+     */
+    async launch(launchContext = this.createLaunchContext()) {
+        const { proxyUrl, launchOptions } = launchContext;
+        if (proxyUrl) {
+            await this._addProxyToLaunchOptions(launchContext);
+        }
+        if (this._isChromiumBasedBrowser(launchContext)) {
+            // This will set the args for chromium based browsers to hide the webdriver.
+            launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
+            // When User-Agent is not set, and we're using Chromium in headless mode,
+            // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
+            // as otherwise 'HeadlessChrome' is present in User-Agent string.
+            const userAgent = launchOptions.args.find((arg) => arg.startsWith('--user-agent'));
+            if (launchOptions.headless && !launchContext.fingerprint && !userAgent) {
+                launchOptions.args.push(`--user-agent=${exports.DEFAULT_USER_AGENT}`);
+            }
+        }
+        return this._launch(launchContext);
+    }
+    _mergeArgsToHideWebdriver(originalArgs) {
+        if (!originalArgs?.length) {
+            return ['--disable-blink-features=AutomationControlled'];
+        }
+        const argumentIndex = originalArgs.findIndex((arg) => arg.startsWith('--disable-blink-features='));
+        if (argumentIndex !== -1) {
+            originalArgs[argumentIndex] += ',AutomationControlled';
+        }
+        else {
+            originalArgs.push('--disable-blink-features=AutomationControlled');
+        }
+        return originalArgs;
+    }
+    ;
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    // eslint-disable-next-line max-len
+    _addProxyToLaunchOptions(launchContext) {
+        (0, utils_1.throwImplementationNeeded)('_addProxyToLaunchOptions');
+    }
+    // @ts-expect-error Give runtime error as well as compile time
+    // eslint-disable-next-line space-before-function-paren, @typescript-eslint/no-unused-vars, max-len
+    _isChromiumBasedBrowser(launchContext) {
+        (0, utils_1.throwImplementationNeeded)('_isChromiumBasedBrowser');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    _launch(launchContext) {
+        (0, utils_1.throwImplementationNeeded)('_launch');
+    }
+    /**
+     * @private
+     */
+    // @ts-expect-error Give runtime error as well as compile time
+    // eslint-disable-next-line space-before-function-paren
+    _createController() {
+        (0, utils_1.throwImplementationNeeded)('_createController');
+    }
+}
+exports.BrowserPlugin = BrowserPlugin;
+//# sourceMappingURL=browser-plugin.js.map