@crawlee/browser-pool 4.0.0-beta.19 → 4.0.0-beta.191

package/README.md

@@ -1,23 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/abstract-classes/browser-controller.d.ts

@@ -1,9 +1,73 @@
+import { type CrawleeLogger } from '@crawlee/core';
 import type { Cookie, Dictionary } from '@crawlee/types';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 import type { LaunchContext } from '../launch-context.js';
 import type { UnwrapPromise } from '../utils.js';
 import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
+/**
+ * The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
+ * Other fields are only available on the concrete `LaunchContext` class.
+ */
+export interface IBrowserLaunchContext {
+    /**
+     * The proxy URL the browser was launched with, if any.
+     */
+    proxyUrl?: string;
+    /**
+     * The fingerprint applied to the browser, if fingerprinting is enabled.
+     * Typed as `unknown` here; cast to the concrete `LaunchContext` if you
+     * need the structured shape.
+     */
+    fingerprint?: unknown;
+    /**
+     * `true` if each page in this browser uses its own context.
+     */
+    useIncognitoPages?: boolean;
+    /**
+     * The actual options the browser was launched with, after pre-launch hooks.
+     */
+    launchOptions?: Dictionary | undefined;
+}
+/**
+ * The minimal public contract of a browser controller.
+ *
+ * Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
+ * promises, …) is intentionally **not** part of this contract.
+ *
+ * @category Browser management
+ */
+export interface IBrowserController<Page = unknown> {
+    /**
+     * A stable identifier for this controller instance. Useful for tracking
+     * which browser served which request.
+     */
+    readonly id: string;
+    /**
+     * The configuration the underlying browser was launched with — proxy URL,
+     * fingerprint, session, launcher-specific options, etc.
+     */
+    readonly launchContext: IBrowserLaunchContext;
+    /**
+     * The raw browser handle from the underlying automation library
+     * (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
+     * Escape hatch for things the controller does not expose directly.
+     */
+    readonly browser: unknown;
+    /**
+     * Reads cookies for the given page.
+     */
+    getCookies(page: Page): Promise<Cookie[]>;
+    /**
+     * Writes cookies for the given page.
+     */
+    setCookies(page: Page, cookies: Cookie[]): Promise<void>;
+    /**
+     * Gracefully closes the browser this controller owns. After this resolves,
+     * the controller is no longer usable.
+     */
+    close(): Promise<void>;
+}
 export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
     [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
 }
@@ -15,12 +79,14 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
  * for the specialized classes, because it's the same for all of them.
  * @hideconstructor
  */
-export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
-    id: string;
+export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
+    #private;
+    readonly id: string;
+    protected readonly log: CrawleeLogger;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
      */
-    browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+    readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
     /**
      * Browser representation of the underlying automation library.
      */
@@ -30,12 +96,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      */
     launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
     /**
-     * The proxy tier tied to this browser controller.
-     * `undefined` if no tiered proxy is used.
-     */
-    proxyTier?: number;
-    /**
-     * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
+     * The proxy URL used by the browser controller.
      * `undefined` if no proxy is used
      */
     proxyUrl?: string;
@@ -43,10 +104,10 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
     activePages: number;
     totalPages: number;
     lastPageOpenedAt: number;
-    private _activate;
-    private isActivePromise;
-    private commitBrowser;
-    private hasBrowserPromise;
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    waitForActive(): Promise<void>;
     constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
     /**
      * Activates the BrowserController. If you try to open new pages before
@@ -77,6 +138,23 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page: NewPageResult): void;
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
     setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
     getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**
@@ -104,4 +182,3 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      */
     abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
 }
-//# sourceMappingURL=browser-controller.d.ts.map

package/abstract-classes/browser-controller.js

@@ -1,7 +1,8 @@
+import { serviceLocator } from '@crawlee/core';
 import { nanoid } from 'nanoid';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import { tryCancel } from '@apify/timeout';
-import { log } from '../logger.js';
+import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
 /**
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +14,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
  */
 export class BrowserController extends TypedEmitter {
     id = nanoid();
+    log;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
      */
@@ -26,29 +28,33 @@ export class BrowserController extends TypedEmitter {
      */
     launchContext = undefined;
     /**
-     * The proxy tier tied to this browser controller.
-     * `undefined` if no tiered proxy is used.
-     */
-    proxyTier;
-    /**
-     * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
+     * The proxy URL used by the browser controller.
      * `undefined` if no proxy is used
      */
     proxyUrl;
     isActive = false;
     activePages = 0;
+    #closedPages = new WeakSet();
+    #pageTeardowns = new WeakMap();
     totalPages = 0;
     lastPageOpenedAt = Date.now();
-    _activate;
-    isActivePromise = new Promise((resolve) => {
-        this._activate = resolve;
+    #activate;
+    #isActivePromise = new Promise((resolve) => {
+        this.#activate = resolve;
     });
-    commitBrowser;
-    hasBrowserPromise = new Promise((resolve) => {
-        this.commitBrowser = resolve;
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    async waitForActive() {
+        await this.#isActivePromise;
+    }
+    #commitBrowser;
+    #hasBrowserPromise = new Promise((resolve) => {
+        this.#commitBrowser = resolve;
     });
     constructor(browserPlugin) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.browserPlugin = browserPlugin;
     }
     /**
@@ -61,7 +67,7 @@ export class BrowserController extends TypedEmitter {
         if (!this.browser) {
             throw new Error('Cannot activate BrowserController without an assigned browser.');
         }
-        this._activate();
+        this.#activate();
         this.isActive = true;
     }
     /**
@@ -73,7 +79,7 @@ export class BrowserController extends TypedEmitter {
         }
         this.browser = browser;
         this.launchContext = launchContext;
-        this.commitBrowser();
+        this.#commitBrowser();
     }
     /**
      * Gracefully closes the browser and makes sure
@@ -82,21 +88,22 @@ export class BrowserController extends TypedEmitter {
      * Emits 'browserClosed' event.
      */
     async close() {
-        await this.hasBrowserPromise;
+        await this.#hasBrowserPromise;
         try {
             await this._close();
             // TODO: shouldn't this go in a finally instead?
             this.isActive = false;
         }
         catch (error) {
-            log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
+            this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
         }
-        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
-        setTimeout(() => {
+        this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
+        const killTimer = setTimeout(() => {
             this._kill().catch((err) => {
-                log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
+                this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
             });
         }, PROCESS_KILL_TIMEOUT_MILLIS);
+        killTimer.unref();
     }
     /**
      * Immediately kills the browser process.
@@ -104,9 +111,9 @@ export class BrowserController extends TypedEmitter {
      * Emits 'browserClosed' event.
      */
     async kill() {
-        await this.hasBrowserPromise;
+        await this.#hasBrowserPromise;
         await this._kill();
-        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
+        this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
     }
     /**
      * Opens new browser page.
@@ -115,12 +122,43 @@ export class BrowserController extends TypedEmitter {
     async newPage(pageOptions) {
         this.activePages++;
         this.totalPages++;
-        await this.isActivePromise;
+        await this.#isActivePromise;
         const page = await this._newPage(pageOptions);
         tryCancel();
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page) {
+        if (this.#closedPages.has(page))
+            return;
+        this.#closedPages.add(page);
+        this.activePages--;
+        const teardown = this.#pageTeardowns.get(page);
+        if (!teardown)
+            return;
+        this.#pageTeardowns.delete(page);
+        teardown().catch((error) => {
+            this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
+        });
+    }
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    registerPageTeardown(page, teardown) {
+        this.#pageTeardowns.set(page, teardown);
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }
@@ -128,4 +166,3 @@ export class BrowserController extends TypedEmitter {
         return this._getCookies(page);
     }
 }
-//# sourceMappingURL=browser-controller.js.map

package/abstract-classes/browser-plugin.d.ts

@@ -1,8 +1,9 @@
-import { CriticalError } from '@crawlee/core';
+import { type CrawleeLogger, CriticalError } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import type { LaunchContextOptions } from '../launch-context.js';
 import { LaunchContext } from '../launch-context.js';
-import type { UnwrapPromise } from '../utils.js';
+import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
+import { type UnwrapPromise } from '../utils.js';
 import type { BrowserController } from './browser-controller.js';
 /**
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
@@ -11,8 +12,6 @@ import type { BrowserController } from './browser-controller.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
 /**
@@ -27,14 +26,13 @@ export interface CommonLibrary {
     launch(opts?: Dictionary): Promise<CommonBrowser>;
     name?: () => string;
 }
-/** @internal */
 export interface CommonBrowser {
     newPage(...args: unknown[]): Promise<CommonPage>;
 }
-/** @internal */
 export interface CommonPage {
     close(...args: unknown[]): Promise<unknown>;
     url(): string | Promise<string>;
+    evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
 }
 export interface BrowserPluginOptions<LibraryOptions> {
     /**
@@ -68,6 +66,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
      * Might cause performance issues, as Crawlee might launch too many browser instances.
      */
     browserPerProxy?: boolean;
+    /**
+     * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+     * This is useful when using HTTPS proxies with self-signed certificates.
+     */
+    ignoreProxyCertificate?: boolean;
 }
 export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
 }
@@ -78,14 +81,43 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
  * feed them to {@link BrowserPool} for use.
  */
 export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
-    name: string;
-    library: Library;
-    launchOptions: LibraryOptions;
-    proxyUrl?: string;
-    userDataDir?: string;
+    readonly name: string;
+    protected readonly log: CrawleeLogger;
+    readonly library: Library;
+    readonly launchOptions: LibraryOptions;
+    readonly proxyUrl?: string;
+    readonly userDataDir?: string;
     useIncognitoPages: boolean;
-    browserPerProxy?: boolean;
+    readonly browserPerProxy?: boolean;
+    readonly ignoreProxyCertificate?: boolean;
+    /**
+     * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
+     * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
+     * remote-session policy lives in the pool, not here.
+     *
+     * @internal
+     */
+    remoteConnection?: RemoteConnection;
+    /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
+    remoteConnectionParameters?: RemoteConnectionParameters;
     constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
+    /**
+     * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
+     * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
+     * (e.g. forcing incognito pages).
+     *
+     * @internal
+     */
+    useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
+    /**
+     * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
+     * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
+     * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
+     *
+     * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
+     * lives here so it stays identical across plugins.
+     */
+    protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
     /**
      * Creates a `LaunchContext` with all the information needed
      * to launch a browser. Aside from library specific launch options,
@@ -98,13 +130,13 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
      * Launches the browser using provided launch context.
      */
     launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
-    private _mergeArgsToHideWebdriver;
-    protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
+    private mergeArgsToHideWebdriver;
+    protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
     /**
      * @private
      */
-    protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
-    protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
+    protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
+    protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
     /**
      * @private
      */
@@ -113,4 +145,3 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
 export declare class BrowserLaunchError extends CriticalError {
     constructor(...args: ConstructorParameters<typeof CriticalError>);
 }
-//# sourceMappingURL=browser-plugin.d.ts.map

package/abstract-classes/browser-plugin.js

@@ -1,6 +1,7 @@
-import { CriticalError } from '@crawlee/core';
+import { CriticalError, serviceLocator } from '@crawlee/core';
 import merge from 'lodash.merge';
 import { LaunchContext } from '../launch-context.js';
+import { sanitizeEndpointForLog } from '../utils.js';
 /**
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
  * when Chromium/Chrome browser is launched:
@@ -8,8 +9,6 @@ import { LaunchContext } from '../launch-context.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
 /**
@@ -20,20 +19,73 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
  */
 export class BrowserPlugin {
     name = this.constructor.name;
+    log;
     library;
     launchOptions;
     proxyUrl;
     userDataDir;
     useIncognitoPages;
     browserPerProxy;
+    ignoreProxyCertificate;
+    /**
+     * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
+     * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
+     * remote-session policy lives in the pool, not here.
+     *
+     * @internal
+     */
+    remoteConnection;
+    /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
+    remoteConnectionParameters;
     constructor(library, options = {}) {
-        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
+        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.library = library;
         this.launchOptions = launchOptions;
         this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
         this.userDataDir = userDataDir;
         this.useIncognitoPages = useIncognitoPages;
         this.browserPerProxy = browserPerProxy;
+        this.ignoreProxyCertificate = ignoreProxyCertificate;
+    }
+    /**
+     * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
+     * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
+     * (e.g. forcing incognito pages).
+     *
+     * @internal
+     */
+    useRemoteConnection(connection, parameters = {}) {
+        this.remoteConnection = connection;
+        this.remoteConnectionParameters = parameters;
+    }
+    /**
+     * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
+     * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
+     * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
+     *
+     * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
+     * lives here so it stays identical across plugins.
+     */
+    async connectToRemoteBrowser(launchContext, connect) {
+        const connection = this.remoteConnection;
+        let url;
+        let token;
+        try {
+            ({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
+        }
+        catch (cause) {
+            throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
+        }
+        launchContext.remoteToken = token;
+        try {
+            return await connect(url);
+        }
+        catch (cause) {
+            await connection.release(token);
+            throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
+                'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
+        }
     }
     /**
      * Creates a `LaunchContext` with all the information needed
@@ -42,7 +94,7 @@ export class BrowserPlugin {
      * management of the pool and extra features.
      */
     createLaunchContext(options = {}) {
-        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
+        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote = !!this.remoteConnection, } = options;
         return new LaunchContext({
             id,
             launchOptions: merge({}, this.launchOptions, launchOptions),
@@ -51,21 +103,23 @@ export class BrowserPlugin {
             useIncognitoPages,
             userDataDir,
             browserPerProxy,
-            proxyTier,
+            ignoreProxyCertificate,
+            isRemote,
         });
     }
     /**
      * Launches the browser using provided launch context.
      */
     async launch(launchContext = this.createLaunchContext()) {
+        // launchOptions is only used by the local launch path below — remote connections ignore it.
         launchContext.launchOptions ??= {};
         const { proxyUrl, launchOptions } = launchContext;
-        if (proxyUrl) {
-            await this._addProxyToLaunchOptions(launchContext);
+        if (proxyUrl && !launchContext.isRemote) {
+            await this.addProxyToLaunchOptions(launchContext);
         }
-        if (this._isChromiumBasedBrowser(launchContext)) {
+        if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
             // This will set the args for chromium based browsers to hide the webdriver.
-            launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
+            launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
             // When User-Agent is not set, and we're using Chromium in headless mode,
             // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
             // as otherwise 'HeadlessChrome' is present in User-Agent string.
@@ -74,9 +128,12 @@ export class BrowserPlugin {
                 launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
             }
         }
+        if (launchContext.isRemote) {
+            this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
+        }
         return this._launch(launchContext);
     }
-    _mergeArgsToHideWebdriver(originalArgs) {
+    mergeArgsToHideWebdriver(originalArgs) {
         if (!originalArgs?.length) {
             return ['--disable-blink-features=AutomationControlled'];
         }
@@ -89,7 +146,7 @@ export class BrowserPlugin {
         }
         return originalArgs;
     }
-    _throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
+    throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
         const errorMessage = ['Failed to launch browser. Please check the following:'];
         if (executablePath) {
             errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
@@ -118,4 +175,3 @@ export class BrowserLaunchError extends CriticalError {
         });
     }
 }
-//# sourceMappingURL=browser-plugin.js.map