@crawlee/browser-pool 4.0.0-beta.2 → 4.0.0-beta.200

package/abstract-classes/browser-plugin.js

@@ -1,6 +1,7 @@
-import { CriticalError } from '@crawlee/core';
+import { CriticalError, serviceLocator } from '@crawlee/core';
 import merge from 'lodash.merge';
 import { LaunchContext } from '../launch-context.js';
+import { sanitizeEndpointForLog } from '../utils.js';
 /**
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
  * when Chromium/Chrome browser is launched:
@@ -8,8 +9,6 @@ import { LaunchContext } from '../launch-context.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
 /**
@@ -20,20 +19,79 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
  */
 export class BrowserPlugin {
     name = this.constructor.name;
+    /**
+     * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
+     * classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
+    log;
     library;
     launchOptions;
     proxyUrl;
     userDataDir;
     useIncognitoPages;
     browserPerProxy;
+    ignoreProxyCertificate;
+    /**
+     * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
+     * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
+     * remote-session policy lives in the pool, not here.
+     *
+     * @internal
+     */
+    remoteConnection;
+    /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
+    remoteConnectionParameters;
     constructor(library, options = {}) {
-        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
+        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.library = library;
         this.launchOptions = launchOptions;
         this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
         this.userDataDir = userDataDir;
         this.useIncognitoPages = useIncognitoPages;
         this.browserPerProxy = browserPerProxy;
+        this.ignoreProxyCertificate = ignoreProxyCertificate;
+    }
+    /**
+     * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
+     * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
+     * (e.g. forcing incognito pages).
+     *
+     * @internal
+     */
+    useRemoteConnection(connection, parameters = {}) {
+        this.remoteConnection = connection;
+        this.remoteConnectionParameters = parameters;
+    }
+    /**
+     * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
+     * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
+     * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
+     *
+     * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
+     * lives here so it stays identical across plugins.
+     */
+    async connectToRemoteBrowser(launchContext, connect) {
+        const connection = this.remoteConnection;
+        let url;
+        let token;
+        try {
+            ({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
+        }
+        catch (cause) {
+            throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
+        }
+        launchContext.remoteToken = token;
+        try {
+            return await connect(url);
+        }
+        catch (cause) {
+            await connection.release(token);
+            throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
+                'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
+        }
     }
     /**
      * Creates a `LaunchContext` with all the information needed
@@ -42,7 +100,7 @@ export class BrowserPlugin {
      * management of the pool and extra features.
      */
     createLaunchContext(options = {}) {
-        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
+        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote = !!this.remoteConnection, } = options;
         return new LaunchContext({
             id,
             launchOptions: merge({}, this.launchOptions, launchOptions),
@@ -51,21 +109,23 @@ export class BrowserPlugin {
             useIncognitoPages,
             userDataDir,
             browserPerProxy,
-            proxyTier,
+            ignoreProxyCertificate,
+            isRemote,
         });
     }
     /**
      * Launches the browser using provided launch context.
      */
     async launch(launchContext = this.createLaunchContext()) {
+        // launchOptions is only used by the local launch path below — remote connections ignore it.
         launchContext.launchOptions ??= {};
         const { proxyUrl, launchOptions } = launchContext;
-        if (proxyUrl) {
-            await this._addProxyToLaunchOptions(launchContext);
+        if (proxyUrl && !launchContext.isRemote) {
+            await this.addProxyToLaunchOptions(launchContext);
         }
-        if (this._isChromiumBasedBrowser(launchContext)) {
+        if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
             // This will set the args for chromium based browsers to hide the webdriver.
-            launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
+            launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
             // When User-Agent is not set, and we're using Chromium in headless mode,
             // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
             // as otherwise 'HeadlessChrome' is present in User-Agent string.
@@ -74,9 +134,12 @@ export class BrowserPlugin {
                 launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
             }
         }
+        if (launchContext.isRemote) {
+            this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
+        }
         return this._launch(launchContext);
     }
-    _mergeArgsToHideWebdriver(originalArgs) {
+    mergeArgsToHideWebdriver(originalArgs) {
         if (!originalArgs?.length) {
             return ['--disable-blink-features=AutomationControlled'];
         }
@@ -89,7 +152,7 @@ export class BrowserPlugin {
         }
         return originalArgs;
     }
-    _throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
+    throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
         const errorMessage = ['Failed to launch browser. Please check the following:'];
         if (executablePath) {
             errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);
@@ -118,4 +181,3 @@ export class BrowserLaunchError extends CriticalError {
         });
     }
 }
-//# sourceMappingURL=browser-plugin.js.map

package/anonymize-proxy.d.ts

@@ -1,4 +1,8 @@
 type PromiseVoid = () => Promise<void>;
-export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string) => Promise<[string | undefined, PromiseVoid]>;
+/** @internal */
+export interface AnonymizeProxySugarOptions {
+    ignoreProxyCertificate?: boolean;
+}
+/** @internal */
+export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
 export {};
-//# sourceMappingURL=anonymize-proxy.d.ts.map

package/anonymize-proxy.js

@@ -1,15 +1,20 @@
-import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
-export const anonymizeProxySugar = async (proxyUrl, username, password) => {
+/** @internal */
+export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
     if (proxyUrl) {
         const url = new URL(proxyUrl);
         if (username || password) {
             url.username = username ?? '';
             url.password = password ?? '';
         }
-        if (url.username || url.password) {
+        if (url.username || url.password || options?.ignoreProxyCertificate) {
             // trim off trailing slash if it's present
             const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
-            const anonymized = await anonymizeProxy(proxyUrlString);
+            const { anonymizeProxy, closeAnonymizedProxy } = await import('proxy-chain');
+            const anonymized = await anonymizeProxy({
+                url: proxyUrlString,
+                port: 0,
+                ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
+            });
             return [
                 anonymized,
                 async () => {
@@ -21,4 +26,3 @@ export const anonymizeProxySugar = async (proxyUrl, username, password) => {
     }
     return [undefined, async () => { }];
 };
-//# sourceMappingURL=anonymize-proxy.js.map

package/browser-pool.d.ts

@@ -1,3 +1,4 @@
+import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
@@ -246,37 +247,39 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
  * });
  * ```
  */
-export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
+export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
+    #private;
     browserPlugins: BrowserPlugins;
-    maxOpenPagesPerBrowser: number;
-    retireBrowserAfterPageCount: number;
-    operationTimeoutMillis: number;
-    closeInactiveBrowserAfterMillis: number;
-    useFingerprints?: boolean;
+    /** @internal */
+    maxOpenBrowsers: number;
+    /** @internal */
     fingerprintOptions: FingerprintOptions;
-    preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
-    postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
-    prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
-    postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
-    prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
-    postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
-    pageCounter: number;
-    pages: Map<string, PageReturn>;
-    pageIds: WeakMap<PageReturn, string>;
-    activeBrowserControllers: Set<BrowserControllerReturn>;
-    retiredBrowserControllers: Set<BrowserControllerReturn>;
-    pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
+    /** @internal */
     fingerprintInjector?: FingerprintInjector;
     fingerprintGenerator?: FingerprintGenerator;
+    /** @internal */
     fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
+    private pages;
+    private activeBrowserControllers;
+    private retiredBrowserControllers;
     private browserKillerInterval?;
-    private browserRetireInterval?;
-    private limiter;
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
     /**
      * Opens a new page in one of the running browsers or launches
      * a new browser and opens a page there, if no browsers are active,
      * or their page limits have been exceeded.
+     *
+     * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
+     * provided, this implementation uses it as a cache key for browser fingerprints (when
+     * fingerprinting is enabled) and reads
+     * {@link ProxyInfo.url|session.proxyInfo.url} /
+     * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
+     * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
+     * `ignoreTlsErrors` values in the options take precedence.
+     *
+     * Beyond fingerprint caching and proxy configuration, no other session
+     * properties are consumed — cookie and header injection remain the
+     * crawler's responsibility.
      */
     newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
     /**
@@ -334,7 +337,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * until it's closed.
      */
     getPageId(page: PageReturn): string | undefined;
-    private _createPageForBrowser;
+    private createPageForBrowser;
     /**
      * Removes a browser controller from the pool. The underlying
      * browser will be closed after all its pages are closed.
@@ -346,6 +349,43 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * closed after all its pages are closed.
      */
     retireBrowserByPage(page: PageReturn): void;
+    /**
+     * Releases a page back to the pool. The page is closed and, if the
+     * optional `error` is a {@link SessionError}, the browser controller
+     * that served the page is retired so that its tainted state (cookies,
+     * storage, etc.) cannot leak into future sessions.
+     *
+     * This is the primary way the crawler should return pages to the pool.
+     *
+     * @param page The page to release.
+     * @param options.error The error that caused the page to be released, if any.
+     */
+    closePage(page: PageReturn, options?: {
+        error?: Error;
+    }): Promise<void>;
+    /**
+     * Extracts the relevant state (currently just cookies) from a page via its
+     * owning {@link BrowserController}. Returns empty state when the page is
+     * no longer associated with a controller.
+     *
+     * As with {@link BrowserPool.injectPageState}, cookies are isolated per
+     * page only when the pool is configured with `useIncognitoPages: true`.
+     * With the default `useIncognitoPages: false`, the extracted cookies
+     * include those set by any sibling page sharing the same browser.
+     */
+    extractPageState(page: PageReturn): Promise<PageState>;
+    /**
+     * Injects state into a page via its owning {@link BrowserController}.
+     *
+     * No-op when the page is no longer associated with a controller.
+     *
+     * Note that cookies are isolated per page only when the pool is configured
+     * with `useIncognitoPages: true` — each page then gets its own browser
+     * context. With the default `useIncognitoPages: false`, all pages in a
+     * browser share a single context, so injected cookies are visible to every
+     * page served by that browser.
+     */
+    injectPageState(page: PageReturn, state: PageState): Promise<void>;
     /**
      * Removes all active browsers from the pool. The browsers will be
      * closed after all their pages are closed.
@@ -356,32 +396,75 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * @return {Promise<void>}
      */
     closeAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Closes all managed browsers and tears down the pool.
+     * Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
+     * browsers when a run ends and may start another run on the same pool.
+     */
+    releaseAllBrowsers(): Promise<void>;
+    /**
+     * Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
+     * re-armed and its listeners are dropped, so it cannot be used again.
      */
     destroy(): Promise<void>;
-    private _teardown;
-    private _getAllBrowserControllers;
-    private _launchBrowser;
+    private getAllBrowserControllers;
+    private launchBrowser;
     /**
      * Picks plugins round robin.
-     * @private
-     */
-    private _pickBrowserPlugin;
-    private _pickBrowserWithFreeCapacity;
-    private _closeInactiveRetiredBrowsers;
-    private _overridePageClose;
-    private _executeHooks;
-    private _closeRetiredBrowserWithNoPages;
-    private _initializeFingerprinting;
-    private _addFingerprintHooks;
+     */
+    private pickBrowserPlugin;
+    private pickBrowserWithFreeCapacity;
+    private closeInactiveRetiredBrowsers;
+    private overridePageClose;
+    private executeHooks;
+    private closeRetiredBrowserWithNoPages;
+    /**
+     * Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
+     * Counts starting, active, and retired browsers.
+     *
+     * A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
+     * has set a cap — {@link RemoteBrowserPool} does, from its own
+     * {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
+     * `BrowserPoolOptions` key for it.
+     *
+     * @internal
+     */
+    hasFreeBrowserSlot(): boolean;
+    /**
+     * Returns `true` if any active browser has room for another page.
+     *
+     * @internal
+     */
+    hasActiveBrowserWithFreeCapacity(): boolean;
+    private initializeFingerprinting;
 }
-export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> {
+export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
+    /**
+     * The proxy URL the pool uses internally to route the page: it keys browser
+     * reuse (with `browserPerProxy`, only a browser already on this proxy is
+     * reused), configures the launched browser, and is applied to incognito
+     * pages. When omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool`'s proxy
+     * handling and is intentionally not part of the {@link IBrowserPool}
+     * contract — through that interface the proxy is supplied via the session.
+     */
+    proxyUrl?: string;
     /**
-     * Assign a custom ID to the page. If you don't a random string ID
-     * will be generated.
+     * Disable TLS certificate verification for MITM proxies. Applied both when
+     * launching a new browser and when creating a page in an existing one. When
+     * omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool` and is
+     * intentionally not part of the {@link IBrowserPool} contract — through
+     * that interface, configure it via the session's `proxyInfo` or through the
+     * browser's `launchOptions`.
      */
-    id?: string;
+    ignoreTlsErrors?: boolean;
     /**
      * Some libraries (Playwright) allow you to open new pages with specific
      * options. Use this property to set those options.
@@ -396,14 +479,6 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
      * see the `newPageInNewBrowser` function.
      */
     browserPlugin?: BP;
-    /**
-     * Proxy URL.
-     */
-    proxyUrl?: string;
-    /**
-     * Proxy tier.
-     */
-    proxyTier?: number;
 }
 export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
     /**
@@ -434,4 +509,3 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
      */
     launchOptions?: BP['launchOptions'];
 }
-//# sourceMappingURL=browser-pool.d.ts.map