@crawlee/browser-pool 4.0.0-beta.8 → 4.0.0-beta.80

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/abstract-classes/browser-controller.d.ts

@@ -1,9 +1,73 @@
+import { type CrawleeLogger } from '@crawlee/core';
 import type { Cookie, Dictionary } from '@crawlee/types';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 import type { LaunchContext } from '../launch-context.js';
 import type { UnwrapPromise } from '../utils.js';
 import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
+/**
+ * The subset of the browser-pool `LaunchContext` that {@link IBrowserController} exposes.
+ * Other fields are only available on the concrete `LaunchContext` class.
+ */
+export interface IBrowserLaunchContext {
+    /**
+     * The proxy URL the browser was launched with, if any.
+     */
+    proxyUrl?: string;
+    /**
+     * The fingerprint applied to the browser, if fingerprinting is enabled.
+     * Typed as `unknown` here; cast to the concrete `LaunchContext` if you
+     * need the structured shape.
+     */
+    fingerprint?: unknown;
+    /**
+     * `true` if each page in this browser uses its own context.
+     */
+    useIncognitoPages?: boolean;
+    /**
+     * The actual options the browser was launched with, after pre-launch hooks.
+     */
+    launchOptions?: Dictionary | undefined;
+}
+/**
+ * The minimal public contract of a browser controller.
+ *
+ * Coordination with the pool (page-counting, `activate`, `assignBrowser`, lifecycle
+ * promises, …) is intentionally **not** part of this contract.
+ *
+ * @category Browser management
+ */
+export interface IBrowserController<Page = unknown> {
+    /**
+     * A stable identifier for this controller instance. Useful for tracking
+     * which browser served which request.
+     */
+    readonly id: string;
+    /**
+     * The configuration the underlying browser was launched with — proxy URL,
+     * fingerprint, session, launcher-specific options, etc.
+     */
+    readonly launchContext: IBrowserLaunchContext;
+    /**
+     * The raw browser handle from the underlying automation library
+     * (Puppeteer `Browser`, Playwright `Browser`/`BrowserContext`, …).
+     * Escape hatch for things the controller does not expose directly.
+     */
+    readonly browser: unknown;
+    /**
+     * Reads cookies for the given page.
+     */
+    getCookies(page: Page): Promise<Cookie[]>;
+    /**
+     * Writes cookies for the given page.
+     */
+    setCookies(page: Page, cookies: Cookie[]): Promise<void>;
+    /**
+     * Gracefully closes the browser this controller owns. After this resolves,
+     * the controller is no longer usable.
+     */
+    close(): Promise<void>;
+}
 export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
     [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
 }
@@ -15,8 +79,9 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
  * for the specialized classes, because it's the same for all of them.
  * @hideconstructor
  */
-export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
+export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
     id: string;
+    protected log: CrawleeLogger;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
      */
@@ -30,12 +95,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      */
     launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
     /**
-     * The proxy tier tied to this browser controller.
-     * `undefined` if no tiered proxy is used.
-     */
-    proxyTier?: number;
-    /**
-     * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
+     * The proxy URL used by the browser controller.
      * `undefined` if no proxy is used
      */
     proxyUrl?: string;
@@ -104,4 +164,3 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      */
     abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
 }
-//# sourceMappingURL=browser-controller.d.ts.map

package/abstract-classes/browser-controller.js

@@ -1,7 +1,7 @@
+import { serviceLocator } from '@crawlee/core';
 import { nanoid } from 'nanoid';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import { tryCancel } from '@apify/timeout';
-import { log } from '../logger.js';
 const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
 /**
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +13,7 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
  */
 export class BrowserController extends TypedEmitter {
     id = nanoid();
+    log;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
      */
@@ -26,12 +27,7 @@ export class BrowserController extends TypedEmitter {
      */
     launchContext = undefined;
     /**
-     * The proxy tier tied to this browser controller.
-     * `undefined` if no tiered proxy is used.
-     */
-    proxyTier;
-    /**
-     * The proxy URL used by the browser controller. This is set every time the browser controller uses proxy (even the tiered one).
+     * The proxy URL used by the browser controller.
      * `undefined` if no proxy is used
      */
     proxyUrl;
@@ -49,6 +45,7 @@ export class BrowserController extends TypedEmitter {
     });
     constructor(browserPlugin) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.browserPlugin = browserPlugin;
     }
     /**
@@ -89,14 +86,15 @@ export class BrowserController extends TypedEmitter {
             this.isActive = false;
         }
         catch (error) {
-            log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
+            this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
         }
         this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
-        setTimeout(() => {
+        const killTimer = setTimeout(() => {
             this._kill().catch((err) => {
-                log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
+                this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
             });
         }, PROCESS_KILL_TIMEOUT_MILLIS);
+        killTimer.unref();
     }
     /**
      * Immediately kills the browser process.
@@ -128,4 +126,3 @@ export class BrowserController extends TypedEmitter {
         return this._getCookies(page);
     }
 }
-//# sourceMappingURL=browser-controller.js.map

package/abstract-classes/browser-plugin.d.ts

@@ -1,8 +1,9 @@
-import { CriticalError } from '@crawlee/core';
+import { type CrawleeLogger, CriticalError } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import type { LaunchContextOptions } from '../launch-context.js';
 import { LaunchContext } from '../launch-context.js';
-import type { UnwrapPromise } from '../utils.js';
+import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';
+import { type UnwrapPromise } from '../utils.js';
 import type { BrowserController } from './browser-controller.js';
 /**
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
@@ -35,6 +36,7 @@ export interface CommonBrowser {
 export interface CommonPage {
     close(...args: unknown[]): Promise<unknown>;
     url(): string | Promise<string>;
+    evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;
 }
 export interface BrowserPluginOptions<LibraryOptions> {
     /**
@@ -68,6 +70,11 @@ export interface BrowserPluginOptions<LibraryOptions> {
      * Might cause performance issues, as Crawlee might launch too many browser instances.
      */
     browserPerProxy?: boolean;
+    /**
+     * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+     * This is useful when using HTTPS proxies with self-signed certificates.
+     */
+    ignoreProxyCertificate?: boolean;
 }
 export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
 }
@@ -79,13 +86,42 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
  */
 export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
     name: string;
+    protected log: CrawleeLogger;
     library: Library;
     launchOptions: LibraryOptions;
     proxyUrl?: string;
     userDataDir?: string;
     useIncognitoPages: boolean;
     browserPerProxy?: boolean;
+    ignoreProxyCertificate?: boolean;
+    /**
+     * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
+     * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
+     * remote-session policy lives in the pool, not here.
+     *
+     * @internal
+     */
+    remoteConnection?: RemoteConnection;
+    /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
+    remoteConnectionParameters?: RemoteConnectionParameters;
     constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
+    /**
+     * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
+     * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
+     * (e.g. forcing incognito pages).
+     *
+     * @internal
+     */
+    useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
+    /**
+     * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
+     * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
+     * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
+     *
+     * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
+     * lives here so it stays identical across plugins.
+     */
+    protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
     /**
      * Creates a `LaunchContext` with all the information needed
      * to launch a browser. Aside from library specific launch options,
@@ -113,4 +149,3 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
 export declare class BrowserLaunchError extends CriticalError {
     constructor(...args: ConstructorParameters<typeof CriticalError>);
 }
-//# sourceMappingURL=browser-plugin.d.ts.map

package/abstract-classes/browser-plugin.js

@@ -1,6 +1,7 @@
-import { CriticalError } from '@crawlee/core';
+import { CriticalError, serviceLocator } from '@crawlee/core';
 import merge from 'lodash.merge';
 import { LaunchContext } from '../launch-context.js';
+import { sanitizeEndpointForLog } from '../utils.js';
 /**
  * The default User Agent used by `PlaywrightCrawler`, `launchPlaywright`, 'PuppeteerCrawler' and 'launchPuppeteer'
  * when Chromium/Chrome browser is launched:
@@ -20,20 +21,73 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
  */
 export class BrowserPlugin {
     name = this.constructor.name;
+    log;
     library;
     launchOptions;
     proxyUrl;
     userDataDir;
     useIncognitoPages;
     browserPerProxy;
+    ignoreProxyCertificate;
+    /**
+     * Set by {@link RemoteBrowserPool} when this plugin connects to a remote browser service instead of
+     * launching locally. Holds the bridge the plugin uses to resolve endpoints and release sessions; all
+     * remote-session policy lives in the pool, not here.
+     *
+     * @internal
+     */
+    remoteConnection;
+    /** Static connect() parameters for a remote connection (protocol, headers, …). @internal */
+    remoteConnectionParameters;
     constructor(library, options = {}) {
-        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, } = options;
+        const { launchOptions = {}, proxyUrl, userDataDir, useIncognitoPages = false, browserPerProxy = false, ignoreProxyCertificate = false, } = options;
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.library = library;
         this.launchOptions = launchOptions;
         this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);
         this.userDataDir = userDataDir;
         this.useIncognitoPages = useIncognitoPages;
         this.browserPerProxy = browserPerProxy;
+        this.ignoreProxyCertificate = ignoreProxyCertificate;
+    }
+    /**
+     * Configures this plugin to connect to a remote browser using the given {@link RemoteConnection}.
+     * Called by {@link RemoteBrowserPool}; subclasses may override to apply library-specific defaults
+     * (e.g. forcing incognito pages).
+     *
+     * @internal
+     */
+    useRemoteConnection(connection, parameters = {}) {
+        this.remoteConnection = connection;
+        this.remoteConnectionParameters = parameters;
+    }
+    /**
+     * Resolves a remote endpoint via the injected {@link RemoteConnection}, stores the session token on
+     * the launch context (so the controller can release it on close), and runs the library-specific `connect`.
+     * On failure the session is released and the error is wrapped in a {@link BrowserLaunchError}.
+     *
+     * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
+     * lives here so it stays identical across plugins.
+     */
+    async _connectToRemoteBrowser(launchContext, connect) {
+        const connection = this.remoteConnection;
+        let url;
+        let token;
+        try {
+            ({ url, token } = await connection.resolve({ proxyUrl: launchContext.proxyUrl }));
+        }
+        catch (cause) {
+            throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
+        }
+        launchContext._remoteToken = token;
+        try {
+            return await connect(url);
+        }
+        catch (cause) {
+            await connection.release(token);
+            throw new BrowserLaunchError(`Failed to connect to remote browser at "${sanitizeEndpointForLog(url)}". ` +
+                'Check that the endpoint is reachable and accepts the configured protocol.', { cause });
+        }
     }
     /**
      * Creates a `LaunchContext` with all the information needed
@@ -42,7 +96,7 @@ export class BrowserPlugin {
      * management of the pool and extra features.
      */
     createLaunchContext(options = {}) {
-        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, proxyTier, } = options;
+        const { id, launchOptions = {}, proxyUrl = this.proxyUrl, useIncognitoPages = this.useIncognitoPages, userDataDir = this.userDataDir, browserPerProxy = this.browserPerProxy, ignoreProxyCertificate = this.ignoreProxyCertificate, isRemote = !!this.remoteConnection, } = options;
         return new LaunchContext({
             id,
             launchOptions: merge({}, this.launchOptions, launchOptions),
@@ -51,19 +105,21 @@ export class BrowserPlugin {
             useIncognitoPages,
             userDataDir,
             browserPerProxy,
-            proxyTier,
+            ignoreProxyCertificate,
+            isRemote,
         });
     }
     /**
      * Launches the browser using provided launch context.
      */
     async launch(launchContext = this.createLaunchContext()) {
+        // launchOptions is only used by the local launch path below — remote connections ignore it.
         launchContext.launchOptions ??= {};
         const { proxyUrl, launchOptions } = launchContext;
-        if (proxyUrl) {
+        if (proxyUrl && !launchContext.isRemote) {
             await this._addProxyToLaunchOptions(launchContext);
         }
-        if (this._isChromiumBasedBrowser(launchContext)) {
+        if (!launchContext.isRemote && this._isChromiumBasedBrowser(launchContext)) {
             // This will set the args for chromium based browsers to hide the webdriver.
             launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
             // When User-Agent is not set, and we're using Chromium in headless mode,
@@ -74,6 +130,9 @@ export class BrowserPlugin {
                 launchOptions.args.push(`--user-agent=${DEFAULT_USER_AGENT}`);
             }
         }
+        if (launchContext.isRemote) {
+            this.log.info('Connecting to remote browser (skipping local proxy and webdriver stealth configuration).');
+        }
         return this._launch(launchContext);
     }
     _mergeArgsToHideWebdriver(originalArgs) {
@@ -118,4 +177,3 @@ export class BrowserLaunchError extends CriticalError {
         });
     }
 }
-//# sourceMappingURL=browser-plugin.js.map

package/anonymize-proxy.d.ts

@@ -1,4 +1,7 @@
 type PromiseVoid = () => Promise<void>;
-export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string) => Promise<[string | undefined, PromiseVoid]>;
+export interface AnonymizeProxySugarOptions {
+    ignoreProxyCertificate?: boolean;
+}
+/** @internal */
+export declare const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
 export {};
-//# sourceMappingURL=anonymize-proxy.d.ts.map

package/anonymize-proxy.js

@@ -1,15 +1,20 @@
-import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
-export const anonymizeProxySugar = async (proxyUrl, username, password) => {
+/** @internal */
+export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
     if (proxyUrl) {
         const url = new URL(proxyUrl);
         if (username || password) {
             url.username = username ?? '';
             url.password = password ?? '';
         }
-        if (url.username || url.password) {
+        if (url.username || url.password || options?.ignoreProxyCertificate) {
             // trim off trailing slash if it's present
             const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
-            const anonymized = await anonymizeProxy(proxyUrlString);
+            const { anonymizeProxy, closeAnonymizedProxy } = await import('proxy-chain');
+            const anonymized = await anonymizeProxy({
+                url: proxyUrlString,
+                port: 0,
+                ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
+            });
             return [
                 anonymized,
                 async () => {
@@ -21,4 +26,3 @@ export const anonymizeProxySugar = async (proxyUrl, username, password) => {
     }
     return [undefined, async () => { }];
 };
-//# sourceMappingURL=anonymize-proxy.js.map

package/browser-pool.d.ts

@@ -1,3 +1,4 @@
+import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
@@ -246,9 +247,10 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
  * });
  * ```
  */
-export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
+export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
     browserPlugins: BrowserPlugins;
     maxOpenPagesPerBrowser: number;
+    maxOpenBrowsers: number;
     retireBrowserAfterPageCount: number;
     operationTimeoutMillis: number;
     closeInactiveBrowserAfterMillis: number;
@@ -263,6 +265,7 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
     pageCounter: number;
     pages: Map<string, PageReturn>;
     pageIds: WeakMap<PageReturn, string>;
+    startingBrowserControllers: Set<BrowserControllerReturn>;
     activeBrowserControllers: Set<BrowserControllerReturn>;
     retiredBrowserControllers: Set<BrowserControllerReturn>;
     pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
@@ -272,11 +275,24 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
     private browserKillerInterval?;
     private browserRetireInterval?;
     private limiter;
+    private log;
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
     /**
      * Opens a new page in one of the running browsers or launches
      * a new browser and opens a page there, if no browsers are active,
      * or their page limits have been exceeded.
+     *
+     * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
+     * provided, this implementation uses it as a cache key for browser fingerprints (when
+     * fingerprinting is enabled) and reads
+     * {@link ProxyInfo.url|session.proxyInfo.url} /
+     * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
+     * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
+     * `ignoreTlsErrors` values in the options take precedence.
+     *
+     * Beyond fingerprint caching and proxy configuration, no other session
+     * properties are consumed — cookie and header injection remain the
+     * crawler's responsibility.
      */
     newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
     /**
@@ -346,6 +362,43 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * closed after all its pages are closed.
      */
     retireBrowserByPage(page: PageReturn): void;
+    /**
+     * Releases a page back to the pool. The page is closed and, if the
+     * optional `error` is a {@link SessionError}, the browser controller
+     * that served the page is retired so that its tainted state (cookies,
+     * storage, etc.) cannot leak into future sessions.
+     *
+     * This is the primary way the crawler should return pages to the pool.
+     *
+     * @param page The page to release.
+     * @param options.error The error that caused the page to be released, if any.
+     */
+    closePage(page: PageReturn, options?: {
+        error?: Error;
+    }): Promise<void>;
+    /**
+     * Extracts the relevant state (currently just cookies) from a page via its
+     * owning {@link BrowserController}. Returns empty state when the page is
+     * no longer associated with a controller.
+     *
+     * As with {@link BrowserPool.injectPageState}, cookies are isolated per
+     * page only when the pool is configured with `useIncognitoPages: true`.
+     * With the default `useIncognitoPages: false`, the extracted cookies
+     * include those set by any sibling page sharing the same browser.
+     */
+    extractPageState(page: PageReturn): Promise<PageState>;
+    /**
+     * Injects state into a page via its owning {@link BrowserController}.
+     *
+     * No-op when the page is no longer associated with a controller.
+     *
+     * Note that cookies are isolated per page only when the pool is configured
+     * with `useIncognitoPages: true` — each page then gets its own browser
+     * context. With the default `useIncognitoPages: false`, all pages in a
+     * browser share a single context, so injected cookies are visible to every
+     * page served by that browser.
+     */
+    injectPageState(page: PageReturn, state: PageState): Promise<void>;
     /**
      * Removes all active browsers from the pool. The browsers will be
      * closed after all their pages are closed.
@@ -373,15 +426,45 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
     private _overridePageClose;
     private _executeHooks;
     private _closeRetiredBrowserWithNoPages;
+    /**
+     * Returns `true` if the pool can accept a new browser launch without exceeding
+     * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
+     */
+    hasFreeBrowserSlot(): boolean;
+    /**
+     * Returns `true` if any active browser has room for another page.
+     */
+    hasActiveBrowserWithFreeCapacity(): boolean;
     private _initializeFingerprinting;
     private _addFingerprintHooks;
 }
-export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> {
+export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
+    /**
+     * The proxy URL the pool uses internally to route the page: it keys browser
+     * reuse (with `browserPerProxy`, only a browser already on this proxy is
+     * reused), configures the launched browser, and is applied to incognito
+     * pages. When omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool`'s proxy
+     * handling and is intentionally not part of the {@link IBrowserPool}
+     * contract — through that interface the proxy is supplied via the session.
+     */
+    proxyUrl?: string;
     /**
-     * Assign a custom ID to the page. If you don't a random string ID
-     * will be generated.
+     * Disable TLS certificate verification for MITM proxies. Applied both when
+     * launching a new browser and when creating a page in an existing one. When
+     * omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool` and is
+     * intentionally not part of the {@link IBrowserPool} contract — through
+     * that interface, configure it via the session's `proxyInfo` or through the
+     * browser's `launchOptions`.
      */
-    id?: string;
+    ignoreTlsErrors?: boolean;
     /**
      * Some libraries (Playwright) allow you to open new pages with specific
      * options. Use this property to set those options.
@@ -396,14 +479,6 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
      * see the `newPageInNewBrowser` function.
      */
     browserPlugin?: BP;
-    /**
-     * Proxy URL.
-     */
-    proxyUrl?: string;
-    /**
-     * Proxy tier.
-     */
-    proxyTier?: number;
 }
 export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
     /**
@@ -434,4 +509,3 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
      */
     launchOptions?: BP['launchOptions'];
 }
-//# sourceMappingURL=browser-pool.d.ts.map