@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/abstract-classes/browser-controller.d.ts

@@ -80,7 +80,14 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
  * @hideconstructor
  */
 export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
+    #private;
     readonly id: string;
+    /**
+     * Kept `protected` rather than `#private` because the concrete controllers in this package are
+     * separate classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     protected readonly log: CrawleeLogger;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -99,14 +106,17 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * `undefined` if no proxy is used
      */
     proxyUrl?: string;
+    /** @internal */
     isActive: boolean;
     activePages: number;
+    /** @internal */
     totalPages: number;
+    /** @internal */
     lastPageOpenedAt: number;
-    private _activate;
-    private isActivePromise;
-    private commitBrowser;
-    private hasBrowserPromise;
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    waitForActive(): Promise<void>;
     constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
     /**
      * Activates the BrowserController. If you try to open new pages before
@@ -137,30 +147,32 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
-    setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
-    getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**
-     * @private
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
      */
-    protected abstract _close(): Promise<void>;
+    registerPageClosed(page: NewPageResult): void;
     /**
-     * @private
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
      */
+    protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
+    setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+    getCookies(page: NewPageResult): Promise<Cookie[]>;
+    protected abstract _close(): Promise<void>;
     protected abstract _kill(): Promise<void>;
-    /**
-     * @private
-     */
     protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
-    /**
-     * @private
-     */
     protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
-    /**
-     * @private
-     */
     protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**
-     * @private
+     * @internal
      */
     abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
 }

package/abstract-classes/browser-controller.js

@@ -2,6 +2,7 @@ import { serviceLocator } from '@crawlee/core';
 import { nanoid } from 'nanoid';
 import { TypedEmitter } from 'tiny-typed-emitter';
 import { tryCancel } from '@apify/timeout';
+import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
 /**
  * The `BrowserController` serves two purposes. First, it is the base class that
@@ -13,6 +14,12 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
  */
 export class BrowserController extends TypedEmitter {
     id = nanoid();
+    /**
+     * Kept `protected` rather than `#private` because the concrete controllers in this package are
+     * separate classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     log;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -31,17 +38,28 @@ export class BrowserController extends TypedEmitter {
      * `undefined` if no proxy is used
      */
     proxyUrl;
+    /** @internal */
     isActive = false;
     activePages = 0;
+    #closedPages = new WeakSet();
+    #pageTeardowns = new WeakMap();
+    /** @internal */
     totalPages = 0;
+    /** @internal */
     lastPageOpenedAt = Date.now();
-    _activate;
-    isActivePromise = new Promise((resolve) => {
-        this._activate = resolve;
+    #activate;
+    #isActivePromise = new Promise((resolve) => {
+        this.#activate = resolve;
     });
-    commitBrowser;
-    hasBrowserPromise = new Promise((resolve) => {
-        this.commitBrowser = resolve;
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    async waitForActive() {
+        await this.#isActivePromise;
+    }
+    #commitBrowser;
+    #hasBrowserPromise = new Promise((resolve) => {
+        this.#commitBrowser = resolve;
     });
     constructor(browserPlugin) {
         super();
@@ -58,7 +76,7 @@ export class BrowserController extends TypedEmitter {
         if (!this.browser) {
             throw new Error('Cannot activate BrowserController without an assigned browser.');
         }
-        this._activate();
+        this.#activate();
         this.isActive = true;
     }
     /**
@@ -70,7 +88,7 @@ export class BrowserController extends TypedEmitter {
         }
         this.browser = browser;
         this.launchContext = launchContext;
-        this.commitBrowser();
+        this.#commitBrowser();
     }
     /**
      * Gracefully closes the browser and makes sure
@@ -79,7 +97,7 @@ export class BrowserController extends TypedEmitter {
      * Emits 'browserClosed' event.
      */
     async close() {
-        await this.hasBrowserPromise;
+        await this.#hasBrowserPromise;
         try {
             await this._close();
             // TODO: shouldn't this go in a finally instead?
@@ -88,7 +106,7 @@ export class BrowserController extends TypedEmitter {
         catch (error) {
             this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
         }
-        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
+        this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
         const killTimer = setTimeout(() => {
             this._kill().catch((err) => {
                 this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
@@ -102,9 +120,9 @@ export class BrowserController extends TypedEmitter {
      * Emits 'browserClosed' event.
      */
     async kill() {
-        await this.hasBrowserPromise;
+        await this.#hasBrowserPromise;
         await this._kill();
-        this.emit("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, this);
+        this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
     }
     /**
      * Opens new browser page.
@@ -113,12 +131,43 @@ export class BrowserController extends TypedEmitter {
     async newPage(pageOptions) {
         this.activePages++;
         this.totalPages++;
-        await this.isActivePromise;
+        await this.#isActivePromise;
         const page = await this._newPage(pageOptions);
         tryCancel();
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page) {
+        if (this.#closedPages.has(page))
+            return;
+        this.#closedPages.add(page);
+        this.activePages--;
+        const teardown = this.#pageTeardowns.get(page);
+        if (!teardown)
+            return;
+        this.#pageTeardowns.delete(page);
+        teardown().catch((error) => {
+            this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
+        });
+    }
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    registerPageTeardown(page, teardown) {
+        this.#pageTeardowns.set(page, teardown);
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }

package/abstract-classes/browser-plugin.d.ts

@@ -12,8 +12,6 @@ import type { BrowserController } from './browser-controller.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
 /**
@@ -83,7 +81,14 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
  * feed them to {@link BrowserPool} for use.
  */
 export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+    #private;
     readonly name: string;
+    /**
+     * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
+     * classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     protected readonly log: CrawleeLogger;
     readonly library: Library;
     readonly launchOptions: LibraryOptions;
@@ -119,7 +124,7 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
      * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
      * lives here so it stays identical across plugins.
      */
-    protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
+    protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;
     /**
      * Creates a `LaunchContext` with all the information needed
      * to launch a browser. Aside from library specific launch options,
@@ -132,16 +137,9 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
      * Launches the browser using provided launch context.
      */
     launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
-    private _mergeArgsToHideWebdriver;
-    protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
-    /**
-     * @private
-     */
-    protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
-    protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
-    /**
-     * @private
-     */
+    protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
+    protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
+    protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
     protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
 }
 export declare class BrowserLaunchError extends CriticalError {

package/abstract-classes/browser-plugin.js

@@ -9,8 +9,6 @@ import { sanitizeEndpointForLog } from '../utils.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
 /**
@@ -21,6 +19,12 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
  */
 export class BrowserPlugin {
     name = this.constructor.name;
+    /**
+     * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
+     * classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     log;
     library;
     launchOptions;
@@ -69,7 +73,7 @@ export class BrowserPlugin {
      * Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding
      * lives here so it stays identical across plugins.
      */
-    async _connectToRemoteBrowser(launchContext, connect) {
+    async connectToRemoteBrowser(launchContext, connect) {
         const connection = this.remoteConnection;
         let url;
         let token;
@@ -79,7 +83,7 @@ export class BrowserPlugin {
         catch (cause) {
             throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });
         }
-        launchContext._remoteToken = token;
+        launchContext.remoteToken = token;
         try {
             return await connect(url);
         }
@@ -117,11 +121,11 @@ export class BrowserPlugin {
         launchContext.launchOptions ??= {};
         const { proxyUrl, launchOptions } = launchContext;
         if (proxyUrl && !launchContext.isRemote) {
-            await this._addProxyToLaunchOptions(launchContext);
+            await this.addProxyToLaunchOptions(launchContext);
         }
-        if (!launchContext.isRemote && this._isChromiumBasedBrowser(launchContext)) {
+        if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
             // This will set the args for chromium based browsers to hide the webdriver.
-            launchOptions.args = this._mergeArgsToHideWebdriver(launchOptions.args);
+            launchOptions.args = this.#mergeArgsToHideWebdriver(launchOptions.args);
             // When User-Agent is not set, and we're using Chromium in headless mode,
             // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
             // as otherwise 'HeadlessChrome' is present in User-Agent string.
@@ -135,7 +139,7 @@ export class BrowserPlugin {
         }
         return this._launch(launchContext);
     }
-    _mergeArgsToHideWebdriver(originalArgs) {
+    #mergeArgsToHideWebdriver(originalArgs) {
         if (!originalArgs?.length) {
             return ['--disable-blink-features=AutomationControlled'];
         }
@@ -148,7 +152,7 @@ export class BrowserPlugin {
         }
         return originalArgs;
     }
-    _throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
+    throwAugmentedLaunchError(cause, executablePath, dockerImage, moduleInstallCommand) {
         const errorMessage = ['Failed to launch browser. Please check the following:'];
         if (executablePath) {
             errorMessage.push(`- Check whether the provided executable path "${executablePath}" is correct.`);

package/anonymize-proxy.d.ts

@@ -1,4 +1,5 @@
 type PromiseVoid = () => Promise<void>;
+/** @internal */
 export interface AnonymizeProxySugarOptions {
     ignoreProxyCertificate?: boolean;
 }

package/browser-pool.d.ts

@@ -248,34 +248,21 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
  * ```
  */
 export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
+    #private;
     browserPlugins: BrowserPlugins;
-    maxOpenPagesPerBrowser: number;
+    /** @internal */
     maxOpenBrowsers: number;
-    retireBrowserAfterPageCount: number;
-    operationTimeoutMillis: number;
-    closeInactiveBrowserAfterMillis: number;
-    useFingerprints?: boolean;
+    /** @internal */
     fingerprintOptions: FingerprintOptions;
-    preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
-    postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
-    prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
-    postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
-    prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
-    postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
-    pageCounter: number;
-    pages: Map<string, PageReturn>;
-    pageIds: WeakMap<PageReturn, string>;
-    startingBrowserControllers: Set<BrowserControllerReturn>;
-    activeBrowserControllers: Set<BrowserControllerReturn>;
-    retiredBrowserControllers: Set<BrowserControllerReturn>;
-    pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
+    /** @internal */
     fingerprintInjector?: FingerprintInjector;
     fingerprintGenerator?: FingerprintGenerator;
+    /** @internal */
     fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
+    private pages;
+    private activeBrowserControllers;
+    private retiredBrowserControllers;
     private browserKillerInterval?;
-    private browserRetireInterval?;
-    private limiter;
-    private log;
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
     /**
      * Opens a new page in one of the running browsers or launches
@@ -350,7 +337,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * until it's closed.
      */
     getPageId(page: PageReturn): string | undefined;
-    private _createPageForBrowser;
     /**
      * Removes a browser controller from the pool. The underlying
      * browser will be closed after all its pages are closed.
@@ -364,8 +350,9 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
     retireBrowserByPage(page: PageReturn): void;
     /**
      * Releases a page back to the pool. The page is closed and, if the
-     * optional `error` is a {@link SessionError}, the browser controller
-     * that served the page is retired so that its tainted state (cookies,
+     * optional `error` is a {@link SessionError} (including
+     * {@link SessionRetiredError}), the browser controller that served
+     * the page is retired so that the finished session's state (cookies,
      * storage, etc.) cannot leak into future sessions.
      *
      * This is the primary way the crawler should return pages to the pool.
@@ -409,34 +396,36 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * @return {Promise<void>}
      */
     closeAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Closes all managed browsers and tears down the pool.
+     * Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
+     * browsers when a run ends and may start another run on the same pool.
      */
-    destroy(): Promise<void>;
-    private _teardown;
-    private _getAllBrowserControllers;
-    private _launchBrowser;
+    releaseAllBrowsers(): Promise<void>;
     /**
-     * Picks plugins round robin.
-     * @private
+     * Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
+     * re-armed and its listeners are dropped, so it cannot be used again.
      */
-    private _pickBrowserPlugin;
-    private _pickBrowserWithFreeCapacity;
-    private _closeInactiveRetiredBrowsers;
-    private _overridePageClose;
-    private _executeHooks;
-    private _closeRetiredBrowserWithNoPages;
+    destroy(): Promise<void>;
+    private closeInactiveRetiredBrowsers;
     /**
-     * Returns `true` if the pool can accept a new browser launch without exceeding
-     * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
+     * Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
+     * Counts starting, active, and retired browsers.
+     *
+     * A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
+     * has set a cap — {@link RemoteBrowserPool} does, from its own
+     * {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
+     * `BrowserPoolOptions` key for it.
+     *
+     * @internal
      */
     hasFreeBrowserSlot(): boolean;
     /**
      * Returns `true` if any active browser has room for another page.
+     *
+     * @internal
      */
     hasActiveBrowserWithFreeCapacity(): boolean;
-    private _initializeFingerprinting;
-    private _addFingerprintHooks;
 }
 export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
     /**