@crawlee/browser-pool 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/abstract-classes/browser-controller.d.ts

@@ -82,6 +82,12 @@ export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryO
 export declare abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
     #private;
     readonly id: string;
+    /**
+     * Kept `protected` rather than `#private` because the concrete controllers in this package are
+     * separate classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     protected readonly log: CrawleeLogger;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -100,11 +106,17 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * `undefined` if no proxy is used
      */
     proxyUrl?: string;
+    /** @internal */
     isActive: boolean;
     activePages: number;
+    /** @internal */
     totalPages: number;
+    /** @internal */
     lastPageOpenedAt: number;
-    private isActivePromise;
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    waitForActive(): Promise<void>;
     constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
     /**
      * Activates the BrowserController. If you try to open new pages before
@@ -135,30 +147,32 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
-    setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
-    getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**
-     * @private
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
      */
-    protected abstract _close(): Promise<void>;
+    registerPageClosed(page: NewPageResult): void;
     /**
-     * @private
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
      */
+    protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
+    setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+    getCookies(page: NewPageResult): Promise<Cookie[]>;
+    protected abstract _close(): Promise<void>;
     protected abstract _kill(): Promise<void>;
-    /**
-     * @private
-     */
     protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
-    /**
-     * @private
-     */
     protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
-    /**
-     * @private
-     */
     protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**
-     * @private
+     * @internal
      */
     abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
 }

package/abstract-classes/browser-controller.js

@@ -14,6 +14,12 @@ const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
  */
 export class BrowserController extends TypedEmitter {
     id = nanoid();
+    /**
+     * Kept `protected` rather than `#private` because the concrete controllers in this package are
+     * separate classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     log;
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -32,15 +38,25 @@ export class BrowserController extends TypedEmitter {
      * `undefined` if no proxy is used
      */
     proxyUrl;
+    /** @internal */
     isActive = false;
     activePages = 0;
+    #closedPages = new WeakSet();
+    #pageTeardowns = new WeakMap();
+    /** @internal */
     totalPages = 0;
+    /** @internal */
     lastPageOpenedAt = Date.now();
     #activate;
-    // kept as TS-private: `BrowserPool` awaits it through cross-object bracket access
-    isActivePromise = new Promise((resolve) => {
+    #isActivePromise = new Promise((resolve) => {
         this.#activate = resolve;
     });
+    /**
+     * Returns a promise that resolves once the browser controller has been activated.
+     */
+    async waitForActive() {
+        await this.#isActivePromise;
+    }
     #commitBrowser;
     #hasBrowserPromise = new Promise((resolve) => {
         this.#commitBrowser = resolve;
@@ -115,12 +131,43 @@ export class BrowserController extends TypedEmitter {
     async newPage(pageOptions) {
         this.activePages++;
         this.totalPages++;
-        await this.isActivePromise;
+        await this.#isActivePromise;
         const page = await this._newPage(pageOptions);
         tryCancel();
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page) {
+        if (this.#closedPages.has(page))
+            return;
+        this.#closedPages.add(page);
+        this.activePages--;
+        const teardown = this.#pageTeardowns.get(page);
+        if (!teardown)
+            return;
+        this.#pageTeardowns.delete(page);
+        teardown().catch((error) => {
+            this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
+        });
+    }
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    registerPageTeardown(page, teardown) {
+        this.#pageTeardowns.set(page, teardown);
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }

package/abstract-classes/browser-plugin.d.ts

@@ -12,8 +12,6 @@ import type { BrowserController } from './browser-controller.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
 /**
@@ -83,7 +81,14 @@ export interface CreateLaunchContextOptions<Library extends CommonLibrary, Libra
  * feed them to {@link BrowserPool} for use.
  */
 export declare abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+    #private;
     readonly name: string;
+    /**
+     * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
+     * classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     protected readonly log: CrawleeLogger;
     readonly library: Library;
     readonly launchOptions: LibraryOptions;
@@ -132,16 +137,9 @@ export declare abstract class BrowserPlugin<Library extends CommonLibrary = Comm
      * Launches the browser using provided launch context.
      */
     launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
-    private mergeArgsToHideWebdriver;
     protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
-    /**
-     * @private
-     */
     protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
     protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
-    /**
-     * @private
-     */
     protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
 }
 export declare class BrowserLaunchError extends CriticalError {

package/abstract-classes/browser-plugin.js

@@ -9,8 +9,6 @@ import { sanitizeEndpointForLog } from '../utils.js';
  *  - without using a fingerprint,
  *  - without specifying a user agent.
  * Last updated on 2022-05-05.
- *
- * After you update it here, please update it also in jsdom-crawler.ts
  */
 export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
 /**
@@ -21,6 +19,12 @@ export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_
  */
 export class BrowserPlugin {
     name = this.constructor.name;
+    /**
+     * Kept `protected` rather than `#private` because the concrete plugins in this package are separate
+     * classes — an ES private field would not be visible to them.
+     *
+     * @internal
+     */
     log;
     library;
     launchOptions;
@@ -121,7 +125,7 @@ export class BrowserPlugin {
         }
         if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {
             // This will set the args for chromium based browsers to hide the webdriver.
-            launchOptions.args = this.mergeArgsToHideWebdriver(launchOptions.args);
+            launchOptions.args = this.#mergeArgsToHideWebdriver(launchOptions.args);
             // When User-Agent is not set, and we're using Chromium in headless mode,
             // it is better to use DEFAULT_USER_AGENT to reduce chance of detection,
             // as otherwise 'HeadlessChrome' is present in User-Agent string.
@@ -135,7 +139,7 @@ export class BrowserPlugin {
         }
         return this._launch(launchContext);
     }
-    mergeArgsToHideWebdriver(originalArgs) {
+    #mergeArgsToHideWebdriver(originalArgs) {
         if (!originalArgs?.length) {
             return ['--disable-blink-features=AutomationControlled'];
         }

package/anonymize-proxy.d.ts

@@ -1,4 +1,5 @@
 type PromiseVoid = () => Promise<void>;
+/** @internal */
 export interface AnonymizeProxySugarOptions {
     ignoreProxyCertificate?: boolean;
 }

package/browser-pool.d.ts

@@ -250,29 +250,18 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
 export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
     #private;
     browserPlugins: BrowserPlugins;
-    maxOpenPagesPerBrowser: number;
+    /** @internal */
     maxOpenBrowsers: number;
-    retireBrowserAfterPageCount: number;
-    operationTimeoutMillis: number;
-    closeInactiveBrowserAfterMillis: number;
-    useFingerprints?: boolean;
+    /** @internal */
     fingerprintOptions: FingerprintOptions;
-    preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
-    postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
-    prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
-    postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
-    prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
-    postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
-    pageCounter: number;
-    pages: Map<string, PageReturn>;
-    pageIds: WeakMap<PageReturn, string>;
-    startingBrowserControllers: Set<BrowserControllerReturn>;
-    activeBrowserControllers: Set<BrowserControllerReturn>;
-    retiredBrowserControllers: Set<BrowserControllerReturn>;
-    pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
+    /** @internal */
     fingerprintInjector?: FingerprintInjector;
     fingerprintGenerator?: FingerprintGenerator;
+    /** @internal */
     fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
+    private pages;
+    private activeBrowserControllers;
+    private retiredBrowserControllers;
     private browserKillerInterval?;
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
     /**
@@ -348,7 +337,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * until it's closed.
      */
     getPageId(page: PageReturn): string | undefined;
-    private createPageForBrowser;
     /**
      * Removes a browser controller from the pool. The underlying
      * browser will be closed after all its pages are closed.
@@ -362,8 +350,9 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
     retireBrowserByPage(page: PageReturn): void;
     /**
      * Releases a page back to the pool. The page is closed and, if the
-     * optional `error` is a {@link SessionError}, the browser controller
-     * that served the page is retired so that its tainted state (cookies,
+     * optional `error` is a {@link SessionError} (including
+     * {@link SessionRetiredError}), the browser controller that served
+     * the page is retired so that the finished session's state (cookies,
      * storage, etc.) cannot leak into future sessions.
      *
      * This is the primary way the crawler should return pages to the pool.
@@ -407,34 +396,36 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * @return {Promise<void>}
      */
     closeAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Closes all managed browsers and tears down the pool.
+     * Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
+     * browsers when a run ends and may start another run on the same pool.
      */
-    destroy(): Promise<void>;
-    private teardown;
-    private getAllBrowserControllers;
-    private launchBrowser;
+    releaseAllBrowsers(): Promise<void>;
     /**
-     * Picks plugins round robin.
-     * @private
+     * Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
+     * re-armed and its listeners are dropped, so it cannot be used again.
      */
-    private pickBrowserPlugin;
-    private pickBrowserWithFreeCapacity;
+    destroy(): Promise<void>;
     private closeInactiveRetiredBrowsers;
-    private overridePageClose;
-    private executeHooks;
-    private closeRetiredBrowserWithNoPages;
     /**
-     * Returns `true` if the pool can accept a new browser launch without exceeding
-     * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
+     * Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
+     * Counts starting, active, and retired browsers.
+     *
+     * A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
+     * has set a cap — {@link RemoteBrowserPool} does, from its own
+     * {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
+     * `BrowserPoolOptions` key for it.
+     *
+     * @internal
      */
     hasFreeBrowserSlot(): boolean;
     /**
      * Returns `true` if any active browser has room for another page.
+     *
+     * @internal
      */
     hasActiveBrowserWithFreeCapacity(): boolean;
-    private initializeFingerprinting;
-    private addFingerprintHooks;
 }
 export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
     /**