@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.1

package/browser-pool.js

@@ -1,17 +1,35 @@
 import { AsyncResource } from 'node:async_hooks';
 import { SessionError, serviceLocator } from '@crawlee/core';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
 import { nanoid } from 'nanoid';
-import ow from 'ow';
 import pLimit from 'p-limit';
 import QuickLRU from 'quick-lru';
 import { TypedEmitter } from 'tiny-typed-emitter';
+import { z } from 'zod';
 import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { BROWSER_POOL_EVENTS } from './events.js';
 import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
 const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
 const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
 const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
+const browserPoolOptionsSchema = z.strictObject({
+    browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
+    maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
+    retireBrowserAfterPageCount: schemas.anyNumber.default(100),
+    operationTimeoutSecs: schemas.anyNumber.default(15),
+    closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
+    retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
+    preLaunchHooks: schemas.anyArray.default(() => []),
+    postLaunchHooks: schemas.anyArray.default(() => []),
+    prePageCreateHooks: schemas.anyArray.default(() => []),
+    postPageCreateHooks: schemas.anyArray.default(() => []),
+    prePageCloseHooks: schemas.anyArray.default(() => []),
+    postPageCloseHooks: schemas.anyArray.default(() => []),
+    useFingerprints: z.boolean().default(true),
+    fingerprintOptions: schemas.anyObject.default(() => ({})),
+});
 /**
  * The `BrowserPool` class is the most important class of the `browser-pool` module.
  * It manages opening and closing of browsers and their pages and its constructor
@@ -61,54 +79,44 @@ const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
  */
 export class BrowserPool extends TypedEmitter {
     browserPlugins;
-    maxOpenPagesPerBrowser;
+    /** @internal */
     maxOpenBrowsers;
-    retireBrowserAfterPageCount;
-    operationTimeoutMillis;
-    closeInactiveBrowserAfterMillis;
-    useFingerprints;
+    /** @internal */
     fingerprintOptions;
-    preLaunchHooks;
-    postLaunchHooks;
-    prePageCreateHooks;
-    postPageCreateHooks;
-    prePageCloseHooks;
-    postPageCloseHooks;
-    pageCounter = 0;
-    pages = new Map();
-    pageIds = new WeakMap();
-    startingBrowserControllers = new Set();
-    activeBrowserControllers = new Set();
-    retiredBrowserControllers = new Set();
-    pageToBrowserController = new WeakMap();
+    /** @internal */
     fingerprintInjector;
     fingerprintGenerator;
+    /** @internal */
     fingerprintCache;
-    browserKillerInterval = setInterval(async () => this._closeInactiveRetiredBrowsers(), BROWSER_KILLER_INTERVAL_MILLIS);
-    browserRetireInterval;
-    limiter = pLimit(1);
-    log;
+    #maxOpenPagesPerBrowser;
+    #retireBrowserAfterPageCount;
+    #operationTimeoutMillis;
+    #closeInactiveBrowserAfterMillis;
+    #preLaunchHooks;
+    #postLaunchHooks;
+    #prePageCreateHooks;
+    #postPageCreateHooks;
+    #prePageCloseHooks;
+    #postPageCloseHooks;
+    #pageCounter = 0;
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- tests observe page tracking and controller retirement directly
+    pages = new Map();
+    #pageIds = new WeakMap();
+    #startingBrowserControllers = new Set();
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
+    activeBrowserControllers = new Set();
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
+    retiredBrowserControllers = new Set();
+    #pageToBrowserController = new WeakMap();
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- tests replace this interval through bracket access
+    browserKillerInterval;
+    #browserRetireInterval;
+    #limiter = pLimit(1);
+    #log;
     constructor(options) {
         super();
-        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
-        this.browserKillerInterval.unref();
-        ow(options, ow.object.exactShape({
-            browserPlugins: ow.array.minLength(1),
-            maxOpenPagesPerBrowser: ow.optional.number,
-            retireBrowserAfterPageCount: ow.optional.number,
-            operationTimeoutSecs: ow.optional.number,
-            closeInactiveBrowserAfterSecs: ow.optional.number,
-            retireInactiveBrowserAfterSecs: ow.optional.number,
-            preLaunchHooks: ow.optional.array,
-            postLaunchHooks: ow.optional.array,
-            prePageCreateHooks: ow.optional.array,
-            postPageCreateHooks: ow.optional.array,
-            prePageCloseHooks: ow.optional.array,
-            postPageCloseHooks: ow.optional.array,
-            useFingerprints: ow.optional.boolean,
-            fingerprintOptions: ow.optional.object,
-        }));
-        const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
+        this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
+        const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
         const firstPluginConstructor = browserPlugins[0].constructor;
         for (let i = 1; i < browserPlugins.length; i++) {
             const providedPlugin = browserPlugins[i];
@@ -119,30 +127,41 @@ export class BrowserPool extends TypedEmitter {
             }
         }
         this.browserPlugins = browserPlugins;
-        this.maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
         this.maxOpenBrowsers = Infinity;
-        this.retireBrowserAfterPageCount = retireBrowserAfterPageCount;
-        this.operationTimeoutMillis = operationTimeoutSecs * 1000;
-        this.closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
-        this.useFingerprints = useFingerprints;
         this.fingerprintOptions = fingerprintOptions;
-        this.browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
+        this.#maxOpenPagesPerBrowser = maxOpenPagesPerBrowser;
+        this.#retireBrowserAfterPageCount = retireBrowserAfterPageCount;
+        this.#operationTimeoutMillis = operationTimeoutSecs * 1000;
+        this.#closeInactiveBrowserAfterMillis = closeInactiveBrowserAfterSecs * 1000;
+        // Sweeping slower than the window it enforces would round any sub-10s
+        // `closeInactiveBrowserAfterSecs` up to the sweep period.
+        this.browserKillerInterval = setInterval(async () => this.closeInactiveRetiredBrowsers(), Math.min(BROWSER_KILLER_INTERVAL_MILLIS, this.#closeInactiveBrowserAfterMillis));
+        this.browserKillerInterval.unref();
+        this.#browserRetireInterval = setInterval(async () => this.activeBrowserControllers.forEach((controller) => {
             if (controller.activePages === 0 &&
                 controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000) {
                 this.retireBrowserController(controller);
             }
         }), retireInactiveBrowserAfterSecs * 1000);
-        this.browserRetireInterval.unref();
+        this.#browserRetireInterval.unref();
         // hooks
-        this.preLaunchHooks = preLaunchHooks;
-        this.postLaunchHooks = postLaunchHooks;
-        this.prePageCreateHooks = prePageCreateHooks;
-        this.postPageCreateHooks = postPageCreateHooks;
-        this.prePageCloseHooks = prePageCloseHooks;
-        this.postPageCloseHooks = postPageCloseHooks;
+        this.#preLaunchHooks = preLaunchHooks;
+        this.#postLaunchHooks = postLaunchHooks;
+        this.#prePageCreateHooks = prePageCreateHooks;
+        this.#postPageCreateHooks = postPageCreateHooks;
+        this.#prePageCloseHooks = prePageCloseHooks;
+        this.#postPageCloseHooks = postPageCloseHooks;
         // fingerprinting
-        if (this.useFingerprints) {
-            this._initializeFingerprinting();
+        if (useFingerprints) {
+            this.#initializeFingerprinting();
+            // The fingerprint pre-launch hook goes last because of the fingerprint cache.
+            // It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
+            this.#preLaunchHooks = [...this.#preLaunchHooks, createFingerprintPreLaunchHook(this)];
+            this.#prePageCreateHooks = [createPrePageCreateHook(), ...this.#prePageCreateHooks];
+            this.#postPageCreateHooks = [
+                createPostPageCreateHook(this.fingerprintInjector),
+                ...this.#postPageCreateHooks,
+            ];
         }
     }
     /**
@@ -163,7 +182,7 @@ export class BrowserPool extends TypedEmitter {
      * crawler's responsibility.
      */
     async newPage(options = {}) {
-        const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
+        const { id = nanoid(), pageOptions, browserPlugin = this.#pickBrowserPlugin(), session, proxyUrl = session?.proxyInfo?.url, ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors, } = options;
         if (this.pages.has(id)) {
             throw new Error(`Page with ID: ${id} already exists.`);
         }
@@ -176,18 +195,21 @@ export class BrowserPool extends TypedEmitter {
         // requests (https://github.com/apify/crawlee/issues/3670). Mirrors the
         // fix p-limit landed upstream in v5 (sindresorhus/p-limit#71); v5 is an
         // ESM-only rewrite, so we can't bump it in Crawlee v3.
+        // Besides the cancelTask leak, the wrapper also keeps the per-request *storage transaction*
+        // ALS-scoped: without it, a queued callback would resume in the previous request's async
+        // context and run request B's storage writes inside request A's transaction.
         // TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.
         // Limiter is necessary - https://github.com/apify/crawlee/issues/1126
-        return this.limiter(AsyncResource.bind(async () => {
-            let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
+        return this.#limiter(AsyncResource.bind(async () => {
+            let browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });
             if (!browserController)
-                browserController = await this._launchBrowser(id, {
+                browserController = await this.#launchBrowser(id, {
                     browserPlugin,
                     proxyUrl,
                     ignoreTlsErrors,
                 });
             tryCancel();
-            return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
+            return await this.#createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);
         }));
     }
     /**
@@ -196,13 +218,13 @@ export class BrowserPool extends TypedEmitter {
      * configure the new browser.
      */
     async newPageInNewBrowser(options = {}) {
-        const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;
+        const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.#pickBrowserPlugin() } = options;
         if (this.pages.has(id)) {
             throw new Error(`Page with ID: ${id} already exists.`);
         }
-        const browserController = await this._launchBrowser(id, { launchOptions, browserPlugin });
+        const browserController = await this.#launchBrowser(id, { launchOptions, browserPlugin });
         tryCancel();
-        return await this._createPageForBrowser(id, browserController, pageOptions);
+        return await this.#createPageForBrowser(id, browserController, pageOptions);
     }
     /**
      * Opens new pages with all available plugins and returns an array
@@ -248,7 +270,7 @@ export class BrowserPool extends TypedEmitter {
      * @param page - Browser plugin page
      */
     getBrowserControllerByPage(page) {
-        return this.pageToBrowserController.get(page);
+        return this.#pageToBrowserController.get(page);
     }
     /**
      * If you provided a custom ID to one of your pages or saved the
@@ -266,13 +288,12 @@ export class BrowserPool extends TypedEmitter {
      * until it's closed.
      */
     getPageId(page) {
-        return this.pageIds.get(page);
+        return this.#pageIds.get(page);
     }
-    async _createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
+    async #createPageForBrowser(pageId, browserController, pageOptions = {}, proxyUrl, ignoreTlsErrors) {
         // This is needed for concurrent newPage calls to wait for the browser launch.
         // It's not ideal though, we need to come up with a better API.
-        // eslint-disable-next-line dot-notation -- accessing private property
-        await browserController['isActivePromise'];
+        await browserController.waitForActive();
         tryCancel();
         const finalPageOptions = browserController.launchContext.useIncognitoPages ? pageOptions : undefined;
         if (finalPageOptions) {
@@ -284,28 +305,28 @@ export class BrowserPool extends TypedEmitter {
                 });
             }
         }
-        await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);
+        await this.#executeHooks(this.#prePageCreateHooks, pageId, browserController, finalPageOptions);
         tryCancel();
         let page;
         try {
-            page = (await addTimeoutToPromise(async () => browserController.newPage(finalPageOptions), this.operationTimeoutMillis, 'browserController.newPage() timed out.'));
+            page = (await addTimeoutToPromise(async () => browserController.newPage(finalPageOptions), this.#operationTimeoutMillis, 'browserController.newPage() timed out.'));
             tryCancel();
             this.pages.set(pageId, page);
-            this.pageIds.set(page, pageId);
-            this.pageToBrowserController.set(page, browserController);
+            this.#pageIds.set(page, pageId);
+            this.#pageToBrowserController.set(page, browserController);
             // if you synchronously trigger a lot of page launches, browser will not get retired soon enough. Not sure if it's a problem, let's monitor it.
-            if (browserController.totalPages >= this.retireBrowserAfterPageCount) {
+            if (browserController.totalPages >= this.#retireBrowserAfterPageCount) {
                 this.retireBrowserController(browserController);
             }
-            this._overridePageClose(page);
+            this.#overridePageClose(page);
         }
         catch (err) {
             this.retireBrowserController(browserController);
             throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${err.message}.`);
         }
-        await this._executeHooks(this.postPageCreateHooks, page, browserController);
+        await this.#executeHooks(this.#postPageCreateHooks, page, browserController);
         tryCancel();
-        this.emit("pageCreated" /* BROWSER_POOL_EVENTS.PAGE_CREATED */, page);
+        this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
         return page;
     }
     /**
@@ -314,14 +335,14 @@ export class BrowserPool extends TypedEmitter {
      *
      */
     retireBrowserController(browserController) {
-        const isStarting = this.startingBrowserControllers.has(browserController);
+        const isStarting = this.#startingBrowserControllers.has(browserController);
         const isActive = this.activeBrowserControllers.has(browserController);
         const hasBeenRetiredOrKilled = !isStarting && !isActive;
         if (hasBeenRetiredOrKilled)
             return;
         this.retiredBrowserControllers.add(browserController);
-        this.emit("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, browserController);
-        this.startingBrowserControllers.delete(browserController);
+        this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
+        this.#startingBrowserControllers.delete(browserController);
         this.activeBrowserControllers.delete(browserController);
     }
     /**
@@ -335,8 +356,9 @@ export class BrowserPool extends TypedEmitter {
     }
     /**
      * Releases a page back to the pool. The page is closed and, if the
-     * optional `error` is a {@link SessionError}, the browser controller
-     * that served the page is retired so that its tainted state (cookies,
+     * optional `error` is a {@link SessionError} (including
+     * {@link SessionRetiredError}), the browser controller that served
+     * the page is retired so that the finished session's state (cookies,
      * storage, etc.) cannot leak into future sessions.
      *
      * This is the primary way the crawler should return pages to the pool.
@@ -348,8 +370,10 @@ export class BrowserPool extends TypedEmitter {
         if (options?.error instanceof SessionError) {
             this.retireBrowserByPage(page);
         }
-        // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
-        await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
+        // The bound on this lives in `overridePageClose`, where it covers the pool's own
+        // bookkeeping as well. Wrapping it here instead would time out the override and abandon
+        // that bookkeeping, leaving the browser holding a slot it can never get back.
+        await page.close();
     }
     /**
      * Extracts the relevant state (currently just cookies) from a page via its
@@ -391,7 +415,7 @@ export class BrowserPool extends TypedEmitter {
      * closed after all their pages are closed.
      */
     retireAllBrowsers() {
-        [...this.startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
+        [...this.#startingBrowserControllers, ...this.activeBrowserControllers].forEach((controller) => {
             this.retireBrowserController(controller);
         });
     }
@@ -400,40 +424,48 @@ export class BrowserPool extends TypedEmitter {
      * @return {Promise<void>}
      */
     async closeAllBrowsers() {
-        const controllers = this._getAllBrowserControllers();
+        const controllers = this.#getAllBrowserControllers();
         const promises = [...controllers]
             .filter((controller) => controller.isActive)
             .map(async (controller) => controller.close());
         await Promise.all(promises);
     }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
+    }
     /**
-     * Closes all managed browsers and tears down the pool.
+     * Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
+     * browsers when a run ends and may start another run on the same pool.
      */
-    async destroy() {
-        clearInterval(this.browserKillerInterval);
-        clearInterval(this.browserRetireInterval);
-        this.browserKillerInterval = undefined;
-        this.browserRetireInterval = undefined;
+    async releaseAllBrowsers() {
         await this.closeAllBrowsers();
-        this._teardown();
-    }
-    _teardown() {
-        this.startingBrowserControllers.clear();
+        this.#startingBrowserControllers.clear();
         this.activeBrowserControllers.clear();
         this.retiredBrowserControllers.clear();
+    }
+    /**
+     * Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
+     * re-armed and its listeners are dropped, so it cannot be used again.
+     */
+    async destroy() {
+        clearInterval(this.browserKillerInterval);
+        clearInterval(this.#browserRetireInterval);
+        this.browserKillerInterval = undefined;
+        this.#browserRetireInterval = undefined;
+        await this.releaseAllBrowsers();
         this.removeAllListeners();
     }
-    _getAllBrowserControllers() {
+    #getAllBrowserControllers() {
         return new Set([
-            ...this.startingBrowserControllers,
+            ...this.#startingBrowserControllers,
             ...this.activeBrowserControllers,
             ...this.retiredBrowserControllers,
         ]);
     }
-    async _launchBrowser(pageId, options) {
+    async #launchBrowser(pageId, options) {
         const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;
         const browserController = browserPlugin.createController();
-        this.startingBrowserControllers.add(browserController);
+        this.#startingBrowserControllers.add(browserController);
         const launchContext = browserPlugin.createLaunchContext({
             id: pageId,
             launchOptions,
@@ -451,27 +483,27 @@ export class BrowserPool extends TypedEmitter {
         try {
             // If the hooks or the launch fails, we need to delete the controller,
             // because otherwise it would be stuck in limbo without a browser.
-            await this._executeHooks(this.preLaunchHooks, pageId, launchContext);
+            await this.#executeHooks(this.#preLaunchHooks, pageId, launchContext);
             tryCancel();
             const browser = await browserPlugin.launch(launchContext);
             tryCancel();
             browserController.assignBrowser(browser, launchContext);
         }
         catch (err) {
-            this.startingBrowserControllers.delete(browserController);
+            this.#startingBrowserControllers.delete(browserController);
             throw err;
         }
-        this.log.debug('Launched new browser.', { id: browserController.id });
+        this.#log.debug('Launched new browser.', { id: browserController.id });
         browserController.proxyUrl = proxyUrl;
         try {
             // If the launch fails on the post-launch hooks, we need to clean up
             // both the controller and the browser before throwing.
-            await this._executeHooks(this.postLaunchHooks, pageId, browserController);
+            await this.#executeHooks(this.#postLaunchHooks, pageId, browserController);
         }
         catch (err) {
-            this.startingBrowserControllers.delete(browserController);
+            this.#startingBrowserControllers.delete(browserController);
             browserController.close().catch((closeErr) => {
-                this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
+                this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
                     id: browserController.id,
                 });
             });
@@ -479,23 +511,22 @@ export class BrowserPool extends TypedEmitter {
         }
         tryCancel();
         browserController.activate();
-        this.startingBrowserControllers.delete(browserController);
+        this.#startingBrowserControllers.delete(browserController);
         this.activeBrowserControllers.add(browserController);
-        this.emit("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, browserController);
+        this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
         return browserController;
     }
     /**
      * Picks plugins round robin.
-     * @private
      */
-    _pickBrowserPlugin() {
-        const pluginIndex = this.pageCounter % this.browserPlugins.length;
-        this.pageCounter++;
+    #pickBrowserPlugin() {
+        const pluginIndex = this.#pageCounter % this.browserPlugins.length;
+        this.#pageCounter++;
         return this.browserPlugins[pluginIndex];
     }
-    _pickBrowserWithFreeCapacity(browserPlugin, options) {
+    #pickBrowserWithFreeCapacity(browserPlugin, options) {
         return [...this.activeBrowserControllers].find((controller) => {
-            const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
+            const hasCapacity = controller.activePages < this.#maxOpenPagesPerBrowser;
             const isCorrectPlugin = controller.browserPlugin === browserPlugin;
             const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
             return (isCorrectPlugin &&
@@ -505,53 +536,88 @@ export class BrowserPool extends TypedEmitter {
                     (!options?.proxyUrl && !controller.proxyUrl)));
         });
     }
-    async _closeInactiveRetiredBrowsers() {
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
+    async closeInactiveRetiredBrowsers() {
         const closedBrowserIds = [];
         for (const controller of this.retiredBrowserControllers) {
             const millisSinceLastPageOpened = Date.now() - controller.lastPageOpenedAt;
-            const isBrowserIdle = millisSinceLastPageOpened >= this.closeInactiveBrowserAfterMillis;
+            const isBrowserIdle = millisSinceLastPageOpened >= this.#closeInactiveBrowserAfterMillis;
             const isBrowserEmpty = controller.activePages === 0;
             if (isBrowserIdle || isBrowserEmpty) {
                 const { id } = controller;
-                this.log.debug('Closing retired browser.', { id });
+                this.#log.debug('Closing retired browser.', { id });
                 await controller.close();
                 this.retiredBrowserControllers.delete(controller);
                 closedBrowserIds.push(id);
             }
         }
         if (closedBrowserIds.length) {
-            this.log.debug('Closed retired browsers.', {
+            this.#log.debug('Closed retired browsers.', {
                 count: closedBrowserIds.length,
                 closedBrowserIds,
             });
         }
     }
-    _overridePageClose(page) {
+    #overridePageClose(page) {
         const originalPageClose = page.close;
-        const browserController = this.pageToBrowserController.get(page);
+        const browserController = this.#pageToBrowserController.get(page);
         const pageId = this.getPageId(page);
         page.close = async (...args) => {
-            await this._executeHooks(this.prePageCloseHooks, page, browserController);
-            await originalPageClose.apply(page, args).catch((err) => {
-                this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
-            });
-            await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
+            // A browser can acknowledge the close and then never destroy the target, in which case
+            // this never settles and the page's own `close` event never fires either (Chromium
+            // 536385539). Bound the whole sequence so nothing here can hang the caller, then
+            // reconcile the pool regardless of the outcome, so a page that refuses to close cannot
+            // leave the pool believing it is still open. `Promise.race` rather than
+            // `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
+            // so a timeout here would cancel the task of whoever awaited `page.close()`.
+            let pageClosed = false;
+            const closing = (async () => {
+                await this.#executeHooks(this.#prePageCloseHooks, page, browserController);
+                await originalPageClose.apply(page, args).catch((err) => {
+                    this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                });
+                // Whether the page itself is gone. Tracked separately from the sequence finishing,
+                // so that a slow hook does not get the browser retired.
+                pageClosed = true;
+                await this.#executeHooks(this.#postPageCloseHooks, pageId, browserController);
+            })();
+            let timeout;
+            const finished = await Promise.race([
+                closing.then(() => true, (err) => {
+                    this.#log.warning(`Closing a page failed, releasing it from the pool anyway.\nCause:${err.message}`, { id: browserController.id, pageId });
+                    return true;
+                }),
+                new Promise((resolve) => {
+                    timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
+                }),
+            ]);
+            clearTimeout(timeout);
+            if (!finished) {
+                this.#log.warning(`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
+                    'releasing it from the pool anyway.', { id: browserController.id, pageId });
+            }
+            if (!pageClosed) {
+                // The page is still attached, so this browser cannot be trusted with more work.
+                // Retiring it lets the reclamation below close the process once its pages drain.
+                this.retireBrowserController(browserController);
+            }
+            browserController.registerPageClosed(page);
             this.pages.delete(pageId);
-            this._closeRetiredBrowserWithNoPages(browserController);
-            this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);
+            this.#closeRetiredBrowserWithNoPages(browserController);
+            this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
         };
     }
-    async _executeHooks(hooks, ...args) {
+    async #executeHooks(hooks, ...args) {
         for (const hook of hooks) {
             await hook(...args);
         }
     }
-    _closeRetiredBrowserWithNoPages(browserController) {
+    #closeRetiredBrowserWithNoPages(browserController) {
         if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {
             // Run this with a delay, otherwise page.close()
             // might fail with "Protocol error (Target.closeTarget): Target closed."
             setTimeout(() => {
-                this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
+                this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
                 void browserController.close().finally(() => {
                     this.retiredBrowserControllers.delete(browserController);
                 });
@@ -559,42 +625,40 @@ export class BrowserPool extends TypedEmitter {
         }
     }
     /**
-     * Returns `true` if the pool can accept a new browser launch without exceeding
-     * {@link BrowserPoolOptions.maxOpenBrowsers}. Counts starting, active, and retired browsers.
+     * Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
+     * Counts starting, active, and retired browsers.
+     *
+     * A plain `BrowserPool` leaves `maxOpenBrowsers` at `Infinity`, so this only returns `false` when something
+     * has set a cap — {@link RemoteBrowserPool} does, from its own
+     * {@link RemoteBrowserPoolOptions.maxOpenBrowsers|`maxOpenBrowsers`} option. There is no
+     * `BrowserPoolOptions` key for it.
+     *
+     * @internal
      */
     hasFreeBrowserSlot() {
-        const total = this.startingBrowserControllers.size +
+        const total = this.#startingBrowserControllers.size +
             this.activeBrowserControllers.size +
             this.retiredBrowserControllers.size;
         return total < this.maxOpenBrowsers;
     }
     /**
      * Returns `true` if any active browser has room for another page.
+     *
+     * @internal
      */
     hasActiveBrowserWithFreeCapacity() {
         for (const controller of this.activeBrowserControllers) {
-            if (controller.activePages < this.maxOpenPagesPerBrowser)
+            if (controller.activePages < this.#maxOpenPagesPerBrowser)
                 return true;
         }
         return false;
     }
-    _initializeFingerprinting() {
+    #initializeFingerprinting() {
         const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;
         this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);
         this.fingerprintInjector = new FingerprintInjector();
         if (useFingerprintCache) {
             this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });
         }
-        this._addFingerprintHooks();
-    }
-    _addFingerprintHooks() {
-        this.preLaunchHooks = [
-            ...this.preLaunchHooks,
-            // This is flipped because of the fingerprint cache.
-            // It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
-            createFingerprintPreLaunchHook(this),
-        ];
-        this.prePageCreateHooks = [createPrePageCreateHook(), ...this.prePageCreateHooks];
-        this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector), ...this.postPageCreateHooks];
     }
 }

package/events.d.ts

@@ -1,10 +1,9 @@
-export declare const enum BROWSER_POOL_EVENTS {
+export declare enum BROWSER_POOL_EVENTS {
     BROWSER_LAUNCHED = "browserLaunched",
     BROWSER_RETIRED = "browserRetired",
-    BROWSER_CLOSED = "browserClosed",
     PAGE_CREATED = "pageCreated",
     PAGE_CLOSED = "pageClosed"
 }
-export declare const enum BROWSER_CONTROLLER_EVENTS {
+export declare enum BROWSER_CONTROLLER_EVENTS {
     BROWSER_CLOSED = "browserClosed"
 }

package/events.js

@@ -2,7 +2,6 @@ export var BROWSER_POOL_EVENTS;
 (function (BROWSER_POOL_EVENTS) {
     BROWSER_POOL_EVENTS["BROWSER_LAUNCHED"] = "browserLaunched";
     BROWSER_POOL_EVENTS["BROWSER_RETIRED"] = "browserRetired";
-    BROWSER_POOL_EVENTS["BROWSER_CLOSED"] = "browserClosed";
     BROWSER_POOL_EVENTS["PAGE_CREATED"] = "pageCreated";
     BROWSER_POOL_EVENTS["PAGE_CLOSED"] = "pageClosed";
 })(BROWSER_POOL_EVENTS || (BROWSER_POOL_EVENTS = {}));