@crawlee/browser-pool 4.0.0-beta.170 → 4.0.0-beta.172

package/abstract-classes/browser-controller.d.ts

@@ -138,6 +138,23 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page: NewPageResult): void;
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
     setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
     getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**

package/abstract-classes/browser-controller.js

@@ -34,6 +34,8 @@ export class BrowserController extends TypedEmitter {
     proxyUrl;
     isActive = false;
     activePages = 0;
+    #closedPages = new WeakSet();
+    #pageTeardowns = new WeakMap();
     totalPages = 0;
     lastPageOpenedAt = Date.now();
     #activate;
@@ -126,6 +128,37 @@ export class BrowserController extends TypedEmitter {
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page) {
+        if (this.#closedPages.has(page))
+            return;
+        this.#closedPages.add(page);
+        this.activePages--;
+        const teardown = this.#pageTeardowns.get(page);
+        if (!teardown)
+            return;
+        this.#pageTeardowns.delete(page);
+        teardown().catch((error) => {
+            this.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
+        });
+    }
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    registerPageTeardown(page, teardown) {
+        this.#pageTeardowns.set(page, teardown);
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }

package/browser-pool.js

@@ -353,8 +353,10 @@ export class BrowserPool extends TypedEmitter {
         if (options?.error instanceof SessionError) {
             this.retireBrowserByPage(page);
         }
-        // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
-        await addTimeoutToPromise(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`);
+        // The bound on this lives in `overridePageClose`, where it covers the pool's own
+        // bookkeeping as well. Wrapping it here instead would time out the override and abandon
+        // that bookkeeping, leaving the browser holding a slot it can never get back.
+        await page.close();
     }
     /**
      * Extracts the relevant state (currently just cookies) from a page via its
@@ -544,11 +546,45 @@ export class BrowserPool extends TypedEmitter {
         const browserController = this.pageToBrowserController.get(page);
         const pageId = this.getPageId(page);
         page.close = async (...args) => {
-            await this.executeHooks(this.prePageCloseHooks, page, browserController);
-            await originalPageClose.apply(page, args).catch((err) => {
-                this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
-            });
-            await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
+            // A browser can acknowledge the close and then never destroy the target, in which case
+            // this never settles and the page's own `close` event never fires either (Chromium
+            // 536385539). Bound the whole sequence so nothing here can hang the caller, then
+            // reconcile the pool regardless of the outcome, so a page that refuses to close cannot
+            // leave the pool believing it is still open. `Promise.race` rather than
+            // `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
+            // so a timeout here would cancel the task of whoever awaited `page.close()`.
+            let pageClosed = false;
+            const closing = (async () => {
+                await this.executeHooks(this.prePageCloseHooks, page, browserController);
+                await originalPageClose.apply(page, args).catch((err) => {
+                    this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                });
+                // Whether the page itself is gone. Tracked separately from the sequence finishing,
+                // so that a slow hook does not get the browser retired.
+                pageClosed = true;
+                await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
+            })();
+            let timeout;
+            const finished = await Promise.race([
+                closing.then(() => true, (err) => {
+                    this.#log.warning(`Closing a page failed, releasing it from the pool anyway.\nCause:${err.message}`, { id: browserController.id, pageId });
+                    return true;
+                }),
+                new Promise((resolve) => {
+                    timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
+                }),
+            ]);
+            clearTimeout(timeout);
+            if (!finished) {
+                this.#log.warning(`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
+                    'releasing it from the pool anyway.', { id: browserController.id, pageId });
+            }
+            if (!pageClosed) {
+                // The page is still attached, so this browser cannot be trusted with more work.
+                // Retiring it lets the reclamation below close the process once its pages drain.
+                this.retireBrowserController(browserController);
+            }
+            browserController.registerPageClosed(page);
             this.pages.delete(pageId);
             this.closeRetiredBrowserWithNoPages(browserController);
             this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser-pool",
-    "version": "4.0.0-beta.170",
+    "version": "4.0.0-beta.172",
     "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -31,9 +31,9 @@
     },
     "dependencies": {
         "@apify/timeout": "^1.0.1",
-        "@crawlee/core": "4.0.0-beta.170",
-        "@crawlee/types": "4.0.0-beta.170",
-        "@crawlee/utils": "4.0.0-beta.170",
+        "@crawlee/core": "4.0.0-beta.172",
+        "@crawlee/types": "4.0.0-beta.172",
+        "@crawlee/utils": "4.0.0-beta.172",
         "fingerprint-generator": "^2.1.68",
         "fingerprint-injector": "^2.1.68",
         "lodash.merge": "^4.6.2",
@@ -64,5 +64,5 @@
             }
         }
     },
-    "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
+    "gitHead": "619809c59ea633121f3aa9c8f333efeb9c6167af"
 }

package/playwright/playwright-controller.js

@@ -46,9 +46,9 @@ export class PlaywrightController extends BrowserController {
         }
         try {
             const page = await this.browser.newPage(contextOptions);
-            page.once('close', async () => {
-                this.activePages--;
-                await close();
+            this.registerPageTeardown(page, close);
+            page.once('close', () => {
+                this.registerPageClosed(page);
             });
             tryCancel();
             return page;

package/puppeteer/puppeteer-controller.js

@@ -51,17 +51,18 @@ export class PuppeteerController extends BrowserController {
                     tryCancel();
                 }
                 */
-                page.once('close', async () => {
-                    this.activePages--;
-                    try {
-                        await context.close();
-                    }
-                    catch (error) {
+                this.registerPageTeardown(page, async () => {
+                    // The proxy server is not chained behind the context close: that can hang on
+                    // the same target the page hung on, and the proxy runs in this process, so it
+                    // has to be reclaimed either way.
+                    const contextClosed = context.close().catch((error) => {
                         this.log.exception(error, 'Failed to close context.');
-                    }
-                    finally {
-                        await close();
-                    }
+                    });
+                    await close();
+                    await contextClosed;
+                });
+                page.once('close', () => {
+                    this.registerPageClosed(page);
                 });
                 return page;
             }
@@ -73,7 +74,7 @@ export class PuppeteerController extends BrowserController {
         const page = await this.browser.newPage();
         tryCancel();
         page.once('close', () => {
-            this.activePages--;
+            this.registerPageClosed(page);
         });
         return page;
     }