@crawlee/browser-pool 3.18.2-beta.4 → 3.18.2-beta.5

package/abstract-classes/browser-controller.d.ts

@@ -41,6 +41,8 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
     proxyUrl?: string;
     isActive: boolean;
     activePages: number;
+    private readonly closedPages;
+    private readonly pageTeardowns;
     totalPages: number;
     lastPageOpenedAt: number;
     private _activate;
@@ -77,6 +79,23 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page: NewPageResult): void;
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    protected registerPageTeardown(page: NewPageResult, teardown: () => Promise<void>): void;
     setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
     getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**

package/abstract-classes/browser-controller.js

@@ -82,6 +82,18 @@ class BrowserController extends tiny_typed_emitter_1.TypedEmitter {
             writable: true,
             value: 0
         });
+        Object.defineProperty(this, "closedPages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new WeakSet()
+        });
+        Object.defineProperty(this, "pageTeardowns", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new WeakMap()
+        });
         Object.defineProperty(this, "totalPages", {
             enumerable: true,
             configurable: true,
@@ -195,6 +207,37 @@ class BrowserController extends tiny_typed_emitter_1.TypedEmitter {
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @internal
+     */
+    registerPageClosed(page) {
+        if (this.closedPages.has(page))
+            return;
+        this.closedPages.add(page);
+        this.activePages--;
+        const teardown = this.pageTeardowns.get(page);
+        if (!teardown)
+            return;
+        this.pageTeardowns.delete(page);
+        teardown().catch((error) => {
+            logger_1.log.debug(`Could not clean up after a closed page.\nCause:${error.message}`, { id: this.id });
+        });
+    }
+    /**
+     * Registers cleanup belonging to a single page: an anonymizing proxy server, an incognito
+     * context. It runs from `registerPageClosed`, so it happens whether the page closed on its own
+     * or the pool gave up on a close that never settled, and it runs at most once.
+     *
+     * Hanging it off the page's `close` event instead leaks it in exactly the case this controller
+     * now handles, because that event does not arrive for a page the browser never destroyed.
+     * @ignore
+     */
+    registerPageTeardown(page, teardown) {
+        this.pageTeardowns.set(page, teardown);
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }

package/browser-pool.js

@@ -14,6 +14,7 @@ const timeout_1 = require("@apify/timeout");
 const hooks_1 = require("./fingerprinting/hooks");
 const logger_1 = require("./logger");
 const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
+const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
 const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
 /**
  * The `BrowserPool` class is the most important class of the `browser-pool` module.
@@ -582,11 +583,48 @@ class BrowserPool extends tiny_typed_emitter_1.TypedEmitter {
         const browserController = this.pageToBrowserController.get(page);
         const pageId = this.getPageId(page);
         page.close = async (...args) => {
-            await this._executeHooks(this.prePageCloseHooks, page, browserController);
-            await originalPageClose.apply(page, args).catch((err) => {
-                logger_1.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
-            });
-            await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
+            // A browser can acknowledge the close and then never destroy the target, in which case
+            // this never settles and the page's own `close` event never fires either (Chromium
+            // 536385539). Bound the whole sequence so nothing here can hang the caller, then
+            // reconcile the pool regardless of the outcome, so a page that refuses to close cannot
+            // leave the pool believing it is still open. `Promise.race` rather than
+            // `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
+            // so a timeout here would cancel the task of whoever awaited `page.close()`.
+            let pageClosed = false;
+            const closing = (async () => {
+                await this._executeHooks(this.prePageCloseHooks, page, browserController);
+                await originalPageClose.apply(page, args).catch((err) => {
+                    logger_1.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                });
+                // Whether the page itself is gone. Tracked separately from the sequence finishing,
+                // so that a slow hook does not get the browser retired.
+                pageClosed = true;
+                await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
+            })();
+            let timeout;
+            const finished = await Promise.race([
+                closing.then(() => true, (err) => {
+                    logger_1.log.warning(`Closing a page failed, releasing it from the pool anyway.\nCause:${err.message}`, {
+                        id: browserController.id,
+                        pageId,
+                    });
+                    return true;
+                }),
+                new Promise((resolve) => {
+                    timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
+                }),
+            ]);
+            clearTimeout(timeout);
+            if (!finished) {
+                logger_1.log.warning(`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
+                    'releasing it from the pool anyway.', { id: browserController.id, pageId });
+            }
+            if (!pageClosed) {
+                // The page is still attached, so this browser cannot be trusted with more work.
+                // Retiring it lets the reclamation below close the process once its pages drain.
+                this.retireBrowserController(browserController);
+            }
+            browserController.registerPageClosed(page);
             this.pages.delete(pageId);
             this._closeRetiredBrowserWithNoPages(browserController);
             this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser-pool",
-    "version": "3.18.2-beta.4",
+    "version": "3.18.2-beta.5",
     "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -38,8 +38,8 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.4.0",
-        "@crawlee/core": "3.18.2-beta.4",
-        "@crawlee/types": "3.18.2-beta.4",
+        "@crawlee/core": "3.18.2-beta.5",
+        "@crawlee/types": "3.18.2-beta.5",
         "fingerprint-generator": "^2.1.68",
         "fingerprint-injector": "^2.1.68",
         "lodash.merge": "^4.6.2",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "ee9640fb13d1414a03fcd21ccc5255bca3963aef"
+    "gitHead": "d126d83399f7aa71bd0920d6a50b6bb9e6373790"
 }

package/playwright/playwright-controller.js

@@ -49,9 +49,9 @@ class PlaywrightController extends browser_controller_1.BrowserController {
         }
         try {
             const page = await this.browser.newPage(contextOptions);
-            page.once('close', async () => {
-                this.activePages--;
-                await close();
+            this.registerPageTeardown(page, close);
+            page.once('close', () => {
+                this.registerPageClosed(page);
             });
             if (this.launchContext.experimentalContainers) {
                 await page.goto('data:text/plain,tabid');

package/puppeteer/puppeteer-controller.js

@@ -55,17 +55,18 @@ class PuppeteerController extends browser_controller_1.BrowserController {
                     tryCancel();
                 }
                 */
-                page.once('close', async () => {
-                    this.activePages--;
-                    try {
-                        await context.close();
-                    }
-                    catch (error) {
+                this.registerPageTeardown(page, async () => {
+                    // The proxy server is not chained behind the context close: that can hang on
+                    // the same target the page hung on, and the proxy runs in this process, so it
+                    // has to be reclaimed either way.
+                    const contextClosed = context.close().catch((error) => {
                         logger_1.log.exception(error, 'Failed to close context.');
-                    }
-                    finally {
-                        await close();
-                    }
+                    });
+                    await close();
+                    await contextClosed;
+                });
+                page.once('close', () => {
+                    this.registerPageClosed(page);
                 });
                 return page;
             }
@@ -77,7 +78,7 @@ class PuppeteerController extends browser_controller_1.BrowserController {
         const page = await this.browser.newPage();
         (0, timeout_1.tryCancel)();
         page.once('close', () => {
-            this.activePages--;
+            this.registerPageClosed(page);
         });
         return page;
     }