@crawlee/browser-pool 3.18.2-beta.1 → 3.18.2-beta.3

package/abstract-classes/browser-controller.d.ts

@@ -41,6 +41,7 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
     proxyUrl?: string;
     isActive: boolean;
     activePages: number;
+    private readonly closedPages;
     totalPages: number;
     lastPageOpenedAt: number;
     private _activate;
@@ -77,6 +78,13 @@ export declare abstract class BrowserController<Library extends CommonLibrary =
      * @ignore
      */
     newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @ignore
+     */
+    registerPageClosed(page: NewPageResult): void;
     setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
     getCookies(page: NewPageResult): Promise<Cookie[]>;
     /**

package/abstract-classes/browser-controller.js

@@ -82,6 +82,12 @@ class BrowserController extends tiny_typed_emitter_1.TypedEmitter {
             writable: true,
             value: 0
         });
+        Object.defineProperty(this, "closedPages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new WeakSet()
+        });
         Object.defineProperty(this, "totalPages", {
             enumerable: true,
             configurable: true,
@@ -195,6 +201,18 @@ class BrowserController extends tiny_typed_emitter_1.TypedEmitter {
         this.lastPageOpenedAt = Date.now();
         return page;
     }
+    /**
+     * Accounts for a page no longer being open. Callable from either side and safe to call twice:
+     * the page's own `close` event, or the pool when it gives up on a close that never settles.
+     * Tracked in a `WeakSet` rather than via `isClosed()`, because that reports the same event.
+     * @ignore
+     */
+    registerPageClosed(page) {
+        if (this.closedPages.has(page))
+            return;
+        this.closedPages.add(page);
+        this.activePages--;
+    }
     async setCookies(page, cookies) {
         return this._setCookies(page, cookies);
     }

package/browser-pool.js

@@ -14,6 +14,7 @@ const timeout_1 = require("@apify/timeout");
 const hooks_1 = require("./fingerprinting/hooks");
 const logger_1 = require("./logger");
 const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
+const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
 const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
 /**
  * The `BrowserPool` class is the most important class of the `browser-pool` module.
@@ -582,11 +583,48 @@ class BrowserPool extends tiny_typed_emitter_1.TypedEmitter {
         const browserController = this.pageToBrowserController.get(page);
         const pageId = this.getPageId(page);
         page.close = async (...args) => {
-            await this._executeHooks(this.prePageCloseHooks, page, browserController);
-            await originalPageClose.apply(page, args).catch((err) => {
-                logger_1.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
-            });
-            await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
+            // A browser can acknowledge the close and then never destroy the target, in which case
+            // this never settles and the page's own `close` event never fires either (Chromium
+            // 536385539). Bound the whole sequence so nothing here can hang the caller, then
+            // reconcile the pool regardless of the outcome, so a page that refuses to close cannot
+            // leave the pool believing it is still open. `Promise.race` rather than
+            // `addTimeoutToPromise`: the latter inherits its AbortController from the calling frame,
+            // so a timeout here would cancel the task of whoever awaited `page.close()`.
+            let pageClosed = false;
+            const closing = (async () => {
+                await this._executeHooks(this.prePageCloseHooks, page, browserController);
+                await originalPageClose.apply(page, args).catch((err) => {
+                    logger_1.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                });
+                // Whether the page itself is gone. Tracked separately from the sequence finishing,
+                // so that a slow hook does not get the browser retired.
+                pageClosed = true;
+                await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
+            })();
+            let timeout;
+            const finished = await Promise.race([
+                closing.then(() => true, (err) => {
+                    logger_1.log.warning(`Page close hook failed.\nCause:${err.message}`, {
+                        id: browserController.id,
+                        pageId,
+                    });
+                    return true;
+                }),
+                new Promise((resolve) => {
+                    timeout = setTimeout(() => resolve(false), PAGE_CLOSE_TIMEOUT_MILLIS);
+                }),
+            ]);
+            clearTimeout(timeout);
+            if (!finished) {
+                logger_1.log.warning(`Closing a page did not finish within ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds, ` +
+                    'releasing it from the pool anyway.', { id: browserController.id, pageId });
+            }
+            if (!pageClosed) {
+                // The page is still attached, so this browser cannot be trusted with more work.
+                // Retiring it lets the reclamation below close the process once its pages drain.
+                this.retireBrowserController(browserController);
+            }
+            browserController.registerPageClosed(page);
             this.pages.delete(pageId);
             this._closeRetiredBrowserWithNoPages(browserController);
             this.emit("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, page);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser-pool",
-    "version": "3.18.2-beta.1",
+    "version": "3.18.2-beta.3",
     "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -38,8 +38,8 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.4.0",
-        "@crawlee/core": "3.18.2-beta.1",
-        "@crawlee/types": "3.18.2-beta.1",
+        "@crawlee/core": "3.18.2-beta.3",
+        "@crawlee/types": "3.18.2-beta.3",
         "fingerprint-generator": "^2.1.68",
         "fingerprint-injector": "^2.1.68",
         "lodash.merge": "^4.6.2",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "5d2fea5f9d24029d1cc8435132556f4c93d2a20b"
+    "gitHead": "feed328453d1967b36ccd45cc4531d44498915b8"
 }

package/playwright/playwright-controller.js

@@ -50,7 +50,7 @@ class PlaywrightController extends browser_controller_1.BrowserController {
         try {
             const page = await this.browser.newPage(contextOptions);
             page.once('close', async () => {
-                this.activePages--;
+                this.registerPageClosed(page);
                 await close();
             });
             if (this.launchContext.experimentalContainers) {

package/puppeteer/puppeteer-controller.js

@@ -56,7 +56,7 @@ class PuppeteerController extends browser_controller_1.BrowserController {
                 }
                 */
                 page.once('close', async () => {
-                    this.activePages--;
+                    this.registerPageClosed(page);
                     try {
                         await context.close();
                     }
@@ -77,7 +77,7 @@ class PuppeteerController extends browser_controller_1.BrowserController {
         const page = await this.browser.newPage();
         (0, timeout_1.tryCancel)();
         page.once('close', () => {
-            this.activePages--;
+            this.registerPageClosed(page);
         });
         return page;
     }