@crawlee/browser-pool 4.0.0-rc.0 → 4.0.0-rc.1

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/browser-pool",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -30,9 +30,10 @@
         "copy": "tsx ../../scripts/copy.ts"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.4",
-        "@crawlee/core": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/core": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "fingerprint-generator": "^2.1.68",
         "fingerprint-injector": "^2.1.68",
         "lodash.merge": "^4.6.2",
@@ -42,7 +43,7 @@
         "quick-lru": "^7.0.1",
         "tiny-typed-emitter": "^2.1.0",
         "tslib": "^2.8.1",
-        "zod": "^4.4.3"
+        "zod": "^4.5.4"
     },
     "peerDependencies": {
         "playwright": "*",
@@ -63,5 +64,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/playwright/playwright-browser.d.ts

@@ -10,6 +10,7 @@ export interface BrowserOptions {
  */
 export declare class PlaywrightBrowser extends EventEmitter {
     #private;
+    /** @internal */
     constructor(options: BrowserOptions);
     [Symbol.asyncDispose](): Promise<void>;
     close(): Promise<void>;

package/playwright/playwright-browser.js

@@ -7,6 +7,7 @@ export class PlaywrightBrowser extends EventEmitter {
     #version;
     #isConnected = true;
     #browserType;
+    /** @internal */
     constructor(options) {
         super();
         const { browserContext, version } = options;

package/playwright/playwright-controller.d.ts

@@ -4,6 +4,7 @@ import type { Browser, BrowserType, Page } from 'playwright';
 import { BrowserController } from '../abstract-classes/browser-controller.js';
 import type { SafeParameters } from '../utils.js';
 export declare class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
+    /** @internal */
     normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
     protected _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page>;
     protected _close(): Promise<void>;

package/playwright/playwright-controller.js

@@ -2,6 +2,7 @@ import { tryCancel } from '@apify/timeout';
 import { BrowserController } from '../abstract-classes/browser-controller.js';
 import { anonymizeProxySugar } from '../anonymize-proxy.js';
 export class PlaywrightController extends BrowserController {
+    /** @internal */
     normalizeProxyOptions(proxyUrl, pageOptions) {
         if (!proxyUrl) {
             return {};
@@ -46,9 +47,9 @@ export class PlaywrightController extends BrowserController {
         }
         try {
             const page = await this.browser.newPage(contextOptions);
-            page.once('close', async () => {
-                this.activePages--;
-                await close();
+            this.registerPageTeardown(page, close);
+            page.once('close', () => {
+                this.registerPageClosed(page);
             });
             tryCancel();
             return page;

package/playwright/playwright-plugin.d.ts

@@ -10,10 +10,11 @@ export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafePar
     /**
      * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
      * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
+     *
+     * @internal
      */
     useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
     protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
-    private throwOnFailedLaunch;
     createController(): PlaywrightController;
     protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
     protected isChromiumBasedBrowser(): boolean;

package/playwright/playwright-plugin.js

@@ -9,6 +9,8 @@ export class PlaywrightPlugin extends BrowserPlugin {
     /**
      * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
      * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
+     *
+     * @internal
      */
     useRemoteConnection(connection, parameters = {}) {
         super.useRemoteConnection(connection, parameters);
@@ -53,7 +55,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
         try {
             if (useIncognitoPages) {
                 browser = await this.library.launch(launchOptions).catch((error) => {
-                    return this.throwOnFailedLaunch(launchContext, error);
+                    return this.#throwOnFailedLaunch(launchContext, error);
                 });
                 if (anonymizedProxyUrl) {
                     browser.on('disconnected', async () => {
@@ -65,7 +67,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
                 const browserContext = await this.library
                     .launchPersistentContext(userDataDir, launchOptions)
                     .catch((error) => {
-                    return this.throwOnFailedLaunch(launchContext, error);
+                    return this.#throwOnFailedLaunch(launchContext, error);
                 });
                 browserContext.once('close', () => {
                     if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -102,7 +104,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
         }
         return browser;
     }
-    throwOnFailedLaunch(launchContext, cause) {
+    #throwOnFailedLaunch(launchContext, cause) {
         this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
     }
     createController() {

package/puppeteer/puppeteer-controller.d.ts

@@ -9,6 +9,7 @@ export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOp
     proxyPassword?: string;
 }
 export declare class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
+    /** @internal */
     normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
     protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
     protected _close(): Promise<void>;

package/puppeteer/puppeteer-controller.js

@@ -3,6 +3,7 @@ import { BrowserController } from '../abstract-classes/browser-controller.js';
 import { anonymizeProxySugar } from '../anonymize-proxy.js';
 const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
 export class PuppeteerController extends BrowserController {
+    /** @internal */
     normalizeProxyOptions(proxyUrl, pageOptions) {
         if (!proxyUrl) {
             return {};
@@ -51,17 +52,18 @@ export class PuppeteerController extends BrowserController {
                     tryCancel();
                 }
                 */
-                page.once('close', async () => {
-                    this.activePages--;
-                    try {
-                        await context.close();
-                    }
-                    catch (error) {
+                this.registerPageTeardown(page, async () => {
+                    // The proxy server is not chained behind the context close: that can hang on
+                    // the same target the page hung on, and the proxy runs in this process, so it
+                    // has to be reclaimed either way.
+                    const contextClosed = context.close().catch((error) => {
                         this.log.exception(error, 'Failed to close context.');
-                    }
-                    finally {
-                        await close();
-                    }
+                    });
+                    await close();
+                    await contextClosed;
+                });
+                page.once('close', () => {
+                    this.registerPageClosed(page);
                 });
                 return page;
             }
@@ -73,7 +75,7 @@ export class PuppeteerController extends BrowserController {
         const page = await this.browser.newPage();
         tryCancel();
         page.once('close', () => {
-            this.activePages--;
+            this.registerPageClosed(page);
         });
         return page;
     }

package/puppeteer/puppeteer-plugin.d.ts

@@ -8,7 +8,11 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
 import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
 import { PuppeteerController } from './puppeteer-controller.js';
 export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
-    /** Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito). */
+    /**
+     * Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
+     *
+     * @internal
+     */
     useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
     protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
     createController(): PuppeteerController;

package/puppeteer/puppeteer-plugin.js

@@ -5,7 +5,11 @@ import { noop } from '../utils.js';
 import { PuppeteerController } from './puppeteer-controller.js';
 const PROXY_SERVER_ARG = '--proxy-server=';
 export class PuppeteerPlugin extends BrowserPlugin {
-    /** Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito). */
+    /**
+     * Pages share cookies/storage on the remote browser (Puppeteer defaults to non-incognito).
+     *
+     * @internal
+     */
     useRemoteConnection(connection, parameters = {}) {
         super.useRemoteConnection(connection, parameters);
         if (!this.useIncognitoPages) {

package/remote-browser-pool.d.ts

@@ -91,7 +91,11 @@ export interface RemoteBrowserPoolOptions {
     connection?: RemoteConnectionParameters;
     /** Extra {@link BrowserPool} options (lifecycle hooks, page limits, fingerprinting, …). */
     browserPoolOptions?: Omit<BrowserPoolOptions, 'browserPlugins'> & BrowserPoolHooks<any, any, any>;
-    /** Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it. @default 500 */
+    /**
+     * Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it.
+     * @default 500
+     * @internal
+     */
     slotPollIntervalMillis?: number;
 }
 /**
@@ -135,7 +139,11 @@ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browse
  */
 export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
     #private;
-    /** The wrapped pool that performs the remote connections and serves pages. */
+    /**
+     * The wrapped pool that performs the remote connections and serves pages.
+     *
+     * @internal
+     */
     readonly browserPool: BrowserPool;
     constructor(options: RemoteBrowserPoolOptions);
     /** Maximum number of remote browsers that may be open at the same time. */
@@ -151,14 +159,12 @@ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<P
     }): Promise<void>;
     extractPageState(page: Page): Promise<PageState>;
     injectPageState(page: Page, state: PageState): Promise<void>;
-    /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
-    destroy(): Promise<void>;
-    /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
-    private waitForFreeSlot;
     /**
-     * Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
-     * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
-     * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
+     * Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
+     * releases its session (see the constructor), so this needs no separate release pass.
      */
-    private nextCapacityChange;
+    releaseAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
+    destroy(): Promise<void>;
 }

package/remote-browser-pool.js

@@ -90,7 +90,11 @@ class RemoteSessionRegistry {
  * @category Browser management
  */
 export class RemoteBrowserPool {
-    /** The wrapped pool that performs the remote connections and serves pages. */
+    /**
+     * The wrapped pool that performs the remote connections and serves pages.
+     *
+     * @internal
+     */
     browserPool;
     /** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
     #pool;
@@ -144,7 +148,7 @@ export class RemoteBrowserPool {
      * allows it (either a new browser slot is free, or an active browser still has page capacity).
      */
     async newPage(options) {
-        await this.waitForFreeSlot();
+        await this.#waitForFreeSlot();
         return this.#pool.newPage(options);
     }
     async closePage(page, options) {
@@ -156,6 +160,16 @@ export class RemoteBrowserPool {
     async injectPageState(page, state) {
         return this.#pool.injectPageState(page, state);
     }
+    /**
+     * Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
+     * releases its session (see the constructor), so this needs no separate release pass.
+     */
+    async releaseAllBrowsers() {
+        await this.browserPool.releaseAllBrowsers();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
+    }
     /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
     async destroy() {
         await this.browserPool.destroy();
@@ -163,9 +177,9 @@ export class RemoteBrowserPool {
         await this.#registry.releaseAll();
     }
     /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
-    async waitForFreeSlot() {
+    async #waitForFreeSlot() {
         while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {
-            await this.nextCapacityChange();
+            await this.#nextCapacityChange();
         }
     }
     /**
@@ -173,7 +187,7 @@ export class RemoteBrowserPool {
      * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
      * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
      */
-    nextCapacityChange() {
+    #nextCapacityChange() {
         this.#capacityChange ??= new Promise((resolve) => {
             const done = () => {
                 clearTimeout(timer);

package/container-proxy-server.d.ts

@@ -1,11 +0,0 @@
-/**
- * Creates a proxy server designed to handle requests from "container" instances.
- * Each container instance is assigned to a different (but still localhost) IP address
- * in order to work around authorization and to enable upstream.
- * @internal
- */
-export declare function createProxyServerForContainers(fallbackProxyUrl?: string): Promise<{
-    port: number;
-    ipToProxy: Map<string, string>;
-    close(closeConnections: boolean): Promise<void>;
-}>;

package/container-proxy-server.js

@@ -1,42 +0,0 @@
-import { Server as ProxyChainServer } from 'proxy-chain';
-/**
- * Creates a proxy server designed to handle requests from "container" instances.
- * Each container instance is assigned to a different (but still localhost) IP address
- * in order to work around authorization and to enable upstream.
- * @internal
- */
-export async function createProxyServerForContainers(fallbackProxyUrl) {
-    const ipToProxy = new Map();
-    const proxyServer = new ProxyChainServer({
-        prepareRequestFunction({ request }) {
-            const prefix4to6 = '::ffff:';
-            const localAddress = request.socket.localAddress.startsWith(prefix4to6)
-                ? request.socket.localAddress.slice(prefix4to6.length)
-                : request.socket.localAddress;
-            const upstreamProxyUrl = ipToProxy.get(localAddress);
-            if (upstreamProxyUrl === undefined) {
-                if (fallbackProxyUrl) {
-                    return {
-                        upstreamProxyUrl: fallbackProxyUrl,
-                        requestAuthentication: false,
-                    };
-                }
-                console.warn(`Request without proxy ${localAddress} ${request.headers.host}`);
-            }
-            return {
-                upstreamProxyUrl,
-                requestAuthentication: false,
-            };
-        },
-        port: 0,
-    });
-    await proxyServer.listen();
-    proxyServer.server.unref();
-    return {
-        port: proxyServer.port,
-        ipToProxy,
-        async close(closeConnections) {
-            return proxyServer.close(closeConnections);
-        },
-    };
-}