@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.1

package/remote-browser-pool.d.ts

@@ -91,14 +91,18 @@ export interface RemoteBrowserPoolOptions {
     connection?: RemoteConnectionParameters;
     /** Extra {@link BrowserPool} options (lifecycle hooks, page limits, fingerprinting, …). */
     browserPoolOptions?: Omit<BrowserPoolOptions, 'browserPlugins'> & BrowserPoolHooks<any, any, any>;
-    /** Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it. @default 500 */
+    /**
+     * Fallback poll interval (ms) while waiting for a free browser slot. The wait is event-driven; this only bounds it.
+     * @default 500
+     * @internal
+     */
     slotPollIntervalMillis?: number;
 }
 /**
- * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. It is the
- * {@link RemoteBrowserPoolOptions} a user supplies *minus* the parts the crawler provides itself — the
- * `browserPlugins` (the crawler builds the correct one for its browser) and `browserPoolOptions` (taken from
- * the crawler's own `browserPoolOptions`). This is what makes the crawler path both terse and mismatch-proof.
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the
+ * {@link RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its
+ * browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` — tuning the wrapping
+ * pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.
  */
 export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
 /**
@@ -134,15 +138,13 @@ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browse
  * @category Browser management
  */
 export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
-    /** The wrapped pool that performs the remote connections and serves pages. */
+    #private;
+    /**
+     * The wrapped pool that performs the remote connections and serves pages.
+     *
+     * @internal
+     */
     readonly browserPool: BrowserPool;
-    /** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
-    private readonly pool;
-    private readonly registry;
-    private readonly slotPollIntervalMillis;
-    private readonly log;
-    /** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
-    private _capacityChange?;
     constructor(options: RemoteBrowserPoolOptions);
     /** Maximum number of remote browsers that may be open at the same time. */
     get maxOpenBrowsers(): number;
@@ -157,14 +159,12 @@ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<P
     }): Promise<void>;
     extractPageState(page: Page): Promise<PageState>;
     injectPageState(page: Page, state: PageState): Promise<void>;
-    /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
-    destroy(): Promise<void>;
-    /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
-    private _waitForFreeSlot;
     /**
-     * Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
-     * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
-     * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
+     * Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
+     * releases its session (see the constructor), so this needs no separate release pass.
      */
-    private _nextCapacityChange;
+    releaseAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
+    destroy(): Promise<void>;
 }

package/remote-browser-pool.js

@@ -1,5 +1,6 @@
 import { serviceLocator } from '@crawlee/core';
 import { BrowserPool } from './browser-pool.js';
+import { BROWSER_CONTROLLER_EVENTS, BROWSER_POOL_EVENTS } from './events.js';
 import { RemoteBrowserProvider } from './remote-browser-provider.js';
 /**
  * Owns the lifecycle of remote browser sessions for a single {@link RemoteBrowserPool}: endpoint
@@ -7,18 +8,18 @@ import { RemoteBrowserProvider } from './remote-browser-provider.js';
  * {@link RemoteConnection} so it can be injected into a plugin.
  */
 class RemoteSessionRegistry {
-    endpoint;
-    onRelease;
-    log;
-    sessions = new Map();
-    nextToken = 0;
+    #sessions = new Map();
+    #nextToken = 0;
+    #endpoint;
+    #onRelease;
+    #log;
     constructor(endpoint, onRelease, log) {
-        this.endpoint = endpoint;
-        this.onRelease = onRelease;
-        this.log = log;
+        this.#endpoint = endpoint;
+        this.#onRelease = onRelease;
+        this.#log = log;
     }
     async resolve(options) {
-        const resolved = typeof this.endpoint === 'function' ? await this.endpoint(options) : this.endpoint;
+        const resolved = typeof this.#endpoint === 'function' ? await this.#endpoint(options) : this.#endpoint;
         let result;
         if (typeof resolved === 'string') {
             if (!resolved)
@@ -31,30 +32,30 @@ class RemoteSessionRegistry {
         else {
             result = resolved;
         }
-        const token = this.nextToken++;
-        this.sessions.set(token, { url: result.url, context: result.context, released: false });
+        const token = this.#nextToken++;
+        this.#sessions.set(token, { url: result.url, context: result.context, released: false });
         return { url: result.url, token };
     }
     async release(token) {
-        const session = this.sessions.get(token);
+        const session = this.#sessions.get(token);
         // Release at most once per session — guards a close()/teardown race (the `released` flag is set
         // synchronously before the awaited onRelease, so releaseAll() can't double-fire an in-flight release).
         if (!session || session.released)
             return;
         session.released = true;
         try {
-            await this.onRelease?.({ endpoint: session.url, context: session.context });
+            await this.#onRelease?.({ endpoint: session.url, context: session.context });
         }
         catch (err) {
-            this.log.warning('Remote browser release() failed.', { error: err?.message });
+            this.#log.warning('Remote browser release() failed.', { error: err?.message });
         }
         finally {
-            this.sessions.delete(token);
+            this.#sessions.delete(token);
         }
     }
     /** Releases every session that is still open. Called on pool teardown so no remote session leaks. */
     async releaseAll() {
-        await Promise.all([...this.sessions.keys()].map(async (token) => this.release(token)));
+        await Promise.all([...this.#sessions.keys()].map(async (token) => this.release(token)));
     }
 }
 /**
@@ -89,19 +90,23 @@ class RemoteSessionRegistry {
  * @category Browser management
  */
 export class RemoteBrowserPool {
-    /** The wrapped pool that performs the remote connections and serves pages. */
+    /**
+     * The wrapped pool that performs the remote connections and serves pages.
+     *
+     * @internal
+     */
     browserPool;
     /** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
-    pool;
-    registry;
-    slotPollIntervalMillis;
-    log;
+    #pool;
+    #registry;
+    #slotPollIntervalMillis;
+    #log;
     /** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
-    _capacityChange;
+    #capacityChange;
     constructor(options) {
         const { browserPlugins, endpoint, release, maxOpenBrowsers, connection = {}, browserPoolOptions = {}, slotPollIntervalMillis = 500, } = options;
-        this.log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
-        this.slotPollIntervalMillis = slotPollIntervalMillis;
+        this.#log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
+        this.#slotPollIntervalMillis = slotPollIntervalMillis;
         // A RemoteBrowserProvider carries its own endpoint, release, and maxOpenBrowsers.
         const provider = endpoint instanceof RemoteBrowserProvider ? endpoint : undefined;
         const resolvedEndpoint = provider
@@ -111,20 +116,20 @@ export class RemoteBrowserPool {
             ? ({ context }) => provider.release(context)
             : release;
         const resolvedMax = maxOpenBrowsers ?? provider?.maxOpenBrowsers;
-        this.registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.log);
+        this.#registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.#log);
         // Wire every plugin for remote connection.
         for (const plugin of browserPlugins) {
-            plugin.useRemoteConnection(this.registry, connection);
+            plugin.useRemoteConnection(this.#registry, connection);
         }
         this.browserPool = new BrowserPool({ ...browserPoolOptions, browserPlugins });
-        this.pool = this.browserPool;
+        this.#pool = this.browserPool;
         // Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed
         // kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.
-        this.browserPool.on("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, (controller) => {
-            controller.once("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
-                const token = controller.launchContext._remoteToken;
+        this.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (controller) => {
+            controller.once(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, () => {
+                const token = controller.launchContext.remoteToken;
                 if (token !== undefined)
-                    void this.registry.release(token);
+                    void this.#registry.release(token);
             });
         });
         if (resolvedMax !== undefined) {
@@ -143,28 +148,38 @@ export class RemoteBrowserPool {
      * allows it (either a new browser slot is free, or an active browser still has page capacity).
      */
     async newPage(options) {
-        await this._waitForFreeSlot();
-        return this.pool.newPage(options);
+        await this.#waitForFreeSlot();
+        return this.#pool.newPage(options);
     }
     async closePage(page, options) {
-        return this.pool.closePage(page, options);
+        return this.#pool.closePage(page, options);
     }
     async extractPageState(page) {
-        return this.pool.extractPageState(page);
+        return this.#pool.extractPageState(page);
     }
     async injectPageState(page, state) {
-        return this.pool.injectPageState(page, state);
+        return this.#pool.injectPageState(page, state);
+    }
+    /**
+     * Closes all browsers and releases their remote sessions, leaving the pool usable. Closing a browser is what
+     * releases its session (see the constructor), so this needs no separate release pass.
+     */
+    async releaseAllBrowsers() {
+        await this.browserPool.releaseAllBrowsers();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
     }
     /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
     async destroy() {
         await this.browserPool.destroy();
         // Backstop: release any sessions whose browser never emitted a close (e.g. dropped on teardown).
-        await this.registry.releaseAll();
+        await this.#registry.releaseAll();
     }
     /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
-    async _waitForFreeSlot() {
+    async #waitForFreeSlot() {
         while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {
-            await this._nextCapacityChange();
+            await this.#nextCapacityChange();
         }
     }
     /**
@@ -172,20 +187,20 @@ export class RemoteBrowserPool {
      * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
      * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
      */
-    _nextCapacityChange() {
-        this._capacityChange ??= new Promise((resolve) => {
+    #nextCapacityChange() {
+        this.#capacityChange ??= new Promise((resolve) => {
             const done = () => {
                 clearTimeout(timer);
-                this.browserPool.off("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
-                this.browserPool.off("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
-                this._capacityChange = undefined;
+                this.browserPool.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
+                this.browserPool.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
+                this.#capacityChange = undefined;
                 resolve();
             };
-            const timer = setTimeout(done, this.slotPollIntervalMillis);
+            const timer = setTimeout(done, this.#slotPollIntervalMillis);
             timer.unref?.();
-            this.browserPool.once("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
-            this.browserPool.once("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
+            this.browserPool.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
+            this.browserPool.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
         });
-        return this._capacityChange;
+        return this.#capacityChange;
     }
 }

package/container-proxy-server.d.ts

@@ -1,11 +0,0 @@
-/**
- * Creates a proxy server designed to handle requests from "container" instances.
- * Each container instance is assigned to a different (but still localhost) IP address
- * in order to work around authorization and to enable upstream.
- * @internal
- */
-export declare function createProxyServerForContainers(fallbackProxyUrl?: string): Promise<{
-    port: number;
-    ipToProxy: Map<string, string>;
-    close(closeConnections: boolean): Promise<void>;
-}>;

package/container-proxy-server.js

@@ -1,42 +0,0 @@
-import { Server as ProxyChainServer } from 'proxy-chain';
-/**
- * Creates a proxy server designed to handle requests from "container" instances.
- * Each container instance is assigned to a different (but still localhost) IP address
- * in order to work around authorization and to enable upstream.
- * @internal
- */
-export async function createProxyServerForContainers(fallbackProxyUrl) {
-    const ipToProxy = new Map();
-    const proxyServer = new ProxyChainServer({
-        prepareRequestFunction({ request }) {
-            const prefix4to6 = '::ffff:';
-            const localAddress = request.socket.localAddress.startsWith(prefix4to6)
-                ? request.socket.localAddress.slice(prefix4to6.length)
-                : request.socket.localAddress;
-            const upstreamProxyUrl = ipToProxy.get(localAddress);
-            if (upstreamProxyUrl === undefined) {
-                if (fallbackProxyUrl) {
-                    return {
-                        upstreamProxyUrl: fallbackProxyUrl,
-                        requestAuthentication: false,
-                    };
-                }
-                console.warn(`Request without proxy ${localAddress} ${request.headers.host}`);
-            }
-            return {
-                upstreamProxyUrl,
-                requestAuthentication: false,
-            };
-        },
-        port: 0,
-    });
-    await proxyServer.listen();
-    proxyServer.server.unref();
-    return {
-        port: proxyServer.port,
-        ipToProxy,
-        async close(closeConnections) {
-            return proxyServer.close(closeConnections);
-        },
-    };
-}