@crawlee/browser-pool 4.0.0-beta.26 → 4.0.0-beta.260

package/anonymize-proxy.js

@@ -1,15 +1,20 @@
-import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
-export const anonymizeProxySugar = async (proxyUrl, username, password) => {
+/** @internal */
+export const anonymizeProxySugar = async (proxyUrl, username, password, options) => {
     if (proxyUrl) {
         const url = new URL(proxyUrl);
         if (username || password) {
             url.username = username ?? '';
             url.password = password ?? '';
         }
-        if (url.username || url.password) {
+        if (url.username || url.password || options?.ignoreProxyCertificate) {
             // trim off trailing slash if it's present
             const proxyUrlString = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
-            const anonymized = await anonymizeProxy(proxyUrlString);
+            const { anonymizeProxy, closeAnonymizedProxy } = await import('proxy-chain');
+            const anonymized = await anonymizeProxy({
+                url: proxyUrlString,
+                port: 0,
+                ignoreProxyCertificate: options?.ignoreProxyCertificate ?? false,
+            });
             return [
                 anonymized,
                 async () => {
@@ -21,4 +26,3 @@ export const anonymizeProxySugar = async (proxyUrl, username, password) => {
     }
     return [undefined, async () => { }];
 };
-//# sourceMappingURL=anonymize-proxy.js.map

package/browser-pool.d.ts

@@ -1,3 +1,4 @@
+import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
@@ -53,6 +54,16 @@ export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin
      * @default 20
      */
     maxOpenPagesPerBrowser?: number;
+    /**
+     * Maximum number of browser processes alive at once. A browser counts from the moment its launch starts until it
+     * actually closes — a retired browser that is still finishing its last pages still occupies a slot. Once the
+     * limit is reached, {@link BrowserPool.newPage|newPage} waits for a slot (a browser closing, or an open
+     * browser with page capacity to spare) instead of launching another. Set it to a remote browser service's
+     * concurrent-session limit to avoid `429` errors.
+     *
+     * @default Infinity
+     */
+    maxOpenBrowsers?: number;
     /**
      * Browsers tend to get bloated after processing a lot of pages. This option
      * configures the maximum number of processed pages after which the browser will
@@ -246,38 +257,37 @@ export interface BrowserPoolHooks<BC extends BrowserController, LC extends Launc
  * });
  * ```
  */
-export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
+export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
+    #private;
     browserPlugins: BrowserPlugins;
-    maxOpenPagesPerBrowser: number;
-    retireBrowserAfterPageCount: number;
-    operationTimeoutMillis: number;
-    closeInactiveBrowserAfterMillis: number;
-    useFingerprints?: boolean;
+    /** @internal */
     fingerprintOptions: FingerprintOptions;
-    preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
-    postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
-    prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
-    postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
-    prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
-    postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
-    pageCounter: number;
-    pages: Map<string, PageReturn>;
-    pageIds: WeakMap<PageReturn, string>;
-    startingBrowserControllers: Set<BrowserControllerReturn>;
-    activeBrowserControllers: Set<BrowserControllerReturn>;
-    retiredBrowserControllers: Set<BrowserControllerReturn>;
-    pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
+    /** @internal */
     fingerprintInjector?: FingerprintInjector;
     fingerprintGenerator?: FingerprintGenerator;
+    /** @internal */
     fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
+    private pages;
+    private activeBrowserControllers;
+    private retiredBrowserControllers;
     private browserKillerInterval?;
-    private browserRetireInterval?;
-    private limiter;
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
     /**
      * Opens a new page in one of the running browsers or launches
      * a new browser and opens a page there, if no browsers are active,
      * or their page limits have been exceeded.
+     *
+     * **Session injection (best-effort):** When a {@link NewPageOptions.session|session} is
+     * provided, this implementation uses it as a cache key for browser fingerprints (when
+     * fingerprinting is enabled) and reads
+     * {@link ProxyInfo.url|session.proxyInfo.url} /
+     * {@link ProxyInfo.ignoreTlsErrors|session.proxyInfo.ignoreTlsErrors} as defaults
+     * for `proxyUrl` and `ignoreTlsErrors` respectively. Explicit `proxyUrl` /
+     * `ignoreTlsErrors` values in the options take precedence.
+     *
+     * Beyond fingerprint caching and proxy configuration, no other session
+     * properties are consumed — cookie and header injection remain the
+     * crawler's responsibility.
      */
     newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
     /**
@@ -335,7 +345,6 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * until it's closed.
      */
     getPageId(page: PageReturn): string | undefined;
-    private _createPageForBrowser;
     /**
      * Removes a browser controller from the pool. The underlying
      * browser will be closed after all its pages are closed.
@@ -347,6 +356,44 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * closed after all its pages are closed.
      */
     retireBrowserByPage(page: PageReturn): void;
+    /**
+     * Releases a page back to the pool. The page is closed and, if the
+     * optional `error` is a {@link SessionError} (including
+     * {@link SessionRetiredError}), the browser controller that served
+     * the page is retired so that the finished session's state (cookies,
+     * storage, etc.) cannot leak into future sessions.
+     *
+     * This is the primary way the crawler should return pages to the pool.
+     *
+     * @param page The page to release.
+     * @param options.error The error that caused the page to be released, if any.
+     */
+    closePage(page: PageReturn, options?: {
+        error?: Error;
+    }): Promise<void>;
+    /**
+     * Extracts the relevant state (currently just cookies) from a page via its
+     * owning {@link BrowserController}. Returns empty state when the page is
+     * no longer associated with a controller.
+     *
+     * As with {@link BrowserPool.injectPageState}, cookies are isolated per
+     * page only when the pool is configured with `useIncognitoPages: true`.
+     * With the default `useIncognitoPages: false`, the extracted cookies
+     * include those set by any sibling page sharing the same browser.
+     */
+    extractPageState(page: PageReturn): Promise<PageState>;
+    /**
+     * Injects state into a page via its owning {@link BrowserController}.
+     *
+     * No-op when the page is no longer associated with a controller.
+     *
+     * Note that cookies are isolated per page only when the pool is configured
+     * with `useIncognitoPages: true` — each page then gets its own browser
+     * context. With the default `useIncognitoPages: false`, all pages in a
+     * browser share a single context, so injected cookies are visible to every
+     * page served by that browser.
+     */
+    injectPageState(page: PageReturn, state: PageState): Promise<void>;
     /**
      * Removes all active browsers from the pool. The browsers will be
      * closed after all their pages are closed.
@@ -357,32 +404,53 @@ export declare class BrowserPool<Options extends BrowserPoolOptions = BrowserPoo
      * @return {Promise<void>}
      */
     closeAllBrowsers(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Closes all managed browsers and tears down the pool.
+     * Closes every managed browser and empties the pool, which stays usable afterwards — a crawler releases its
+     * browsers when a run ends and may start another run on the same pool.
+     */
+    releaseAllBrowsers(): Promise<void>;
+    /**
+     * Closes all managed browsers and tears the pool down for good: its intervals are cleared without being
+     * re-armed and its listeners are dropped, so it cannot be used again.
      */
     destroy(): Promise<void>;
-    private _teardown;
-    private _getAllBrowserControllers;
-    private _launchBrowser;
-    /**
-     * Picks plugins round robin.
-     * @private
-     */
-    private _pickBrowserPlugin;
-    private _pickBrowserWithFreeCapacity;
-    private _closeInactiveRetiredBrowsers;
-    private _overridePageClose;
-    private _executeHooks;
-    private _closeRetiredBrowserWithNoPages;
-    private _initializeFingerprinting;
-    private _addFingerprintHooks;
+    private closeInactiveRetiredBrowsers;
+    /**
+     * Returns `true` if the pool can accept a new browser launch without exceeding `maxOpenBrowsers`.
+     * Counts starting, active, and retired browsers.
+     *
+     * @internal
+     */
+    hasFreeBrowserSlot(): boolean;
 }
-export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> {
+export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
+    /**
+     * The proxy URL the pool uses internally to route the page: it keys browser
+     * reuse (with `browserPerProxy`, only a browser already on this proxy is
+     * reused), configures the launched browser, and is applied to incognito
+     * pages. When omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool`'s proxy
+     * handling and is intentionally not part of the {@link IBrowserPool}
+     * contract — through that interface the proxy is supplied via the session.
+     */
+    proxyUrl?: string;
     /**
-     * Assign a custom ID to the page. If you don't a random string ID
-     * will be generated.
+     * Disable TLS certificate verification for MITM proxies. Applied both when
+     * launching a new browser and when creating a page in an existing one. When
+     * omitted, it is derived from the
+     * {@link NewPageOptions.session|session}'s `proxyInfo`; an explicit value
+     * here takes precedence.
+     *
+     * This is an implementation detail of the built-in `BrowserPool` and is
+     * intentionally not part of the {@link IBrowserPool} contract — through
+     * that interface, configure it via the session's `proxyInfo` or through the
+     * browser's `launchOptions`.
      */
-    id?: string;
+    ignoreTlsErrors?: boolean;
     /**
      * Some libraries (Playwright) allow you to open new pages with specific
      * options. Use this property to set those options.
@@ -397,14 +465,6 @@ export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin
      * see the `newPageInNewBrowser` function.
      */
     browserPlugin?: BP;
-    /**
-     * Proxy URL.
-     */
-    proxyUrl?: string;
-    /**
-     * Proxy tier.
-     */
-    proxyTier?: number;
 }
 export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
     /**
@@ -435,4 +495,3 @@ export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends B
      */
     launchOptions?: BP['launchOptions'];
 }
-//# sourceMappingURL=browser-pool.d.ts.map