@crawlee/browser 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/internals/browser-crawler.d.ts

@@ -1,9 +1,8 @@
-import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, Request, RequestHandler, RouterHandler } from '@crawlee/basic';
+import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
 import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
 import { z } from 'zod';
-import type { BrowserLaunchContext } from './browser-launcher.js';
 interface BaseResponse {
     status(): number;
     /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
@@ -11,18 +10,14 @@ interface BaseResponse {
 }
 /**
  * The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
- * additionally exposes `destroy()` — the crawler only ever tears down pools it created, which is why {@link IBrowserPool}
- * itself intentionally omits `destroy`.
+ * additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
+ * {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
+ * `destroy()` once the crawler itself is destroyed.
  */
 export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
+    releaseAllBrowsers: () => Promise<void>;
     destroy: () => Promise<void>;
 };
-/**
- * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
- * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
- * into a second, half-working way of configuring the same pool.
- */
-export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
 export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
     /**
@@ -32,7 +27,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
     /**
      * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
      */
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     /**
      * The HTTP response object returned by the browser's navigation.
      */
@@ -52,8 +47,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
     enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
 }
 export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
-    launchContext?: BrowserLaunchContext<any, any>;
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
     /**
      * The browser pool the crawler should serve its pages from. This is the single way to run a pool with
      * non-default options: build one with the factory that matches your crawler
@@ -105,7 +99,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
     requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
@@ -244,28 +238,19 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
  *
  * @category Crawlers
  */
-export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
     get browserPool(): IBrowserPool<Page>;
-    launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
     protected readonly ignoreShadowRoots: boolean;
     protected readonly ignoreIframes: boolean;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
-        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
-        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
-        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -297,25 +282,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
-    };
-    protected static optionsSchema: z.ZodObject<{
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -323,6 +306,9 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         ignoreIframes: z.ZodDefault<z.ZodBoolean>;
         ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -354,22 +340,34 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
+        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
     }, z.core.$strict>;
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
+     * @internal
      */
     protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
         contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
@@ -380,41 +378,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
          */
         browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
     });
+    /** @internal */
     protected getNavigationTimeoutMillis(): number;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
-    private containsSelectors;
-    private isRequestBlocked;
-    private preparePage;
-    private prepareNavigation;
     private navigate;
-    private finalizeNavigation;
-    /**
-     * Copies cookies from the live browser page into the session cookie jar.
-     */
-    private persistCookiesFromPage;
     /**
      * Runs the user request handler, then re-reads browser cookies so login flows /
      * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     * @internal
      */
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
-    private handleBlockedRequestByContent;
-    private restoreRequestState;
-    private applyCookies;
-    /**
-     * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
-     */
-    private handleNavigationTimeout;
-    /**
-     * Transforms proxy-related errors to `SessionError`.
-     */
-    private throwIfProxyError;
     protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
-    private processResponse;
     /**
-     * Function for cleaning up after all requests are processed.
-     * @ignore
+     * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
+     * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
      */
     teardown(): Promise<void>;
+    destroy(): Promise<void>;
 }
 /**
  * Extracts URLs from a given page.

package/internals/browser-crawler.js

@@ -1,22 +1,8 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
-import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
+import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
 import { sleep } from '@crawlee/utils';
 import { z } from 'zod';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
-/**
- * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
- * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
- * into a second, half-working way of configuring the same pool.
- */
-export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
-    const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
-    if (names.length === 0) {
-        return;
-    }
-    throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
-        `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
-        'itself. Configure the pool you pass in instead.');
-}
 const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
 const readContextField = (ctx, key) => ctx[key];
 /**
@@ -80,13 +66,15 @@ export class BrowserCrawler extends BasicCrawler {
     get browserPool() {
         return this.#browserPoolDep.value;
     }
-    launchContext;
     ignoreShadowRoots;
     ignoreIframes;
     #navigationTimeoutMillis;
     #preNavigationHooks;
     #postNavigationHooks;
     #saveResponseCookies;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: schemas.anyNumber
@@ -94,7 +82,6 @@ export class BrowserCrawler extends BasicCrawler {
             .default(60),
         preNavigationHooks: schemas.anyArray.default(() => []),
         postNavigationHooks: schemas.anyArray.default(() => []),
-        launchContext: schemas.anyObject.default(() => ({})),
         browserPool: validators.browserPool.optional(),
         browserPoolBuilder: schemas.anyFunction.optional(),
         remoteBrowser: schemas.anyObject.optional(),
@@ -103,18 +90,18 @@ export class BrowserCrawler extends BasicCrawler {
         ignoreIframes: z.boolean().default(false),
         ignoreShadowRoots: z.boolean().default(false),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
+     * @internal
      */
     constructor(options) {
-        const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
+        const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
         if (browserPool) {
             assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
         }
-        const skipGuard = (action) => ({
-            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
-        });
+        const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
         super({
             ...basicCrawlerOptions,
             contextPipelineBuilder: () => {
@@ -129,7 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
                     }
                     return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
                 });
-                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+                let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
                 for (const hook of this.#preNavigationHooks) {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
@@ -138,13 +125,12 @@ export class BrowserCrawler extends BasicCrawler {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
                 return pipeline
-                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
-                    .compose({ action: this.handleBlockedRequestByContent.bind(this) })
-                    .compose({ action: this.restoreRequestState.bind(this) });
+                    .compose(skipGuard(this.#finalizeNavigation.bind(this)))
+                    .compose(this.#handleBlockedRequestByContent.bind(this))
+                    .compose(this.#restoreRequestState.bind(this));
             },
             extendContext,
         });
-        this.launchContext = launchContext;
         this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         // The public option hooks are extension-aware; internal storage uses the base context type
         // (the pipeline composes hooks against the concrete context, which does not statically carry
@@ -156,43 +142,32 @@ export class BrowserCrawler extends BasicCrawler {
         this.#saveResponseCookies = saveResponseCookies;
         this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
     }
+    /** @internal */
     getNavigationTimeoutMillis() {
         return this.#navigationTimeoutMillis;
     }
     buildContextPipeline() {
-        return ContextPipeline.create().compose({
-            action: this.preparePage.bind(this),
-            cleanup: async (context) => {
-                context.registerDeferredCleanup(async () => {
-                    const error = !context.session.isUsable()
-                        ? new SessionError('Session is no longer usable')
-                        : undefined;
-                    await this.browserPool
-                        .closePage(context.page, { error })
-                        .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
-                });
-            },
-        });
+        return ContextPipeline.create().compose(this.#preparePage.bind(this));
     }
-    async containsSelectors(page, selectors) {
+    async #containsSelectors(page, selectors) {
         const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
             .map((x, i) => [x, selectors[i]])
             .filter(([x]) => x !== null)
             .map(([, selector]) => selector);
         return foundSelectors.length > 0 ? foundSelectors : null;
     }
-    async isRequestBlocked(crawlingContext) {
+    async #isRequestBlocked(crawlingContext) {
         const { page, response } = crawlingContext;
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
-        if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
+        if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
             // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
-            const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
+            const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
             if (!foundSelectors)
                 return false;
             return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
         }
-        const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
+        const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
         const statusCode = response?.status() ?? 0;
         if (foundSelectors)
             return `Found selectors: ${foundSelectors.join(', ')}`;
@@ -200,12 +175,34 @@ export class BrowserCrawler extends BasicCrawler {
             return `Received blocked status code: ${statusCode}`;
         return false;
     }
-    async preparePage(crawlingContext) {
+    async #preparePage(crawlingContext, onCleanup) {
         const page = await this.browserPool.newPage({
             id: crawlingContext.id,
             session: crawlingContext.session,
         });
         tryCancel();
+        onCleanup((failure) => {
+            crawlingContext.registerDeferredCleanup(async () => {
+                const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
+                    ? failure.cause
+                    : failure;
+                // Only tells the pool what to do with the page's browser state - the request's failure, if any, has
+                // already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
+                // unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
+                // count, so the block check must come first.
+                const { session } = crawlingContext;
+                const closeReason = cause instanceof SessionError
+                    ? cause
+                    : session.isBlocked()
+                        ? new SessionError()
+                        : session.isUsable()
+                            ? undefined
+                            : new SessionRetiredError();
+                await this.browserPool
+                    .closePage(page, { error: closeReason })
+                    .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
+            });
+        });
         const addRequests = crawlingContext.addRequests;
         const extractLinks = async (options) => {
             return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
@@ -235,16 +232,19 @@ export class BrowserCrawler extends BasicCrawler {
             },
         };
     }
-    async prepareNavigation(crawlingContext) {
+    async #prepareNavigation(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 request: new Proxy(crawlingContext.request, {
-                    get(target, propertyName, receiver) {
+                    get(target, propertyName) {
                         if (propertyName === 'loadedUrl') {
                             throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
-                        return Reflect.get(target, propertyName, receiver);
+                        // `target` as receiver and bound methods, or `#` members throw on the proxy
+                        const value = Reflect.get(target, propertyName, target);
+                        return typeof value === 'function' ? value.bind(target) : value;
                     },
+                    set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
                 }),
                 get response() {
                     throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -259,6 +259,7 @@ export class BrowserCrawler extends BasicCrawler {
             [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
         };
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async navigate(crawlingContext) {
         tryCancel();
         const gotoOptions = crawlingContext.gotoOptions;
@@ -276,22 +277,22 @@ export class BrowserCrawler extends BasicCrawler {
         }
         const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
         const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
-        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
+        await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
         let response;
         try {
             response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
-            await this.handleNavigationTimeout(crawlingContext, error);
+            await this.#handleNavigationTimeout(crawlingContext, error);
             crawlingContext.request.state = RequestState.ERROR;
-            this.throwIfProxyError(error);
+            this.#throwIfProxyError(error);
             throw error;
         }
         tryCancel();
         crawlingContext.request.state = RequestState.AFTER_NAV;
         return { response };
     }
-    async finalizeNavigation(crawlingContext) {
+    async #finalizeNavigation(crawlingContext) {
         tryCancel();
         let response;
         try {
@@ -301,17 +302,17 @@ export class BrowserCrawler extends BasicCrawler {
             // `preparePage` installs a throwing getter for `response`; reaching this branch means
             // navigation produced no response and no hook overrode it. Treat as undefined.
         }
-        await this.processResponse(response, crawlingContext);
+        await this.#processResponse(response, crawlingContext);
         tryCancel();
         // Persist cookies from the navigation response before the user handler runs.
         // Cookies set during `requestHandler` are saved again afterwards.
-        await this.persistCookiesFromPage(crawlingContext);
+        await this.#persistCookiesFromPage(crawlingContext);
         return { request: crawlingContext.request };
     }
     /**
      * Copies cookies from the live browser page into the session cookie jar.
      */
-    async persistCookiesFromPage(crawlingContext) {
+    async #persistCookiesFromPage(crawlingContext) {
         if (!this.#saveResponseCookies || !crawlingContext.session) {
             return;
         }
@@ -333,6 +334,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Runs the user request handler, then re-reads browser cookies so login flows /
      * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     * @internal
      */
     async runRequestHandler(crawlingContext) {
         try {
@@ -341,7 +343,7 @@ export class BrowserCrawler extends BasicCrawler {
         finally {
             if (!crawlingContext.request.skipNavigation) {
                 try {
-                    await this.persistCookiesFromPage(crawlingContext);
+                    await this.#persistCookiesFromPage(crawlingContext);
                 }
                 catch {
                     // Page may already be closed on some failure paths; ignore.
@@ -349,19 +351,19 @@ export class BrowserCrawler extends BasicCrawler {
             }
         }
     }
-    async handleBlockedRequestByContent(crawlingContext) {
+    async #handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
+            const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
         return {};
     }
-    async restoreRequestState(crawlingContext) {
+    async #restoreRequestState(crawlingContext) {
         crawlingContext.request.state = RequestState.REQUEST_HANDLER;
         return {};
     }
-    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+    async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
         const sessionCookie = session
             ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
             : [];
@@ -375,7 +377,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async handleNavigationTimeout(crawlingContext, error) {
+    async #handleNavigationTimeout(crawlingContext, error) {
         const { session, page } = crawlingContext;
         // Fire-and-forget: no user code will run on this page after a failed navigation.
         // Swallow rejections: the page may already be detached.
@@ -390,12 +392,12 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    throwIfProxyError(error) {
+    #throwIfProxyError(error) {
         if (this.isProxyError(error)) {
             throw new SessionError(this.getMessageFromError(error));
         }
     }
-    async processResponse(response, crawlingContext) {
+    async #processResponse(response, crawlingContext) {
         const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
@@ -427,13 +429,17 @@ export class BrowserCrawler extends BasicCrawler {
         request.loadedUrl = await page.url();
     }
     /**
-     * Function for cleaning up after all requests are processed.
-     * @ignore
+     * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
+     * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
      */
     async teardown() {
-        await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
+        await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
         await super.teardown();
     }
+    async destroy() {
+        await super.destroy();
+        await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
+    }
 }
 /**
  * Extracts URLs from a given page.

package/internals/browser-launcher.d.ts

@@ -72,18 +72,19 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
  * the browser they run against is only known to the concrete `*BrowserPool()` factory, which is where the
  * caller-facing types are pinned down.
  */
-export type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
+type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
     [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
 };
 /**
  * The {@link RemoteBrowserPool} counterpart of {@link LauncherBrowserPoolOptions}.
  */
-export type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
+type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
 /**
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
  * @ignore
  */
 export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
+    #private;
     readonly configuration: Configuration;
     launcher: Launcher;
     proxyUrl?: string;
@@ -92,6 +93,9 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
     otherLaunchContextProps: Dictionary;
     Plugin: T;
     userAgent?: string;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
         proxyUrl: z.ZodOptional<z.ZodURL>;
         useChrome: z.ZodOptional<z.ZodBoolean>;
@@ -102,6 +106,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
         launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         userAgent: z.ZodOptional<z.ZodString>;
     };
+    /** @internal */
     protected static optionsSchema: z.ZodObject<{
         proxyUrl: z.ZodOptional<z.ZodURL>;
         useChrome: z.ZodOptional<z.ZodBoolean>;
@@ -136,11 +141,6 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
      * @internal
      */
     createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
-    /**
-     * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
-     * explicitly requested user agent wins.
-     */
-    private resolveFingerprinting;
     /**
      * Launches a browser instance based on the plugin.
      * @returns Browser instance.
@@ -148,10 +148,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
     launch(): LaunchResult;
     createLaunchOptions(): Dictionary;
     protected getDefaultHeadlessOption(): boolean;
-    private getChromeExecutablePath;
-    /**
-     * Gets a typical path to Chrome executable, depending on the current operating system.
-     */
-    private getTypicalChromeExecutablePath;
-    private validateProxyUrlProtocol;
 }
+export {};

package/internals/browser-launcher.js

@@ -1,8 +1,9 @@
 import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import os from 'node:os';
-import { Configuration, schemas, serviceLocator } from '@crawlee/basic';
+import { Configuration, serviceLocator } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
+import { schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 const DEFAULT_VIEWPORT = {
     width: 1366,
@@ -23,6 +24,9 @@ export class BrowserLauncher {
     // to be provided by child classes;
     Plugin;
     userAgent;
+    /**
+     * @internal
+     */
     static optionsShape = {
         proxyUrl: z.url().optional(),
         useChrome: z.boolean().optional(),
@@ -33,6 +37,7 @@ export class BrowserLauncher {
         launchOptions: schemas.anyObject.optional(),
         userAgent: z.string().optional(),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
     static requireLauncherOrThrow(launcher, apifyImageName) {
         try {
@@ -56,7 +61,7 @@ export class BrowserLauncher {
     constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
         this.configuration = configuration;
         const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
-        this.validateProxyUrlProtocol(proxyUrl);
+        this.#validateProxyUrlProtocol(proxyUrl);
         // those need to be reassigned otherwise they are {} in types
         this.launcher = launcher;
         this.proxyUrl = proxyUrl;
@@ -86,7 +91,7 @@ export class BrowserLauncher {
         // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
         // caller-facing hook types get pinned down.
         return new BrowserPool({
-            ...this.resolveFingerprinting(options),
+            ...this.#resolveFingerprinting(options),
             browserPlugins: [this.createBrowserPlugin()],
         });
     }
@@ -99,14 +104,14 @@ export class BrowserLauncher {
         return new RemoteBrowserPool({
             ...options,
             browserPlugins: [this.createBrowserPlugin()],
-            browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),
+            browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
         });
     }
     /**
      * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
      * explicitly requested user agent wins.
      */
-    resolveFingerprinting(options) {
+    #resolveFingerprinting(options) {
         if (!this.userAgent) {
             return options;
         }
@@ -143,20 +148,20 @@ export class BrowserLauncher {
             launchOptions.headless = this.getDefaultHeadlessOption();
         }
         if (this.useChrome && !launchOptions.executablePath) {
-            launchOptions.executablePath = this.getChromeExecutablePath();
+            launchOptions.executablePath = this.#getChromeExecutablePath();
         }
         return launchOptions;
     }
     getDefaultHeadlessOption() {
         return this.configuration.headless && !this.configuration.xvfb;
     }
-    getChromeExecutablePath() {
-        return this.configuration.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();
+    #getChromeExecutablePath() {
+        return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
     }
     /**
      * Gets a typical path to Chrome executable, depending on the current operating system.
      */
-    getTypicalChromeExecutablePath() {
+    #getTypicalChromeExecutablePath() {
         /**
          * Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
          * Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
@@ -184,7 +189,7 @@ export class BrowserLauncher {
                 return '/usr/bin/google-chrome';
         }
     }
-    validateProxyUrlProtocol(proxyUrl) {
+    #validateProxyUrlProtocol(proxyUrl) {
         if (!proxyUrl)
             return;
         if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,14 +47,13 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-rc.0",
-        "@crawlee/browser-pool": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
-        "@crawlee/utils": "4.0.0-rc.0",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/basic": "4.0.0-rc.1",
+        "@crawlee/browser-pool": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0",
-        "zod": "^4.4.3"
+        "zod": "^4.5.4"
     },
     "peerDependencies": {
         "playwright": "*",
@@ -75,5 +74,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }