@crawlee/browser 4.0.0-beta.10 → 4.0.0-beta.100

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,4 +1,3 @@
 export * from '@crawlee/basic';
 export * from './internals/browser-crawler.js';
 export * from './internals/browser-launcher.js';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,4 +1,3 @@
 export * from '@crawlee/basic';
 export * from './internals/browser-crawler.js';
 export * from './internals/browser-launcher.js';
-//# sourceMappingURL=index.js.map

package/internals/browser-crawler.d.ts

@@ -1,20 +1,57 @@
-import type { Awaitable, BasicCrawlerOptions, CrawlingContext, Dictionary, EnqueueLinksOptions, ErrorHandler, ProxyConfiguration, RequestHandler, RequestProvider, SkippedRequestCallback } from '@crawlee/basic';
-import { BasicCrawler, Configuration } from '@crawlee/basic';
-import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
-import { BrowserPool } from '@crawlee/browser-pool';
+import type { BasicCrawlerOptions, BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, IRequestManager, LoadedRequest, Request, RequestHandler, RouterHandler, SkippedRequestCallback } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
+import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, CrawlerRemoteBrowserOptions, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
+import type { Awaitable, BatchAddRequestsResult, Dictionary, IBrowserPool } from '@crawlee/types';
 import type { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep } from 'type-fest';
 import type { BrowserLaunchContext } from './browser-launcher.js';
-export interface BrowserCrawlingContext<Crawler = unknown, Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<Crawler, UserData> {
-    browserController: ProvidedController;
+interface BaseResponse {
+    status(): number;
+}
+export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = Dictionary, GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+    /**
+     * The browser page object where the web page is loaded and rendered.
+     */
     page: Page;
-    response?: Response;
+    /**
+     * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
+     */
+    request: LoadedRequest<Request<UserData>>;
+    /**
+     * The HTTP response object returned by the browser's navigation.
+     */
+    response: Response;
+    /**
+     * Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
+     * object (or return `{ gotoOptions: ... }`) to influence the navigation.
+     */
+    gotoOptions: GoToOptions;
+    /**
+     * Helper function for extracting URLs from the current page and adding them to the request queue.
+     */
+    enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
 }
-export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;
-export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = ErrorHandler<Context>;
-export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
-export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
     launchContext?: BrowserLaunchContext<any, any>;
+    /**
+     * An existing browser pool instance to use. When provided, the crawler will use this pool directly instead of
+     * constructing a new one from `browserPoolOptions`, enabling browser sharing across multiple crawlers. The crawler
+     * will not tear down a shared pool — the caller is responsible for its lifecycle.
+     */
+    browserPool?: IBrowserPool<Page>;
+    /**
+     * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
+     *
+     * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
+     * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
+     * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
+     * or a {@link RemoteBrowserProvider}.
+     *
+     * Ignored when `browserPool` is set. For sharing a remote pool across crawlers, construct a
+     * {@link RemoteBrowserPool} yourself and pass it as `browserPool` instead.
+     */
+    remoteBrowser?: CrawlerRemoteBrowserOptions;
     /**
      * Function that is called to process each request.
      *
@@ -25,7 +62,6 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * - {@link BrowserCrawlingContext.page|`page`} is an instance of the
      * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
      * Playwright [Page](https://playwright.dev/docs/api/class-page);
-     * - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
      * - {@link BrowserCrawlingContext.response|`response`} is an instance of the
      * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
      * Playwright [Response](https://playwright.dev/docs/api/class-response),
@@ -42,7 +78,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler?: BrowserRequestHandler<Context>;
+    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
      * User-provided function that allows modifying the request object before it gets retried by the crawler.
      * It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
@@ -53,7 +89,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    errorHandler?: BrowserErrorHandler<Context>;
+    errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
@@ -63,28 +99,22 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    failedRequestHandler?: BrowserErrorHandler<Context>;
+    failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * Custom options passed to the underlying {@link BrowserPool} constructor.
      * We can tweak those to fine-tune browser management.
      */
     browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
-    /**
-     * If set, the crawler will be configured for all connections to use
-     * the Proxy URLs provided and rotated according to the configuration.
-     */
-    proxyConfiguration?: ProxyConfiguration;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
-     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+     * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
      *
      * **Example:**
      *
      * ```js
      * preNavigationHooks: [
-     *     async (crawlingContext, gotoOptions) => {
-     *         const { page } = crawlingContext;
+     *     async ({ page, gotoOptions }) => {
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
      *         gotoOptions.timeout = 60_000;
      *         gotoOptions.waitUntil = 'domcontentloaded';
@@ -92,14 +122,22 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * ]
      * ```
      *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@link PrePageCreateHook}
+     * A hook may optionally return a partial object whose properties are merged into the crawling context,
+     * allowing the hook to override context members for subsequent hooks and pipeline stages.
+     *
+     * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
+     * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
+     * This means the members added by `extendContext` are already available here, but navigation-dependent
+     * members (e.g. `page`, `response`) are not.
      */
-    preNavigationHooks?: BrowserHook<Context>[];
+    preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
      *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context.
+     * This is useful for overriding context members (e.g. `response`) after solving a challenge.
+     *
      * **Example:**
      *
      * ```js
@@ -110,19 +148,27 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      *             await solveCaptcha(page);
      *         }
      *     },
+     *     async (crawlingContext) => {
+     *         if (await needsRevalidation(crawlingContext)) {
+     *             return { response: await crawlingContext.page.reload() };
+     *         }
+     *     },
      * ]
      * ```
      */
-    postNavigationHooks?: BrowserHook<Context>[];
+    postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
-     * Timeout in which page navigation needs to finish, in seconds.
+     * Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,
+     * the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the
+     * navigation uses. Separate from the
+     * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
+     * request handler.
      */
     navigationTimeoutSecs?: number;
     /**
-     * Defines whether the cookies should be persisted for sessions.
-     * This can only be used when `useSessionPool` is set to `true`.
+     * Defines whether the cookies should be persisted for sessions. Enabled by default.
      */
-    persistCookiesPerSession?: boolean;
+    saveResponseCookies?: boolean;
     /**
      * Whether to run browser in headless mode. Defaults to `true`.
      * Can be also set via {@link Configuration}.
@@ -151,51 +197,49 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * The source URLs are represented by the {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
+ * and pass the result as `requestManager`.
+ *
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BrowserCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
  *
  * @category Crawlers
  */
-export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext = BrowserCrawlingContext, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context> {
-    readonly config: Configuration;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration?: ProxyConfiguration;
+export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
+    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
+    private browserPoolDep;
     /**
-     * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
+     * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
+     * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool: BrowserPool<InternalBrowserPoolOptions>;
+    get browserPool(): IBrowserPool<Page>;
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
-    protected userProvidedRequestHandler: BrowserRequestHandler<Context>;
-    protected navigationTimeoutMillis: number;
-    protected requestHandlerTimeoutInnerMillis: number;
-    protected preNavigationHooks: BrowserHook<Context>[];
-    protected postNavigationHooks: BrowserHook<Context>[];
-    protected persistCookiesPerSession: boolean;
+    protected readonly ignoreShadowRoots: boolean;
+    protected readonly ignoreIframes: boolean;
+    private readonly navigationTimeoutMillis;
+    private readonly preNavigationHooks;
+    private readonly postNavigationHooks;
+    private readonly saveResponseCookies;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -208,19 +252,19 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
 // @ts-ignore optional peer dependency or compatibility with es2022
         headless: import("ow").AnyPredicate<string | boolean>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        browserPoolOptions: import("ow").ObjectPredicate<object>;
+        browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -237,24 +281,42 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -263,42 +325,47 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    protected constructor(options?: BrowserCrawlerOptions<Context>, config?: Configuration);
-    protected _cleanupContext(crawlingContext: Context): Promise<void>;
+    protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext> & {
+        contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
+    });
+    protected getNavigationTimeoutMillis(): number;
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
     private containsSelectors;
-    protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
+    private isRequestBlocked;
+    private preparePage;
+    private prepareNavigation;
+    private navigate;
+    private finalizeNavigation;
     /**
-     * Wrapper around requestHandler that opens and closes pages etc.
+     * Copies cookies from the live browser page into the session cookie jar.
      */
-    protected _runRequestHandler(crawlingContext: Context): Promise<void>;
-    protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, createNewSession?: boolean): void;
-    protected _handleNavigation(crawlingContext: Context): Promise<void>;
-    protected _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string): Promise<void>;
+    private persistCookiesFromPage;
     /**
-     * Marks session bad in case of navigation timeout.
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
      */
-    protected _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void>;
+    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
+    private handleBlockedRequestByContent;
+    private restoreRequestState;
+    private applyCookies;
     /**
-     * Transforms proxy-related errors to `SessionError`.
+     * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    protected _throwIfProxyError(error: Error): void;
-    protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
+    private handleNavigationTimeout;
     /**
-     * Should be overridden in case of different automation library that does not support this response API.
+     * Transforms proxy-related errors to `SessionError`.
      */
-    protected _responseHandler(crawlingContext: Context): Promise<void>;
-    protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
-    protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
+    private throwIfProxyError;
+    protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
+    private processResponse;
     /**
      * Function for cleaning up after all requests are processed.
      * @ignore
@@ -307,17 +374,24 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
 }
 /** @internal */
 interface EnqueueLinksInternalOptions {
-    options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
+    options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
     page: CommonPage;
-    requestQueue: RequestProvider;
+    requestManager: IRequestManager;
     robotsTxtFile?: RobotsTxtFile;
     onSkippedRequest?: SkippedRequestCallback;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
 /** @internal */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
+interface BoundEnqueueLinksInternalOptions {
+    enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+    options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
+    page: CommonPage;
+}
+/** @internal */
+export declare function browserCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
 /**
  * Extracts URLs from a given page.
  * @ignore
@@ -326,4 +400,3 @@ export declare function extractUrlsFromPage(page: {
     $$eval: Function;
 }, selector: string, baseUrl: string): Promise<string[]>;
 export {};
-//# sourceMappingURL=browser-crawler.d.ts.map