@crawlee/browser 4.0.0-beta.1 → 4.0.0-beta.11

package/internals/browser-crawler.d.ts

@@ -1,19 +1,38 @@
-import type { Awaitable, BasicCrawlerOptions, CrawlingContext, Dictionary, EnqueueLinksOptions, ErrorHandler, LoadedContext, ProxyConfiguration, RequestHandler, RequestProvider, SkippedRequestCallback } from '@crawlee/basic';
-import { BasicCrawler, Configuration } from '@crawlee/basic';
+import type { Awaitable, BasicCrawlerOptions, CrawlingContext, Dictionary, EnqueueLinksOptions, ErrorHandler, LoadedRequest, ProxyConfiguration, Request, RequestHandler, RequestProvider, SkippedRequestCallback } from '@crawlee/basic';
+import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
 import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
 import { BrowserPool } from '@crawlee/browser-pool';
+import type { BatchAddRequestsResult } from '@crawlee/types';
 import type { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep } from 'type-fest';
 import type { BrowserLaunchContext } from './browser-launcher.js';
-export interface BrowserCrawlingContext<Crawler = unknown, Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<Crawler, UserData> {
+interface BaseResponse {
+    status(): number;
+}
+export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+    /**
+     * An instance of the {@link BrowserController} that manages the browser instance and provides access to its API.
+     */
     browserController: ProvidedController;
+    /**
+     * The browser page object where the web page is loaded and rendered.
+     */
     page: Page;
-    response?: Response;
+    /**
+     * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
+     */
+    request: LoadedRequest<Request<UserData>>;
+    /**
+     * The HTTP response object returned by the browser's navigation.
+     */
+    response: Response;
+    /**
+     * Helper function for extracting URLs from the current page and adding them to the request queue.
+     */
+    enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
 }
-export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;
-export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = ErrorHandler<Context>;
 export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
-export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController extends BrowserController = BrowserController, Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>, ContextExtension = {}, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
     launchContext?: BrowserLaunchContext<any, any>;
     /**
      * Function that is called to process each request.
@@ -42,7 +61,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler?: BrowserRequestHandler<LoadedContext<Context>>;
+    requestHandler?: RequestHandler<ExtendedContext>;
     /**
      * User-provided function that allows modifying the request object before it gets retried by the crawler.
      * It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
@@ -53,7 +72,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    errorHandler?: BrowserErrorHandler<Context>;
+    errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
@@ -63,7 +82,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    failedRequestHandler?: BrowserErrorHandler<Context>;
+    failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * Custom options passed to the underlying {@link BrowserPool} constructor.
      * We can tweak those to fine-tune browser management.
@@ -178,7 +197,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
  *
  * @category Crawlers
  */
-export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext = BrowserCrawlingContext, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context> {
+export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController extends BrowserController = BrowserController, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>, ContextExtension = {}, ExtendedContext extends Context = Context & ContextExtension, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
     readonly config: Configuration;
     /**
      * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
@@ -190,9 +209,9 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
      */
     browserPool: BrowserPool<InternalBrowserPoolOptions>;
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
-    protected userProvidedRequestHandler: BrowserRequestHandler<Context>;
+    protected readonly ignoreShadowRoots: boolean;
+    protected readonly ignoreIframes: boolean;
     protected navigationTimeoutMillis: number;
-    protected requestHandlerTimeoutInnerMillis: number;
     protected preNavigationHooks: BrowserHook<Context>[];
     protected postNavigationHooks: BrowserHook<Context>[];
     protected persistCookiesPerSession: boolean;
@@ -218,9 +237,9 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
 // @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -273,30 +292,27 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    protected constructor(options?: BrowserCrawlerOptions<Context>, config?: Configuration);
-    protected _cleanupContext(crawlingContext: Context): Promise<void>;
+    protected constructor(options: BrowserCrawlerOptions<Page, Response, ProvidedController, Context, ContextExtension, ExtendedContext> & {
+        contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
+    }, config?: Configuration);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>>;
     private containsSelectors;
-    protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    protected _runRequestHandler(crawlingContext: Context): Promise<void>;
-    protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, createNewSession?: boolean): void;
-    protected _handleNavigation(crawlingContext: Context): Promise<void>;
-    protected _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string): Promise<void>;
+    protected isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>): Promise<string | false>;
+    private preparePage;
+    private performNavigation;
+    private handleBlockedRequestByContent;
+    private restoreRequestState;
+    protected _applyCookies({ session, request, page, browserController }: BrowserCrawlingContext, preHooksCookies: string, postHooksCookies: string): Promise<void>;
     /**
      * Marks session bad in case of navigation timeout.
      */
-    protected _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void>;
+    protected _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void>;
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
     protected _throwIfProxyError(error: Error): void;
-    protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
-    /**
-     * Should be overridden in case of different automation library that does not support this response API.
-     */
-    protected _responseHandler(crawlingContext: Context): Promise<void>;
+    protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
+    private processResponse;
     protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
     protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
     /**
@@ -316,8 +332,7 @@ interface EnqueueLinksInternalOptions {
     finalRequestUrl?: string;
 }
 /** @internal */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
+export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<BatchAddRequestsResult>;
 /**
  * Extracts URLs from a given page.
  * @ignore

package/internals/browser-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,SAAS,EACT,mBAAmB,EACnB,eAAe,EACf,UAAU,EACV,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,kBAAkB,EAElB,cAAc,EACd,eAAe,EAEf,sBAAsB,EACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEH,YAAY,EAEZ,aAAa,EAUhB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACR,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAE/E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAI9C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAElE,MAAM,WAAW,sBAAsB,CACnC,OAAO,GAAG,OAAO,EACjB,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,GAAG,UAAU,EACrB,kBAAkB,GAAG,iBAAiB,EACtC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC1C,SAAQ,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC;IACxC,iBAAiB,EAAE,kBAAkB,CAAC;IACtC,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED,MAAM,MAAM,qBAAqB,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,IAC7F,cAAc,CAAC,OAAO,CAAC,CAAC;AAE5B,MAAM,MAAM,mBAAmB,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,IAC3F,YAAY,CAAC,OAAO,CAAC,CAAC;AAE1B,MAAM,MAAM,WAAW,CAAC,OAAO,GAAG,sBAAsB,EAAE,WAAW,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,IAAI,CACjH,eAAe,EAAE,OAAO,EACxB,WAAW,EAAE,WAAW,KACvB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,qBAAqB,CAClC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACN,mBAAmB,EAEnB,gBAAgB,GAAG,sBAAsB,GAAG,cAAc,CAC7D;IACD,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,EAAE,qBAAqB,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IAE/D;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAC5C,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAEhF;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,KAAK,CAAC;IAEnC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,8BAAsB,cAAc,CAChC,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,EACzD,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,WAAW,SAAS,UAAU,GAAG,UAAU,CAC7C,SAAQ,YAAY,CAAC,OAAO,CAAC;aA4CL,MAAM;IA3C5B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE5D,SAAS,CAAC,0BAA0B,EAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACtE,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,gCAAgC,EAAE,MAAM,CAAC;IACnD,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAE5C,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAgBpC;IAEF;;OAEG;IACH,SAAS,aACL,OAAO,GAAE,qBAAqB,CAAC,OAAO,CAAM,EAC1B,MAAM,gBAAkC;cAwErC,eAAe,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YASnE,iBAAiB;cASN,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IA8B5F;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;IAyFpE,SAAS,CAAC,mCAAmC,CACzC,eAAe,EAAE,OAAO,EACxB,IAAI,EAAE,UAAU,EAChB,gBAAgB,CAAC,EAAE,OAAO,GAC3B,IAAI;cAiCS,iBAAiB,CAAC,eAAe,EAAE,OAAO;cA6B1C,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,OAAO,EACtD,eAAe,EAAE,MAAM,EACvB,gBAAgB,EAAE,MAAM;IAc5B;;OAEG;cACa,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/F;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAMzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACjC,eAAe,EAAE,OAAO,EACxB,WAAW,EAAE,WAAW,GACzB,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAElD;;OAEG;cACa,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;cAoBzD,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BlG,SAAS,CAAC,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI;IAoBjH;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC9G,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DAsB7B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAErC,IAAI,EAAE;IAAE,MAAM,EAAE,QAAQ,CAAA;CAAE,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BnB"}
+{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,SAAS,EACT,mBAAmB,EAEnB,eAAe,EACf,UAAU,EACV,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,kBAAkB,EAElB,OAAO,EACP,cAAc,EACd,eAAe,EAEf,sBAAsB,EACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAUlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACR,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAC/E,OAAO,KAAK,EAAE,sBAAsB,EAA0B,MAAM,gBAAgB,CAAC;AACrF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAI9C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAElE,UAAU,YAAY;IAClB,MAAM,IAAI,MAAM,CAAC;CACpB;AAID,MAAM,WAAW,sBAAsB,CACnC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,GAAG,iBAAiB,EACtC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC1C,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,iBAAiB,EAAE,kBAAkB,CAAC;IAEtC;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,mBAAmB,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACpF;AAED,MAAM,MAAM,WAAW,CAAC,OAAO,GAAG,sBAAsB,EAAE,WAAW,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,IAAI,CACjH,eAAe,EAAE,OAAO,EACxB,WAAW,EAAE,WAAW,KACvB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,qBAAqB,CAClC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,SAAS,iBAAiB,GAAG,iBAAiB,EAChE,OAAO,SAAS,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,GAAG,sBAAsB,CAC3G,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,UAAU,CACb,EACD,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,EAC5D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACN,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,EAE7C,gBAAgB,GAAG,sBAAsB,GAAG,cAAc,CAC7D;IACD,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAE9D;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAEtE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAC5C,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAEhF;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;;;OAgBG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,KAAK,CAAC;IAEnC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,8BAAsB,cAAc,CAChC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,SAAS,iBAAiB,GAAG,iBAAiB,EAChE,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,EACzD,OAAO,SAAS,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,GAAG,sBAAsB,CAC3G,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,UAAU,CACb,EACD,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,EAC5D,WAAW,SAAS,UAAU,GAAG,UAAU,CAC7C,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;aAoDxC,MAAM;IAnD5B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE5D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IAC9C,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAE1C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAE5C,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;IACH,SAAS,aACL,OAAO,EAAE,qBAAqB,CAC1B,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,OAAO,EACP,gBAAgB,EAChB,eAAe,CAClB,GAAG;QACA,sBAAsB,EAAE,MAAM,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;KAC3E,EACiB,MAAM,gBAAkC;IAuE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAC7C,eAAe,EACf,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,CACzE;YAgBa,iBAAiB;cASf,gBAAgB,CAC5B,eAAe,EAAE,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,CAAC,GAC5E,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;YA8BZ,WAAW;YAmEX,iBAAiB;YA0EjB,6BAA6B;YAW7B,mBAAmB;cAKjB,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,sBAAsB,EACrE,eAAe,EAAE,MAAM,EACvB,gBAAgB,EAAE,MAAM;IAc5B;;OAEG;cACa,wBAAwB,CAAC,eAAe,EAAE,sBAAsB,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9G;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAMzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACjC,eAAe,EAAE,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,CAAC,EAC3E,WAAW,EAAE,WAAW,GACzB,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;YAEpC,eAAe;cAuBb,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BlG,SAAS,CAAC,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI;IAoBjH;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC9G,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,mCAsB7B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAErC,IAAI,EAAE;IAAE,MAAM,EAAE,QAAQ,CAAA;CAAE,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BnB"}

package/internals/browser-crawler.js

@@ -1,8 +1,8 @@
-import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { tryCancel } from '@apify/timeout';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -54,9 +54,9 @@ export class BrowserCrawler extends BasicCrawler {
      */
     browserPool;
     launchContext;
-    userProvidedRequestHandler;
+    ignoreShadowRoots;
+    ignoreIframes;
     navigationTimeoutMillis;
-    requestHandlerTimeoutInnerMillis;
     preNavigationHooks;
     postNavigationHooks;
     persistCookiesPerSession;
@@ -72,34 +72,33 @@ export class BrowserCrawler extends BasicCrawler {
         persistCookiesPerSession: ow.optional.boolean,
         useSessionPool: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        ignoreShadowRoots: ow.optional.boolean,
-        ignoreIframes: ow.optional.boolean,
     };
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options, config = Configuration.getGlobalConfig()) {
         ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], requestHandler, failedRequestHandler, headless, ignoreShadowRoots, ignoreIframes, ...basicCrawlerOptions } = options;
+        const { navigationTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
         super({
             ...basicCrawlerOptions,
-            requestHandler: async (...args) => this._runRequestHandler(...args),
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+            contextPipelineBuilder: () => contextPipelineBuilder()
+                .compose({ action: this.performNavigation.bind(this) })
+                .compose({ action: this.handleBlockedRequestByContent.bind(this) })
+                .compose({ action: this.restoreRequestState.bind(this) }),
+            extendContext: extendContext,
         }, config);
         this.config = config;
-        // FIXME any
-        this.userProvidedRequestHandler = requestHandler ?? this.router;
-        this.failedRequestHandler = failedRequestHandler; // FIXME is this even needed?
         // Cookies should be persisted per session only if session pool is used
         if (!this.useSessionPool && persistCookiesPerSession) {
             throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
         }
         this.launchContext = launchContext;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
         this.proxyConfiguration = proxyConfiguration;
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
+        this.ignoreIframes = ignoreIframes;
+        this.ignoreShadowRoots = ignoreShadowRoots;
         if (headless != null) {
             this.launchContext.launchOptions ??= {};
             this.launchContext.launchOptions.headless = headless;
@@ -122,12 +121,17 @@ export class BrowserCrawler extends BasicCrawler {
             postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
         });
     }
-    async _cleanupContext(crawlingContext) {
-        const { page } = crawlingContext;
-        // Page creation may be aborted
-        if (page) {
-            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
-        }
+    buildContextPipeline() {
+        return ContextPipeline.create().compose({
+            action: this.preparePage.bind(this),
+            cleanup: async (context) => {
+                context.registerDeferredCleanup(async () => {
+                    await context.page
+                        .close()
+                        .catch((error) => this.log.debug('Error while closing page', { error }));
+                });
+            },
+        });
     }
     async containsSelectors(page, selectors) {
         const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
@@ -161,10 +165,7 @@ export class BrowserCrawler extends BasicCrawler {
             return `Received blocked status code: ${blockedStatusCode}`;
         return false;
     }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
+    async preparePage(crawlingContext) {
         const newPageOptions = {
             id: crawlingContext.id,
         };
@@ -190,76 +191,46 @@ export class BrowserCrawler extends BasicCrawler {
         }
         const page = (await this.browserPool.newPage(newPageOptions));
         tryCancel();
-        this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
-        // DO NOT MOVE THIS LINE ABOVE!
-        // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
-        // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
-        // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
-        const { request, session } = crawlingContext;
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            tryCancel();
-            await this._responseHandler(crawlingContext);
-            tryCancel();
-            // save cookies
-            // TODO: Should we save the cookies also after/only the handle page?
-            if (this.persistCookiesPerSession) {
-                const cookies = await crawlingContext.browserController.getCookies(page);
-                tryCancel();
-                session?.setCookies(cookies, request.loadedUrl);
-            }
-        }
-        if (!this.requestMatchesEnqueueStrategy(request)) {
-            this.log.debug(
-            // eslint-disable-next-line dot-notation
-            `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
-            request.noRetry = true;
-            request.state = RequestState.SKIPPED;
-            return;
-        }
-        if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
-            if (error)
-                throw new SessionError(error);
-        }
-        request.state = RequestState.REQUEST_HANDLER;
-        try {
-            await addTimeoutToPromise(async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutInnerMillis, `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`);
-            request.state = RequestState.DONE;
-        }
-        catch (e) {
-            request.state = RequestState.ERROR;
-            throw e;
-        }
-        tryCancel();
-    }
-    _enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession) {
-        crawlingContext.page = page;
-        // This switch is because the crawlingContexts are created on per request basis.
-        // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
-        // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
-        // which configures proxy and session for every new request
         const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
-        crawlingContext.browserController = browserControllerInstance;
-        if (!createNewSession) {
-            crawlingContext.session = browserControllerInstance.launchContext.session;
-        }
-        if (!crawlingContext.proxyInfo) {
-            crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
-        }
-        crawlingContext.enqueueLinks = async (enqueueOptions) => {
-            return browserCrawlerEnqueueLinks({
-                options: enqueueOptions,
-                page,
-                requestQueue: await this.getRequestQueue(),
-                robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                onSkippedRequest: this.onSkippedRequest,
-                originalRequestUrl: crawlingContext.request.url,
-                finalRequestUrl: crawlingContext.request.loadedUrl,
-            });
+        return {
+            page,
+            get response() {
+                throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
+            },
+            browserController: browserControllerInstance,
+            session: useIncognitoPages
+                ? crawlingContext.session
+                : browserControllerInstance.launchContext.session,
+            proxyInfo: crawlingContext.proxyInfo ?? browserControllerInstance.launchContext.proxyInfo,
+            enqueueLinks: async (enqueueOptions = {}) => {
+                return browserCrawlerEnqueueLinks({
+                    options: enqueueOptions,
+                    page,
+                    requestQueue: await this.getRequestQueue(),
+                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+                    onSkippedRequest: this.onSkippedRequest,
+                    originalRequestUrl: crawlingContext.request.url,
+                    finalRequestUrl: crawlingContext.request.loadedUrl,
+                });
+            },
         };
     }
-    async _handleNavigation(crawlingContext) {
+    async performNavigation(crawlingContext) {
+        if (crawlingContext.request.skipNavigation) {
+            return {
+                request: new Proxy(crawlingContext.request, {
+                    get(target, propertyName, receiver) {
+                        if (propertyName === 'loadedUrl') {
+                            throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                        }
+                        return Reflect.get(target, propertyName, receiver);
+                    },
+                }),
+                get response() {
+                    throw new Error('The `response` property is not available - `skipNavigation` was used');
+                },
+            };
+        }
         const gotoOptions = { timeout: this.navigationTimeoutMillis };
         const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
         crawlingContext.request.state = RequestState.BEFORE_NAV;
@@ -267,8 +238,9 @@ export class BrowserCrawler extends BasicCrawler {
         tryCancel();
         const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
         await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+        let response;
         try {
-            crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
             await this._handleNavigationTimeout(crawlingContext, error);
@@ -279,6 +251,36 @@ export class BrowserCrawler extends BasicCrawler {
         tryCancel();
         crawlingContext.request.state = RequestState.AFTER_NAV;
         await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+        await this.processResponse(response, crawlingContext);
+        tryCancel();
+        // save cookies
+        // TODO: Should we save the cookies also after/only the handle page?
+        if (this.persistCookiesPerSession) {
+            const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
+            tryCancel();
+            crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl);
+        }
+        if (response !== undefined) {
+            return {
+                request: crawlingContext.request,
+                response,
+            };
+        }
+        return {
+            request: crawlingContext.request,
+        };
+    }
+    async handleBlockedRequestByContent(crawlingContext) {
+        if (this.retryOnBlocked) {
+            const error = await this.isRequestBlocked(crawlingContext);
+            if (error)
+                throw new SessionError(error);
+        }
+        return {};
+    }
+    async restoreRequestState(crawlingContext) {
+        crawlingContext.request.state = RequestState.REQUEST_HANDLER;
+        return {};
     }
     async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
         const sessionCookie = session?.getCookies(request.url) ?? [];
@@ -306,11 +308,8 @@ export class BrowserCrawler extends BasicCrawler {
             throw new SessionError(this._getMessageFromError(error));
         }
     }
-    /**
-     * Should be overridden in case of different automation library that does not support this response API.
-     */
-    async _responseHandler(crawlingContext) {
-        const { response, session, request, page } = crawlingContext;
+    async processResponse(response, crawlingContext) {
+        const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
             this.stats.registerStatusCode(status);

package/internals/browser-crawler.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAeA,OAAO,EACH,iCAAiC,EACjC,YAAY,EACZ,oBAAoB,IAAI,4BAA4B,EACpD,aAAa,EACb,yBAAyB,EACzB,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAG/E,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAiLhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAKpB,SAAQ,YAAqB;IA4CL;IA3CtB;;;OAGG;IACH,kBAAkB,CAAsB;IAExC;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAElD,0BAA0B,CAAkC;IAC5D,uBAAuB,CAAS;IAChC,gCAAgC,CAAS;IACzC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,wBAAwB,CAAU;IAElC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;QAC9E,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACrC,CAAC;IAEF;;OAEG;IACH,YACI,UAA0C,EAAE,EAC1B,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,yBAAyB,GAAG,EAAE,EAC9B,wBAAwB,EACxB,kBAAkB,EAClB,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,cAAc,EACd,oBAAoB,EACpB,QAAQ,EACR,iBAAiB,EACjB,aAAa,EACb,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,cAAc,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAI,IAAkB,CAAC;YAClF,yBAAyB,EACrB,qBAAqB,GAAG,yBAAyB,GAAG,iCAAiC;SAC5F,EACD,MAAM,CACT,CAAC;QA5BgB,WAAM,GAAN,MAAM,CAAkC;QA8B1D,YAAY;QACZ,IAAI,CAAC,0BAA0B,GAAI,cAAsB,IAAI,IAAI,CAAC,MAAM,CAAC;QACzE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,CAAC,6BAA6B;QAE/E,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,gCAAgC,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACzE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;QAED,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAElF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,IAAY;YAChB,cAAc,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC;YACzE,eAAe,EAAE,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,eAAe,CAAC;SACzF,CAAC,CAAC;IACP,CAAC;IAEkB,KAAK,CAAC,eAAe,CAAC,eAAwB;QAC7D,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEjC,+BAA+B;QAC/B,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACtG,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,4BAA4B,CAAC;QAEvC,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,iBAAiB;YAAE,OAAO,iCAAiC,iBAAiB,EAAE,CAAC;QAEnF,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAEhE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,EAAE;gBACtE,OAAO,EAAE,eAAe,CAAC,OAAO;aACnC,CAAC,CAAC;YACH,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,SAAS,GAAG,SAAS,EAAE,SAAS,CAAC;YAEhD,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;oBACvB,mBAAmB,EAAE,IAAI;iBAC5B,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAe,CAAC;QAC5E,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,+BAA+B;QAC/B,iEAAiE;QACjE,6EAA6E;QAC7E,+GAA+G;QAC/G,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,SAAS,EAAE,CAAC;YAEZ,eAAe;YACf,oEAAoE;YACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,SAAU,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,GAAG,CAAC,KAAK;YACV,wCAAwC;YACxC,oBAAoB,OAAO,CAAC,EAAE,mBAAmB,OAAO,CAAC,GAAG,mBAAmB,OAAO,CAAC,SAAS,qDAAqD,OAAO,CAAC,iBAAiB,CAAC,IAAI,CACtL,CAAC;YAEF,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;YAErC,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,eAAyC,CAAC,CAAC,EACvG,IAAI,CAAC,gCAAgC,EACrC,kCAAkC,IAAI,CAAC,gCAAgC,GAAG,IAAI,WAAW,CAC5F,CAAC;YAEF,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;QACD,SAAS,EAAE,CAAC;IAChB,CAAC;IAES,mCAAmC,CACzC,eAAwB,EACxB,IAAgB,EAChB,gBAA0B;QAE1B,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACkB,CAAC;QAClC,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAkB,CAAC;QACzF,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC;YAC7B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAsB,CAAC;QAC/F,CAAC;QAED,eAAe,CAAC,YAAY,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;YACpD,OAAO,0BAA0B,CAAC;gBAC9B,OAAO,EAAE,cAAc;gBACvB,IAAI;gBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;gBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;gBAC7E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;gBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;aACrD,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI,CAAC;YACD,eAAe,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1G,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAErE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YAEnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAW,EACtD,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAwB,EAAE,KAAY;QAC3E,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACrD,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOD;;OAEG;IACO,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QACrD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAe,EAAE,aAA4B;QAC9E,MAAM,oBAAoB,GAAiD,EAAE,CAAC;QAE9E,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACvE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,EAAE,EAAE,EAAE;gBAC3F,SAAS,EAAG,aAAa,CAAC,SAAoB,IAAI,SAAS;aAC9D,CAAC,CAAC;YACH,aAAa,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACxC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACF,aAAa,CAAC,aAA4B,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBACpE,aAAa,CAAC,aAA4B,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAES,+BAA+B,CAAC,OAAe,EAAE,iBAA+C;QACtG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;gBAClC,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAM,aAAa,CAAC,OAAmB,CAAC,EAAE,EAAE,CAAC;oBACvD,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAEI,CACP,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YACrD,iBAAiB,CAAC,EAAE,iEAA2C,GAAG,EAAE;gBAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAcL,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GACW;IAC1B,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,YAAY,CAAC;QAChB,YAAY;QACZ,aAAa;QACb,gBAAgB;QAChB,IAAI;QACJ,OAAO;QACP,GAAI,OAA+B;KACtC,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
+{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAiBA,OAAO,EACH,YAAY,EACZ,oBAAoB,IAAI,4BAA4B,EACpD,aAAa,EACb,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAG/E,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AA+M3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAepB,SAAQ,YAAwD;IAoDxC;IAnDtB;;;OAGG;IACH,kBAAkB,CAAsB;IAExC;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAEzC,iBAAiB,CAAU;IAC3B,aAAa,CAAU;IAEhC,uBAAuB,CAAS;IAChC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,wBAAwB,CAAU;IAElC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;KACjF,CAAC;IAEF;;OAEG;IACH,YACI,OASC,EACiB,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,wBAAwB,EACxB,kBAAkB,EAClB,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,iBAAiB,GAAG,KAAK,EACzB,sBAAsB,EACtB,aAAa,EACb,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,sBAAsB,EAAE,GAAG,EAAE,CACzB,sBAAsB,EAAE;iBACnB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBACtD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBAClE,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,aAAkE;SACpF,EACD,MAAM,CACT,CAAC;QA9BgB,WAAM,GAAN,MAAM,CAAkC;QAgC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;QAED,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAElF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,IAAY;YAChB,cAAc,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC;YACzE,eAAe,EAAE,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,eAAe,CAAC;SACzF,CAAC,CAAC;IACP,CAAC;IAES,oBAAoB;QAI1B,OAAO,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YACrD,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,OAGf,EAAE,EAAE;gBACD,OAAO,CAAC,uBAAuB,CAAC,KAAK,IAAI,EAAE;oBACvC,MAAM,OAAO,CAAC,IAAI;yBACb,KAAK,EAAE;yBACP,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACP,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC5B,eAA2E;QAE3E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,4BAA4B,CAAC;QAEvC,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,iBAAiB;YAAE,OAAO,iCAAiC,iBAAiB,EAAE,CAAC;QAEnF,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACrB,eAAgC;QAIhC,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAEhE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,EAAE;gBACtE,OAAO,EAAE,eAAe,CAAC,OAAO;aACnC,CAAC,CAAC;YACH,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,SAAS,GAAG,SAAS,EAAE,SAAS,CAAC;YAEhD,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;oBACvB,mBAAmB,EAAE,IAAI;iBAC5B,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAS,CAAC;QACtE,SAAS,EAAE,CAAC;QAEZ,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACQ,CAAC;QAExB,OAAO;YACH,IAAI;YACJ,IAAI,QAAQ;gBACR,MAAM,IAAI,KAAK,CACX,gMAAgM,CACnM,CAAC;YACN,CAAC;YACD,iBAAiB,EAAE,yBAAyB;YAC5C,OAAO,EAAE,iBAAiB;gBACtB,CAAC,CAAC,eAAe,CAAC,OAAO;gBACzB,CAAC,CAAE,yBAAyB,CAAC,aAAa,CAAC,OAAmB;YAClE,SAAS,EAAE,eAAe,CAAC,SAAS,IAAK,yBAAyB,CAAC,aAAa,CAAC,SAAuB;YACxG,YAAY,EAAE,KAAK,EAAE,iBAAsC,EAAE,EAAE,EAAE;gBAC7D,OAAO,0BAA0B,CAAC;oBAC9B,OAAO,EAAE,cAAc;oBACvB,IAAI;oBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;oBACvC,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QAIpD,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;oBACxC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI,QAA8B,CAAC;QAEnC,IAAI,CAAC;YACD,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAErE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YAEnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAEjF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACtD,SAAS,EAAE,CAAC;QAEZ,eAAe;QACf,oEAAoE;QACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACzF,SAAS,EAAE,CAAC;YACZ,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,SAAU,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,eAAe,CAAC,OAAiC;gBAC1D,QAAQ;aACX,CAAC;QACN,CAAC;QAED,OAAO;YACH,OAAO,EAAE,eAAe,CAAC,OAAiC;SAC7D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CACvC,eAA2E;QAE3E,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,eAAgC;QAC9D,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7D,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAA0B,EACrE,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAuC,EAAE,KAAY;QAC1F,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACrD,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOO,KAAK,CAAC,eAAe,CACzB,QAA8B,EAC9B,eAAuC;QAEvC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEnD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAe,EAAE,aAA4B;QAC9E,MAAM,oBAAoB,GAAiD,EAAE,CAAC;QAE9E,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACvE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,EAAE,EAAE,EAAE;gBAC3F,SAAS,EAAG,aAAa,CAAC,SAAoB,IAAI,SAAS;aAC9D,CAAC,CAAC;YACH,aAAa,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACxC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACF,aAAa,CAAC,aAA4B,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBACpE,aAAa,CAAC,aAA4B,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAES,+BAA+B,CAAC,OAAe,EAAE,iBAA+C;QACtG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;gBAClC,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAM,aAAa,CAAC,OAAmB,CAAC,EAAE,EAAE,CAAC;oBACvD,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAEI,CACP,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YACrD,iBAAiB,CAAC,EAAE,iEAA2C,GAAG,EAAE;gBAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAcL,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GACW;IAC1B,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,YAAY,CAAC;QAChB,YAAY;QACZ,aAAa;QACb,gBAAgB;QAChB,IAAI;QACJ,OAAO;QACP,GAAI,OAA+B;KACtC,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.1",
+    "version": "4.0.0-beta.11",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.1",
-        "@crawlee/browser-pool": "4.0.0-beta.1",
-        "@crawlee/types": "4.0.0-beta.1",
-        "@crawlee/utils": "4.0.0-beta.1",
+        "@crawlee/basic": "4.0.0-beta.11",
+        "@crawlee/browser-pool": "4.0.0-beta.11",
+        "@crawlee/types": "4.0.0-beta.11",
+        "@crawlee/utils": "4.0.0-beta.11",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "4375a5af786811feda9dc872bbbb3406b2f13974"
+    "gitHead": "790ff75aeb412a1fd306891c85b5cc15c337cfaf"
 }