@crawlee/browser 4.0.0-beta.80 → 4.0.0-beta.82

package/internals/browser-crawler.d.ts

@@ -31,8 +31,8 @@ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Re
      */
     enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
 }
-export type BrowserHook<Context = BrowserCrawlingContext> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
     launchContext?: BrowserLaunchContext<any, any>;
     /**
      * An existing browser pool instance to use. When provided, the crawler will use this pool directly instead of
@@ -124,8 +124,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      *
      * A hook may optionally return a partial object whose properties are merged into the crawling context,
      * allowing the hook to override context members for subsequent hooks and pipeline stages.
+     *
+     * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
+     * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
+     * This means the members added by `extendContext` are already available here, but navigation-dependent
+     * members (e.g. `page`, `response`) are not.
      */
-    preNavigationHooks?: BrowserHook<Context>[];
+    preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
@@ -151,7 +156,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      * ]
      * ```
      */
-    postNavigationHooks?: BrowserHook<Context>[];
+    postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
      * Timeout in which page navigation needs to finish, in seconds.
      */
@@ -233,10 +238,10 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
     protected readonly ignoreShadowRoots: boolean;
     protected readonly ignoreIframes: boolean;
-    protected navigationTimeoutMillis: number;
-    protected preNavigationHooks: BrowserHook<Context>[];
-    protected postNavigationHooks: BrowserHook<Context>[];
-    protected saveResponseCookies: boolean;
+    private readonly navigationTimeoutMillis;
+    private readonly preNavigationHooks;
+    private readonly postNavigationHooks;
+    private readonly saveResponseCookies;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -333,22 +338,22 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     });
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
     private containsSelectors;
-    protected isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false>;
+    private isRequestBlocked;
     private preparePage;
     private prepareNavigation;
     private navigate;
     private finalizeNavigation;
     private handleBlockedRequestByContent;
     private restoreRequestState;
-    protected _applyCookies({ session, request, page }: BrowserCrawlingContext<Page, Response>, preHooksCookies: string, postHooksCookies: string): Promise<void>;
+    private applyCookies;
     /**
      * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    protected _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void>;
+    private handleNavigationTimeout;
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    protected _throwIfProxyError(error: Error): void;
+    private throwIfProxyError;
     protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
     private processResponse;
     /**

package/internals/browser-crawler.js

@@ -104,10 +104,13 @@ export class BrowserCrawler extends BasicCrawler {
                     .compose({ action: this.handleBlockedRequestByContent.bind(this) })
                     .compose({ action: this.restoreRequestState.bind(this) });
             },
-            extendContext: extendContext,
+            extendContext,
         });
         this.launchContext = launchContext;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        // The public option hooks are extension-aware; internal storage uses the base context type
+        // (the pipeline composes hooks against the concrete context, which does not statically carry
+        // `ContextExtension`). The extension members are present at runtime regardless.
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
         this.ignoreIframes = ignoreIframes;
@@ -247,15 +250,15 @@ export class BrowserCrawler extends BasicCrawler {
         const gotoOptions = crawlingContext.gotoOptions;
         const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
         const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
-        await this._applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
+        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
         let response;
         try {
             response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
-            await this._handleNavigationTimeout(crawlingContext, error);
+            await this.handleNavigationTimeout(crawlingContext, error);
             crawlingContext.request.state = RequestState.ERROR;
-            this._throwIfProxyError(error);
+            this.throwIfProxyError(error);
             throw error;
         }
         tryCancel();
@@ -304,7 +307,7 @@ export class BrowserCrawler extends BasicCrawler {
         crawlingContext.request.state = RequestState.REQUEST_HANDLER;
         return {};
     }
-    async _applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
         const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
@@ -316,7 +319,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async _handleNavigationTimeout(crawlingContext, error) {
+    async handleNavigationTimeout(crawlingContext, error) {
         const { session, page } = crawlingContext;
         if (error?.constructor.name === 'TimeoutError') {
             handleRequestTimeout({ session, errorMessage: error.message });
@@ -328,7 +331,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    _throwIfProxyError(error) {
+    throwIfProxyError(error) {
         if (this.isProxyError(error)) {
             throw new SessionError(this._getMessageFromError(error));
         }

package/internals/browser-launcher.d.ts

@@ -110,11 +110,11 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
      */
     launch(): LaunchResult;
     createLaunchOptions(): Dictionary;
-    protected _getDefaultHeadlessOption(): boolean;
-    protected _getChromeExecutablePath(): string;
+    protected getDefaultHeadlessOption(): boolean;
+    private getChromeExecutablePath;
     /**
      * Gets a typical path to Chrome executable, depending on the current operating system.
      */
-    protected _getTypicalChromeExecutablePath(): string;
-    protected _validateProxyUrlProtocol(proxyUrl?: string): void;
+    private getTypicalChromeExecutablePath;
+    private validateProxyUrlProtocol;
 }

package/internals/browser-launcher.js

@@ -54,7 +54,7 @@ export class BrowserLauncher {
     constructor(launchContext, config = Configuration.getGlobalConfig()) {
         this.config = config;
         const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
-        this._validateProxyUrlProtocol(proxyUrl);
+        this.validateProxyUrlProtocol(proxyUrl);
         // those need to be reassigned otherwise they are {} in types
         this.launcher = launcher;
         this.proxyUrl = proxyUrl;
@@ -95,23 +95,23 @@ export class BrowserLauncher {
             launchOptions.args.push(`--user-agent=${this.userAgent}`);
         }
         if (launchOptions.headless == null) {
-            launchOptions.headless = this._getDefaultHeadlessOption();
+            launchOptions.headless = this.getDefaultHeadlessOption();
         }
         if (this.useChrome && !launchOptions.executablePath) {
-            launchOptions.executablePath = this._getChromeExecutablePath();
+            launchOptions.executablePath = this.getChromeExecutablePath();
         }
         return launchOptions;
     }
-    _getDefaultHeadlessOption() {
+    getDefaultHeadlessOption() {
         return this.config.headless && !this.config.xvfb;
     }
-    _getChromeExecutablePath() {
-        return this.config.chromeExecutablePath ?? this._getTypicalChromeExecutablePath();
+    getChromeExecutablePath() {
+        return this.config.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();
     }
     /**
      * Gets a typical path to Chrome executable, depending on the current operating system.
      */
-    _getTypicalChromeExecutablePath() {
+    getTypicalChromeExecutablePath() {
         /**
          * Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
          * Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
@@ -139,7 +139,7 @@ export class BrowserLauncher {
                 return '/usr/bin/google-chrome';
         }
     }
-    _validateProxyUrlProtocol(proxyUrl) {
+    validateProxyUrlProtocol(proxyUrl) {
         if (!proxyUrl)
             return;
         if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.82",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.80",
-        "@crawlee/browser-pool": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/basic": "4.0.0-beta.82",
+        "@crawlee/browser-pool": "4.0.0-beta.82",
+        "@crawlee/types": "4.0.0-beta.82",
+        "@crawlee/utils": "4.0.0-beta.82",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "eb1096f7c7743d124375ef011fbbadb19476822e"
 }