@crawlee/browser 4.0.0-beta.105 → 4.0.0-beta.107

package/internals/browser-crawler.d.ts

@@ -227,8 +227,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
  * @category Crawlers
  */
 export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
-    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
-    private browserPoolDep;
+    #private;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
@@ -237,10 +236,6 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
     protected readonly ignoreShadowRoots: boolean;
     protected readonly ignoreIframes: boolean;
-    private readonly navigationTimeoutMillis;
-    private readonly preNavigationHooks;
-    private readonly postNavigationHooks;
-    private readonly saveResponseCookies;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -367,7 +362,7 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
      * Transforms proxy-related errors to `SessionError`.
      */
     private throwIfProxyError;
-    protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
+    protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
     private processResponse;
     /**
      * Function for cleaning up after all requests are processed.

package/internals/browser-crawler.js

@@ -58,21 +58,21 @@ function isNavigationTimeoutError(error) {
  */
 export class BrowserCrawler extends BasicCrawler {
     /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
-    browserPoolDep;
+    #browserPoolDep;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
     get browserPool() {
-        return this.browserPoolDep.value;
+        return this.#browserPoolDep.value;
     }
     launchContext;
     ignoreShadowRoots;
     ignoreIframes;
-    navigationTimeoutMillis;
-    preNavigationHooks;
-    postNavigationHooks;
-    saveResponseCookies;
+    #navigationTimeoutMillis;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number.greaterThan(0),
@@ -103,18 +103,18 @@ export class BrowserCrawler extends BasicCrawler {
                 // hook eats into the same window the navigation uses. The navigation itself is bounded by
                 // capping its `gotoOptions.timeout` to the remaining budget.
                 const windowGuard = (step) => skipGuard(async (ctx) => {
-                    const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+                    const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
                     if (remaining <= 0) {
-                        throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                        throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
                     }
-                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
                 });
                 let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
-                for (const hook of this.preNavigationHooks) {
+                for (const hook of this.#preNavigationHooks) {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
                 pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
-                for (const hook of this.postNavigationHooks) {
+                for (const hook of this.#postNavigationHooks) {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
                 return pipeline
@@ -125,23 +125,23 @@ export class BrowserCrawler extends BasicCrawler {
             extendContext,
         });
         this.launchContext = launchContext;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         // The public option hooks are extension-aware; internal storage uses the base context type
         // (the pipeline composes hooks against the concrete context, which does not statically carry
         // `ContextExtension`). The extension members are present at runtime regardless.
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = postNavigationHooks;
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = postNavigationHooks;
         this.ignoreIframes = ignoreIframes;
         this.ignoreShadowRoots = ignoreShadowRoots;
         if (headless != null) {
             this.launchContext.launchOptions ??= {};
             this.launchContext.launchOptions.headless = headless;
         }
-        this.saveResponseCookies = saveResponseCookies;
+        this.#saveResponseCookies = saveResponseCookies;
         // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.
         // The default is only built when no pool was injected, so all the option/launchContext fiddling below stays
         // inside the factory.
-        this.browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {
+        this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {
             const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
             if (launchContext?.userAgent) {
                 if (resolvedBrowserPoolOptions.useFingerprints)
@@ -168,7 +168,7 @@ export class BrowserCrawler extends BasicCrawler {
         });
     }
     getNavigationTimeoutMillis() {
-        return this.navigationTimeoutMillis;
+        return this.#navigationTimeoutMillis;
     }
     buildContextPipeline() {
         return ContextPipeline.create().compose({
@@ -263,31 +263,31 @@ export class BrowserCrawler extends BasicCrawler {
         return {
             // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
             // to the remaining shared window unless a hook overrode it (see there).
-            gotoOptions: { timeout: this.navigationTimeoutMillis },
-            [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
+            gotoOptions: { timeout: this.#navigationTimeoutMillis },
+            [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
         };
     }
     async navigate(crawlingContext) {
         tryCancel();
         const gotoOptions = crawlingContext.gotoOptions;
-        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
         if (remaining <= 0) {
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
         // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
         // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
         // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
         // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
         const gotoTimeout = gotoOptions;
-        if (gotoTimeout.timeout === this.navigationTimeoutMillis) {
+        if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
             gotoTimeout.timeout = remaining;
         }
         const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
-        const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
+        const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
         await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
         let response;
         try {
-            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
             await this.handleNavigationTimeout(crawlingContext, error);
@@ -320,7 +320,7 @@ export class BrowserCrawler extends BasicCrawler {
      * Copies cookies from the live browser page into the session cookie jar.
      */
     async persistCookiesFromPage(crawlingContext) {
-        if (!this.saveResponseCookies || !crawlingContext.session) {
+        if (!this.#saveResponseCookies || !crawlingContext.session) {
             return;
         }
         const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
@@ -329,7 +329,7 @@ export class BrowserCrawler extends BasicCrawler {
         const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
         for (const cookie of cookies) {
             try {
-                crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
                     ignoreError: false,
                 });
             }
@@ -370,7 +370,9 @@ export class BrowserCrawler extends BasicCrawler {
         return {};
     }
     async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
+        const sessionCookie = session
+            ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
+            : [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
@@ -390,7 +392,7 @@ export class BrowserCrawler extends BasicCrawler {
             session?.markBad();
             // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
             // hooks have run), so it names that value in its own error; report the configured window instead.
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
     }
     /**
@@ -398,7 +400,7 @@ export class BrowserCrawler extends BasicCrawler {
      */
     throwIfProxyError(error) {
         if (this.isProxyError(error)) {
-            throw new SessionError(this._getMessageFromError(error));
+            throw new SessionError(this.getMessageFromError(error));
         }
     }
     async processResponse(response, crawlingContext) {
@@ -415,7 +417,7 @@ export class BrowserCrawler extends BasicCrawler {
         }
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(response.status());
+                this.throwOnBlockedRequest(response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -428,7 +430,7 @@ export class BrowserCrawler extends BasicCrawler {
      * @ignore
      */
     async teardown() {
-        await this.browserPoolDep.ifOwned((pool) => pool.destroy());
+        await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
         await super.teardown();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.105",
+    "version": "4.0.0-beta.107",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.105",
-        "@crawlee/browser-pool": "4.0.0-beta.105",
-        "@crawlee/types": "4.0.0-beta.105",
-        "@crawlee/utils": "4.0.0-beta.105",
+        "@crawlee/basic": "4.0.0-beta.107",
+        "@crawlee/browser-pool": "4.0.0-beta.107",
+        "@crawlee/types": "4.0.0-beta.107",
+        "@crawlee/utils": "4.0.0-beta.107",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
+    "gitHead": "5a3cb6242d0ae261df93c5e03cfebbb9b736e6b7"
 }