@crawlee/browser 4.0.0-beta.95 → 4.0.0-beta.97

package/internals/browser-crawler.d.ts

@@ -158,7 +158,11 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      */
     postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
-     * Timeout in which page navigation needs to finish, in seconds.
+     * Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,
+     * the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the
+     * navigation uses. Separate from the
+     * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
+     * request handler.
      */
     navigationTimeoutSecs?: number;
     /**
@@ -332,6 +336,7 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext> & {
         contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
     });
+    protected getNavigationTimeoutMillis(): number;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
     private containsSelectors;
     private isRequestBlocked;
@@ -339,6 +344,15 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     private prepareNavigation;
     private navigate;
     private finalizeNavigation;
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    private persistCookiesFromPage;
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     private handleBlockedRequestByContent;
     private restoreRequestState;
     private applyCookies;

package/internals/browser-crawler.js

@@ -1,10 +1,21 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, handleRequestTimeout, NavigationSkippedError, OwnedOrInjected, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
-import { tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
 const readContextField = (ctx, key) => ctx[key];
+/**
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
+ * naming the raw millisecond value rather than the configured window.
+ */
+function isNavigationTimeoutError(error) {
+    return (error instanceof TimeoutError ||
+        error?.name === 'TimeoutError' ||
+        error?.constructor?.name === 'TimeoutError' ||
+        /timeout.*exceeded/i.test(error?.message ?? ''));
+}
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -87,13 +98,24 @@ export class BrowserCrawler extends BasicCrawler {
         super({
             ...basicCrawlerOptions,
             contextPipelineBuilder: () => {
+                // A single navigation window covers the pre-navigation hooks, the navigation, and the
+                // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
+                // hook eats into the same window the navigation uses. The navigation itself is bounded by
+                // capping its `gotoOptions.timeout` to the remaining budget.
+                const windowGuard = (step) => skipGuard(async (ctx) => {
+                    const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+                    if (remaining <= 0) {
+                        throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                    }
+                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                });
                 let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
                 for (const hook of this.preNavigationHooks) {
-                    pipeline = pipeline.compose(skipGuard(hook));
+                    pipeline = pipeline.compose(windowGuard(hook));
                 }
                 pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
                 for (const hook of this.postNavigationHooks) {
-                    pipeline = pipeline.compose(skipGuard(hook));
+                    pipeline = pipeline.compose(windowGuard(hook));
                 }
                 return pipeline
                     .compose(skipGuard(this.finalizeNavigation.bind(this)))
@@ -145,6 +167,9 @@ export class BrowserCrawler extends BasicCrawler {
             });
         });
     }
+    getNavigationTimeoutMillis() {
+        return this.navigationTimeoutMillis;
+    }
     buildContextPipeline() {
         return ContextPipeline.create().compose({
             action: this.preparePage.bind(this),
@@ -236,6 +261,8 @@ export class BrowserCrawler extends BasicCrawler {
         }
         crawlingContext.request.state = RequestState.BEFORE_NAV;
         return {
+            // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
+            // to the remaining shared window unless a hook overrode it (see there).
             gotoOptions: { timeout: this.navigationTimeoutMillis },
             [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
         };
@@ -243,6 +270,18 @@ export class BrowserCrawler extends BasicCrawler {
     async navigate(crawlingContext) {
         tryCancel();
         const gotoOptions = crawlingContext.gotoOptions;
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        }
+        // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
+        // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
+        // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
+        // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
+        const gotoTimeout = gotoOptions;
+        if (gotoTimeout.timeout === this.navigationTimeoutMillis) {
+            gotoTimeout.timeout = remaining;
+        }
         const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
         const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
         await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
@@ -272,23 +311,51 @@ export class BrowserCrawler extends BasicCrawler {
         }
         await this.processResponse(response, crawlingContext);
         tryCancel();
-        // TODO: Should we save the cookies also after/only the handle page?
-        if (this.saveResponseCookies && crawlingContext.session) {
-            const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
-            tryCancel();
-            const url = crawlingContext.request.loadedUrl;
-            for (const cookie of cookies) {
+        // Persist cookies from the navigation response before the user handler runs.
+        // Cookies set during `requestHandler` are saved again afterwards.
+        await this.persistCookiesFromPage(crawlingContext);
+        return { request: crawlingContext.request };
+    }
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    async persistCookiesFromPage(crawlingContext) {
+        if (!this.saveResponseCookies || !crawlingContext.session) {
+            return;
+        }
+        const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
+        tryCancel();
+        // Prefer the live page URL — the handler may have navigated after the initial load.
+        const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
+        for (const cookie of cookies) {
+            try {
+                crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                    ignoreError: false,
+                });
+            }
+            catch (e) {
+                this.log.debug(`Could not set cookie: ${e.message}`);
+            }
+        }
+    }
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    async runRequestHandler(crawlingContext) {
+        try {
+            await super.runRequestHandler(crawlingContext);
+        }
+        finally {
+            if (!crawlingContext.request.skipNavigation) {
                 try {
-                    crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
-                        ignoreError: false,
-                    });
+                    await this.persistCookiesFromPage(crawlingContext);
                 }
-                catch (e) {
-                    this.log.debug(`Could not set cookie: ${e.message}`);
+                catch {
+                    // Page may already be closed on some failure paths; ignore.
                 }
             }
         }
-        return { request: crawlingContext.request };
     }
     async handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
@@ -316,12 +383,15 @@ export class BrowserCrawler extends BasicCrawler {
      */
     async handleNavigationTimeout(crawlingContext, error) {
         const { session, page } = crawlingContext;
-        if (error?.constructor.name === 'TimeoutError') {
-            handleRequestTimeout({ session, errorMessage: error.message });
-        }
         // Fire-and-forget: no user code will run on this page after a failed navigation.
         // Swallow rejections: the page may already be detached.
         void page.evaluate(() => window.stop()).catch(() => { });
+        if (isNavigationTimeoutError(error)) {
+            session?.markBad();
+            // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
+            // hooks have run), so it names that value in its own error; report the configured window instead.
+            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        }
     }
     /**
      * Transforms proxy-related errors to `SessionError`.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.95",
+    "version": "4.0.0-beta.97",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -47,11 +47,11 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.95",
-        "@crawlee/browser-pool": "4.0.0-beta.95",
-        "@crawlee/types": "4.0.0-beta.95",
-        "@crawlee/utils": "4.0.0-beta.95",
+        "@apify/timeout": "^0.4.4",
+        "@crawlee/basic": "4.0.0-beta.97",
+        "@crawlee/browser-pool": "4.0.0-beta.97",
+        "@crawlee/types": "4.0.0-beta.97",
+        "@crawlee/utils": "4.0.0-beta.97",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "ba8602d011706fb5c9930232402156eb5b657eb3"
+    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
 }