@crawlee/browser 4.0.0-beta.95 → 4.0.0-beta.96

package/internals/browser-crawler.d.ts

@@ -339,6 +339,15 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
     private prepareNavigation;
     private navigate;
     private finalizeNavigation;
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    private persistCookiesFromPage;
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     private handleBlockedRequestByContent;
     private restoreRequestState;
     private applyCookies;

package/internals/browser-crawler.js

@@ -272,23 +272,51 @@ export class BrowserCrawler extends BasicCrawler {
         }
         await this.processResponse(response, crawlingContext);
         tryCancel();
-        // TODO: Should we save the cookies also after/only the handle page?
-        if (this.saveResponseCookies && crawlingContext.session) {
-            const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
-            tryCancel();
-            const url = crawlingContext.request.loadedUrl;
-            for (const cookie of cookies) {
+        // Persist cookies from the navigation response before the user handler runs.
+        // Cookies set during `requestHandler` are saved again afterwards.
+        await this.persistCookiesFromPage(crawlingContext);
+        return { request: crawlingContext.request };
+    }
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    async persistCookiesFromPage(crawlingContext) {
+        if (!this.saveResponseCookies || !crawlingContext.session) {
+            return;
+        }
+        const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
+        tryCancel();
+        // Prefer the live page URL — the handler may have navigated after the initial load.
+        const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
+        for (const cookie of cookies) {
+            try {
+                crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                    ignoreError: false,
+                });
+            }
+            catch (e) {
+                this.log.debug(`Could not set cookie: ${e.message}`);
+            }
+        }
+    }
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    async runRequestHandler(crawlingContext) {
+        try {
+            await super.runRequestHandler(crawlingContext);
+        }
+        finally {
+            if (!crawlingContext.request.skipNavigation) {
                 try {
-                    crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
-                        ignoreError: false,
-                    });
+                    await this.persistCookiesFromPage(crawlingContext);
                 }
-                catch (e) {
-                    this.log.debug(`Could not set cookie: ${e.message}`);
+                catch {
+                    // Page may already be closed on some failure paths; ignore.
                 }
             }
         }
-        return { request: crawlingContext.request };
     }
     async handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.95",
+    "version": "4.0.0-beta.96",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.95",
-        "@crawlee/browser-pool": "4.0.0-beta.95",
-        "@crawlee/types": "4.0.0-beta.95",
-        "@crawlee/utils": "4.0.0-beta.95",
+        "@crawlee/basic": "4.0.0-beta.96",
+        "@crawlee/browser-pool": "4.0.0-beta.96",
+        "@crawlee/types": "4.0.0-beta.96",
+        "@crawlee/utils": "4.0.0-beta.96",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "ba8602d011706fb5c9930232402156eb5b657eb3"
+    "gitHead": "b5c20fe6cecad57a6e3dd26b4137776d4b62588a"
 }