@crawlee/browser 3.17.1-beta.7 → 3.17.1-beta.71

package/index.mjs

@@ -50,6 +50,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -79,6 +80,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -100,5 +102,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/browser-crawler.d.ts

@@ -331,6 +331,11 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     protected _cleanupContext(crawlingContext: Context): Promise<void>;
     private containsSelectors;
     protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
+    /**
+     * `BrowserCrawler` hands `BasicCrawler` a wrapper that opens a page before delegating, and keeps the user's
+     * own handler in `userProvidedRequestHandler` — so resolve router-aware metadata against that instead.
+     */
+    protected get userRequestHandler(): RequestHandler<Context>;
     /**
      * Wrapper around requestHandler that opens and closes pages etc.
      */

package/internals/browser-crawler.js

@@ -9,6 +9,7 @@ const browser_pool_1 = require("@crawlee/browser-pool");
 const utils_1 = require("@crawlee/utils");
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const timeout_1 = require("@apify/timeout");
+const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -185,7 +186,8 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         const { page } = crawlingContext;
         // Page creation may be aborted
         if (page) {
-            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
+            // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
+            await (0, timeout_1.addTimeoutToPromise)(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`).catch((error) => this.log.debug('Error while closing page', { error }));
         }
     }
     async containsSelectors(page, selectors) {
@@ -220,6 +222,13 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             return `Received blocked status code: ${blockedStatusCode}`;
         return false;
     }
+    /**
+     * `BrowserCrawler` hands `BasicCrawler` a wrapper that opens a page before delegating, and keeps the user's
+     * own handler in `userProvidedRequestHandler` — so resolve router-aware metadata against that instead.
+     */
+    get userRequestHandler() {
+        return this.userProvidedRequestHandler;
+    }
     /**
      * Wrapper around requestHandler that opens and closes pages etc.
      */
@@ -475,8 +484,8 @@ async function browserCrawlerEnqueueLinks(options) {
     if (containsEnqueueLinks(options)) {
         return options.enqueueLinks({
             urls,
-            baseUrl,
             ...enqueueLinksOptions,
+            baseUrl,
         });
     }
     return (0, basic_1.enqueueLinks)({
@@ -484,8 +493,8 @@ async function browserCrawlerEnqueueLinks(options) {
         robotsTxtFile: options.robotsTxtFile,
         onSkippedRequest: options.onSkippedRequest,
         urls,
-        baseUrl,
         ...enqueueLinksOptions,
+        baseUrl,
     });
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "3.17.1-beta.7",
+    "version": "3.17.1-beta.71",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -53,11 +53,11 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.0",
-        "@crawlee/basic": "3.17.1-beta.7",
-        "@crawlee/browser-pool": "3.17.1-beta.7",
-        "@crawlee/types": "3.17.1-beta.7",
-        "@crawlee/utils": "3.17.1-beta.7",
+        "@apify/timeout": "^0.4.0",
+        "@crawlee/basic": "3.17.1-beta.71",
+        "@crawlee/browser-pool": "3.17.1-beta.71",
+        "@crawlee/types": "3.17.1-beta.71",
+        "@crawlee/utils": "3.17.1-beta.71",
         "ow": "^0.28.1",
         "tslib": "^2.4.0",
         "type-fest": "^4.0.0"
@@ -81,5 +81,5 @@
             }
         }
     },
-    "gitHead": "541c9687ee7f8421189d489ea817aaa08cb9b4ec"
+    "gitHead": "94c4c029140e4068184ad3f03d0a71fd8d6984dc"
 }