@crawlee/browser 3.17.1-beta.8 → 3.17.1-beta.80

package/index.mjs

@@ -50,6 +50,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -79,6 +80,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -100,5 +102,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/browser-crawler.d.ts

@@ -331,6 +331,11 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     protected _cleanupContext(crawlingContext: Context): Promise<void>;
     private containsSelectors;
     protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
+    /**
+     * `BrowserCrawler` hands `BasicCrawler` a wrapper that opens a page before delegating, and keeps the user's
+     * own handler in `userProvidedRequestHandler` — so resolve router-aware metadata against that instead.
+     */
+    protected get userRequestHandler(): RequestHandler<Context>;
     /**
      * Wrapper around requestHandler that opens and closes pages etc.
      */

package/internals/browser-crawler.js

@@ -9,6 +9,7 @@ const browser_pool_1 = require("@crawlee/browser-pool");
 const utils_1 = require("@crawlee/utils");
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const timeout_1 = require("@apify/timeout");
+const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -185,7 +186,8 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         const { page } = crawlingContext;
         // Page creation may be aborted
         if (page) {
-            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
+            // Puppeteer 25+ can hang `page.close()` indefinitely when the page's navigation was aborted, don't let it block the crawler.
+            await (0, timeout_1.addTimeoutToPromise)(async () => page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, `page.close() timed out after ${PAGE_CLOSE_TIMEOUT_MILLIS / 1000} seconds`).catch((error) => this.log.debug('Error while closing page', { error }));
         }
     }
     async containsSelectors(page, selectors) {
@@ -220,6 +222,13 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             return `Received blocked status code: ${blockedStatusCode}`;
         return false;
     }
+    /**
+     * `BrowserCrawler` hands `BasicCrawler` a wrapper that opens a page before delegating, and keeps the user's
+     * own handler in `userProvidedRequestHandler` — so resolve router-aware metadata against that instead.
+     */
+    get userRequestHandler() {
+        return this.userProvidedRequestHandler;
+    }
     /**
      * Wrapper around requestHandler that opens and closes pages etc.
      */
@@ -311,7 +320,9 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         const contextEnqueueLinks = crawlingContext.enqueueLinks;
         crawlingContext.enqueueLinks = async (enqueueOptions) => {
             return browserCrawlerEnqueueLinks({
-                options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
+                // `contextEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;
+                // pre-clamping it here would make the crawler log the internal limit as a user-provided one
+                options: enqueueOptions,
                 page,
                 requestQueue: await this.getRequestQueue(),
                 robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
@@ -475,8 +486,8 @@ async function browserCrawlerEnqueueLinks(options) {
     if (containsEnqueueLinks(options)) {
         return options.enqueueLinks({
             urls,
-            baseUrl,
             ...enqueueLinksOptions,
+            baseUrl,
         });
     }
     return (0, basic_1.enqueueLinks)({
@@ -484,8 +495,8 @@ async function browserCrawlerEnqueueLinks(options) {
         robotsTxtFile: options.robotsTxtFile,
         onSkippedRequest: options.onSkippedRequest,
         urls,
-        baseUrl,
         ...enqueueLinksOptions,
+        baseUrl,
     });
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "3.17.1-beta.8",
+    "version": "3.17.1-beta.80",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -53,11 +53,11 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.0",
-        "@crawlee/basic": "3.17.1-beta.8",
-        "@crawlee/browser-pool": "3.17.1-beta.8",
-        "@crawlee/types": "3.17.1-beta.8",
-        "@crawlee/utils": "3.17.1-beta.8",
+        "@apify/timeout": "^0.4.0",
+        "@crawlee/basic": "3.17.1-beta.80",
+        "@crawlee/browser-pool": "3.17.1-beta.80",
+        "@crawlee/types": "3.17.1-beta.80",
+        "@crawlee/utils": "3.17.1-beta.80",
         "ow": "^0.28.1",
         "tslib": "^2.4.0",
         "type-fest": "^4.0.0"
@@ -81,5 +81,5 @@
             }
         }
     },
-    "gitHead": "4f4b8c1d53b9185095ac68017aff0ef2b8182b6b"
+    "gitHead": "5b3c2d52f128643de6f515e87eeb531d55a187f0"
 }