@crawlee/browser 4.0.0-beta.110 → 4.0.0-beta.112

package/internals/browser-crawler.d.ts

@@ -7,6 +7,8 @@ import type { ReadonlyDeep } from 'type-fest';
 import type { BrowserLaunchContext } from './browser-launcher.js';
 interface BaseResponse {
     status(): number;
+    /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
+    headers?(): Record<string, string>;
 }
 export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
@@ -265,6 +267,8 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022

package/internals/browser-crawler.js

@@ -1,6 +1,7 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
-import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
+import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
+import { sleep } from '@crawlee/utils';
 import ow from 'ow';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
@@ -408,6 +409,15 @@ export class BrowserCrawler extends BasicCrawler {
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
             this.stats.registerStatusCode(status);
+            // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
+            // rate limit the domain should back off from.
+            if (status === 429) {
+                // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
+                const retryAfter = response.headers?.()['retry-after'];
+                if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                    throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+                }
+            }
             if (this.isErrorStatusCode(status)) {
                 if (this.additionalHttpErrorStatusCodes.has(status)) {
                     throw new Error(`${status} - Error status code was set by user.`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.110",
+    "version": "4.0.0-beta.112",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.110",
-        "@crawlee/browser-pool": "4.0.0-beta.110",
-        "@crawlee/types": "4.0.0-beta.110",
-        "@crawlee/utils": "4.0.0-beta.110",
+        "@crawlee/basic": "4.0.0-beta.112",
+        "@crawlee/browser-pool": "4.0.0-beta.112",
+        "@crawlee/types": "4.0.0-beta.112",
+        "@crawlee/utils": "4.0.0-beta.112",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "71f42c48f431ca46b04491eca65b191691e75033"
+    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
 }