@crawlee/browser 4.0.0-beta.111 → 4.0.0-beta.113

package/internals/browser-crawler.d.ts

@@ -7,6 +7,8 @@ import type { ReadonlyDeep } from 'type-fest';
 import type { BrowserLaunchContext } from './browser-launcher.js';
 interface BaseResponse {
     status(): number;
+    /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
+    headers?(): Record<string, string>;
 }
 export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
@@ -265,6 +267,8 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022

package/internals/browser-crawler.js

@@ -1,4 +1,4 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
 import { sleep } from '@crawlee/utils';
@@ -409,6 +409,15 @@ export class BrowserCrawler extends BasicCrawler {
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
             this.stats.registerStatusCode(status);
+            // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
+            // rate limit the domain should back off from.
+            if (status === 429) {
+                // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
+                const retryAfter = response.headers?.()['retry-after'];
+                if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                    throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+                }
+            }
             if (this.isErrorStatusCode(status)) {
                 if (this.additionalHttpErrorStatusCodes.has(status)) {
                     throw new Error(`${status} - Error status code was set by user.`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.111",
+    "version": "4.0.0-beta.113",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.111",
-        "@crawlee/browser-pool": "4.0.0-beta.111",
-        "@crawlee/types": "4.0.0-beta.111",
-        "@crawlee/utils": "4.0.0-beta.111",
+        "@crawlee/basic": "4.0.0-beta.113",
+        "@crawlee/browser-pool": "4.0.0-beta.113",
+        "@crawlee/types": "4.0.0-beta.113",
+        "@crawlee/utils": "4.0.0-beta.113",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "a6b7dec651fa7d81c61689e0a3cd0a8d06fdb586"
+    "gitHead": "a80d4b05a84ecc651dcd68fafd7a1ff35e6e7532"
 }