@crawlee/http 4.0.0-beta.110 → 4.0.0-beta.112

package/internals/http-crawler.d.ts

@@ -2,7 +2,7 @@ import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions,
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import type { JsonValue } from 'type-fest';
 /**
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
@@ -269,6 +269,8 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022

package/internals/http-crawler.js

@@ -1,9 +1,9 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
 import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
-import { getCookiesFromResponse } from '@crawlee/core';
+import { RequestThrottledError, getCookiesFromResponse } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
+import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
 import ow from 'ow';
@@ -262,6 +262,17 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
+        // Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
+        // error is still a rate limit the domain should back off from.
+        if (crawlingContext.response.status === 429) {
+            const retryAfter = crawlingContext.response.headers.get('retry-after');
+            if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                // This is the one path that never reads the body, so cancel it to release the connection
+                // rather than leaving it to the garbage collector.
+                await crawlingContext.response.body?.cancel().catch(() => { });
+                throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+            }
+        }
         // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
         // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
         // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.

package/internals/utils.js

@@ -1,6 +1,6 @@
 import { extname } from 'node:path';
 import { Readable } from 'node:stream';
-import { applySearchParams } from '@crawlee/utils';
+import { applySearchParams } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import mime from 'mime-types';
 import ow, { ObjectPredicate } from 'ow';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.110",
+    "version": "4.0.0-beta.112",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.110",
-        "@crawlee/core": "4.0.0-beta.110",
-        "@crawlee/http-client": "4.0.0-beta.110",
-        "@crawlee/types": "4.0.0-beta.110",
-        "@crawlee/utils": "4.0.0-beta.110",
+        "@crawlee/basic": "4.0.0-beta.112",
+        "@crawlee/core": "4.0.0-beta.112",
+        "@crawlee/http-client": "4.0.0-beta.112",
+        "@crawlee/types": "4.0.0-beta.112",
+        "@crawlee/utils": "4.0.0-beta.112",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "71f42c48f431ca46b04491eca65b191691e75033"
+    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
 }