@crawlee/http 4.0.0-beta.187 → 4.0.0-beta.189

package/internals/http-crawler.js

@@ -226,6 +226,7 @@ export class HttpCrawler extends BasicCrawler {
         request.state = RequestState.BEFORE_NAV;
         return {};
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async makeHttpRequest(crawlingContext) {
         tryCancel();
         const { request, session } = crawlingContext;
@@ -382,13 +383,12 @@ export class HttpCrawler extends BasicCrawler {
     async parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
-        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
-        const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
             this.statistics.registerStatusCode(status);
         }
         if (this.isErrorStatusCode(status)) {
-            const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
+            const { response: decoded } = this.encodeResponse(request, response, charset);
+            const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -404,26 +404,27 @@ export class HttpCrawler extends BasicCrawler {
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            if (!charset && !this.#forceResponseEncoding) {
-                const rawBytes = Buffer.from(await response.arrayBuffer());
-                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
-                const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
-                const body = iconv.encodingExists(charsetToUse)
-                    ? iconv.decode(rawBytes, charsetToUse)
-                    : rawBytes.toString('utf8');
-                return { response, contentType: { type, encoding: 'utf-8' }, body };
-            }
-            return { response, contentType, body: await reencodedResponse.text() };
+        if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
+            // The charset comes from the document itself, so the raw bytes are what we need -
+            // decoding them through `encodeResponse` first would consume the body for nothing.
+            const rawBytes = Buffer.from(await response.arrayBuffer());
+            const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+            const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
+            const body = iconv.encodingExists(charsetToUse)
+                ? iconv.decode(rawBytes, charsetToUse)
+                : rawBytes.toString('utf8');
+            return { response, contentType: { type, encoding: 'utf-8' }, body };
         }
-        else {
-            const body = Buffer.from(await reencodedResponse.bytes());
-            return {
-                body,
-                response,
-                contentType,
-            };
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
+        const contentType = { type, encoding };
+        if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            return { response, contentType, body: await reencodedResponse.text() };
         }
+        return {
+            body: Buffer.from(await reencodedResponse.bytes()),
+            response,
+            contentType,
+        };
     }
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.187",
+    "version": "4.0.0-beta.189",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/basic": "4.0.0-beta.187",
-        "@crawlee/core": "4.0.0-beta.187",
-        "@crawlee/http-client": "4.0.0-beta.187",
-        "@crawlee/types": "4.0.0-beta.187",
-        "@crawlee/utils": "4.0.0-beta.187",
+        "@crawlee/basic": "4.0.0-beta.189",
+        "@crawlee/core": "4.0.0-beta.189",
+        "@crawlee/http-client": "4.0.0-beta.189",
+        "@crawlee/types": "4.0.0-beta.189",
+        "@crawlee/utils": "4.0.0-beta.189",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "1281760f20f7ee1181e3171d759efb055647e6c5"
+    "gitHead": "1dbb0f272b9da9c7a6bf3bdc1ae48f84eda3f21c"
 }