@crawlee/http 4.0.0-beta.216 → 4.0.0-beta.218

package/internals/http-crawler.js

@@ -3,7 +3,7 @@ import util from 'node:util';
 import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import iconv from 'iconv-lite';
 import { z } from 'zod';
 import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
@@ -488,13 +488,11 @@ export class HttpCrawler extends BasicCrawler {
                 this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
-            try {
-                const parsedType = contentTypeParser.parse(mimeType);
-                this.#supportedMimeTypes.add(parsedType.type);
-            }
-            catch (err) {
+            const parsedType = parseContentType(mimeType);
+            if (!isTypeValid(parsedType.type)) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
             }
+            this.#supportedMimeTypes.add(parsedType.type);
         }
     }
     /**

package/internals/utils.js

@@ -1,7 +1,7 @@
 import { extname } from 'node:path';
 import { Readable } from 'node:stream';
 import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import mime from 'mime-types';
 import { z } from 'zod';
 const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
@@ -59,18 +59,16 @@ export function parseContentTypeFromResponse(response) {
     const { url, headers } = response;
     let parsedContentType;
     if (headers.get('content-type')) {
-        try {
-            parsedContentType = contentTypeParser.parse(headers.get('content-type'));
-        }
-        catch {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
+        const parsed = parseContentType(headers.get('content-type'));
+        // Can not parse content type from Content-Type header. Try to parse it from file extension.
+        if (isTypeValid(parsed.type))
+            parsedContentType = parsed;
     }
     // Parse content type from file extension as fallback
     if (!parsedContentType) {
         const parsedUrl = new URL(url);
         const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
+        parsedContentType = parseContentType(contentTypeFromExtname);
     }
     return {
         type: parsedContentType.type,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.216",
+    "version": "4.0.0-beta.218",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,14 +49,13 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/basic": "4.0.0-beta.216",
-        "@crawlee/core": "4.0.0-beta.216",
-        "@crawlee/http-client": "4.0.0-beta.216",
-        "@crawlee/types": "4.0.0-beta.216",
-        "@crawlee/utils": "4.0.0-beta.216",
-        "@types/content-type": "^1.1.8",
+        "@crawlee/basic": "4.0.0-beta.218",
+        "@crawlee/core": "4.0.0-beta.218",
+        "@crawlee/http-client": "4.0.0-beta.218",
+        "@crawlee/types": "4.0.0-beta.218",
+        "@crawlee/utils": "4.0.0-beta.218",
         "cheerio": "^1.0.0",
-        "content-type": "^1.0.5",
+        "content-type": "^3.1.1",
         "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
         "tslib": "^2.8.1",
@@ -70,5 +69,5 @@
             }
         }
     },
-    "gitHead": "0faae75ba532836e8510f0733893e754ea3f823f"
+    "gitHead": "53560afb8d133a1a08b95cda31c83f988f0b1429"
 }