@crawlee/http 4.0.0-beta.116 → 4.0.0-beta.118

package/internals/http-crawler.d.ts

@@ -29,9 +29,14 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      */
     navigationTimeoutSecs?: number;
     /**
-     * If set to true, SSL certificate errors will be ignored.
+     * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
+     * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every navigation request, so custom
+     * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
+     * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
+     *
+     * @default true
      */
-    ignoreSslErrors?: boolean;
+    ignoreTlsErrors?: boolean;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
      * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
@@ -243,12 +248,11 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
     #private;
-    private ignoreSslErrors;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         additionalMimeTypes: import("ow").ArrayPredicate<string>;
 // @ts-ignore optional peer dependency or compatibility with es2022

package/internals/http-crawler.js

@@ -113,15 +113,14 @@ export class HttpCrawler extends BasicCrawler {
     #postNavigationHooks;
     #saveResponseCookies;
     #navigationTimeoutMillis;
-    // kept as TS-private: tests read it at runtime
-    ignoreSslErrors;
+    #ignoreTlsErrors;
     #suggestResponseEncoding;
     #forceResponseEncoding;
     #supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number,
-        ignoreSslErrors: ow.optional.boolean,
+        ignoreTlsErrors: ow.optional.boolean,
         additionalMimeTypes: ow.optional.array.ofType(ow.string),
         suggestResponseEncoding: ow.optional.string,
         forceResponseEncoding: ow.optional.string,
@@ -134,7 +133,7 @@ export class HttpCrawler extends BasicCrawler {
      */
     constructor(options = {}) {
         ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
+        const { navigationTimeoutSecs = 30, ignoreTlsErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
         // BasicCrawler
         contextPipelineBuilder, ...basicCrawlerOptions } = options;
         super({
@@ -149,7 +148,7 @@ export class HttpCrawler extends BasicCrawler {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
         this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
+        this.#ignoreTlsErrors = ignoreTlsErrors;
         this.#suggestResponseEncoding = suggestResponseEncoding;
         this.#forceResponseEncoding = forceResponseEncoding;
         // Cast away the extension-aware option types to the base internal storage types (see the field
@@ -436,22 +435,12 @@ export class HttpCrawler extends BasicCrawler {
             timeout: this.#navigationTimeoutMillis,
             sessionToken: session,
             headers: request.headers,
-            https: {
-                rejectUnauthorized: !this.ignoreSslErrors,
-            },
             body: undefined,
         };
         if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
             requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
             delete requestOptions.headers.cookie;
         }
-        // Disable SSL verification for MITM proxies
-        if (session.proxyInfo?.ignoreTlsErrors) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
-        }
         if (/PATCH|POST|PUT/.test(request.method))
             requestOptions.body = request.payload ?? '';
         return requestOptions;
@@ -553,6 +542,7 @@ export class HttpCrawler extends BasicCrawler {
             cookieJar,
             signal: cancelSignal,
             timeoutMillis: cancelSignal ? undefined : opts.timeout,
+            ignoreTlsErrors: this.#ignoreTlsErrors,
         });
         return response;
     };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.116",
+    "version": "4.0.0-beta.118",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.116",
-        "@crawlee/core": "4.0.0-beta.116",
-        "@crawlee/http-client": "4.0.0-beta.116",
-        "@crawlee/types": "4.0.0-beta.116",
-        "@crawlee/utils": "4.0.0-beta.116",
+        "@crawlee/basic": "4.0.0-beta.118",
+        "@crawlee/core": "4.0.0-beta.118",
+        "@crawlee/http-client": "4.0.0-beta.118",
+        "@crawlee/types": "4.0.0-beta.118",
+        "@crawlee/utils": "4.0.0-beta.118",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "2bdc17928f47674f08fff7a83de861d881b7947a"
+    "gitHead": "7306bd05a126662898a4f2fe7779f6b53425cadf"
 }