@crawlee/http-client 4.0.0-beta.116 → 4.0.0-beta.117

package/base-http-client.d.ts

@@ -24,6 +24,14 @@ export interface CustomFetchOptions {
      * rest on a best-effort basis. Sourced from `SendRequestOptions.session.fingerprint`.
      */
     fingerprint?: SessionFingerprint;
+    /**
+     * When `true`, TLS certificate errors should be ignored for this request.
+     * Set when `SendRequestOptions.ignoreTlsErrors` is passed (e.g. from the
+     * `ignoreTlsErrors` crawler option) or when the session's proxy is a MITM
+     * proxy (`session.proxyInfo.ignoreTlsErrors`). Best-effort: clients that
+     * cannot disable TLS verification ignore it.
+     */
+    ignoreTlsErrors?: boolean;
 }
 /**
  * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed

package/base-http-client.js

@@ -56,6 +56,7 @@ export class BaseHttpClient {
             cookieJar,
             signal,
             fingerprint: options?.session?.fingerprint,
+            ignoreTlsErrors: options?.ignoreTlsErrors || options?.session?.proxyInfo?.ignoreTlsErrors,
         };
     }
     async #createDefaultCookieJar() {
@@ -107,7 +108,7 @@ export class BaseHttpClient {
         const maxRedirects = 10;
         let currentRequest = initialRequest;
         let redirectCount = 0;
-        const { proxyUrl, cookieJar, signal, fingerprint } = await this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.resolveRequestContext(options);
         currentRequest = initialRequest.clone();
         while (true) {
             await this.applyCookies(currentRequest, cookieJar);
@@ -116,6 +117,7 @@ export class BaseHttpClient {
                 proxyUrl,
                 cookieJar,
                 fingerprint,
+                ignoreTlsErrors,
                 redirect: 'manual',
             });
             await this.setCookies(response, cookieJar);

package/fetch-http-client.d.ts

@@ -1,3 +1,4 @@
+import type { CrawleeLogger } from '@crawlee/types';
 import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
 /**
  * A HTTP client implementation using the native `fetch` API.
@@ -5,5 +6,9 @@ import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export declare class FetchHttpClient extends BaseHttpClient {
+    #private;
+    constructor(options?: {
+        logger?: CrawleeLogger;
+    });
     fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }

package/fetch-http-client.js

@@ -5,7 +5,16 @@ import { BaseHttpClient } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export class FetchHttpClient extends BaseHttpClient {
+    #logger;
+    constructor(options) {
+        super(options);
+        this.#logger = options?.logger;
+    }
     async fetch(request, options) {
+        if (options?.ignoreTlsErrors) {
+            this.#logger?.warningOnce('FetchHttpClient cannot disable TLS certificate verification, the `ignoreTlsErrors` option is ignored. ' +
+                'Install the optional @crawlee/impit-client dependency to make it work.');
+        }
         return fetch(request, options);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http-client",
-    "version": "4.0.0-beta.116",
+    "version": "4.0.0-beta.117",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -47,7 +47,7 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-beta.116",
+        "@crawlee/types": "4.0.0-beta.117",
         "tough-cookie": "^6.0.0"
     },
     "lerna": {
@@ -57,5 +57,5 @@
             }
         }
     },
-    "gitHead": "2bdc17928f47674f08fff7a83de861d881b7947a"
+    "gitHead": "7c9c8c83da02da0444130cb95368045d30881c3b"
 }