@crawlee/http-client 4.0.0-beta.99 → 4.0.0-rc.0

package/base-http-client.d.ts

@@ -1,5 +1,4 @@
-import type { BaseHttpClient as BaseHttpClientInterface, CrawleeLogger, SendRequestOptions, SessionFingerprint } from '@crawlee/types';
-import { CookieJar } from 'tough-cookie';
+import type { BaseHttpClient as BaseHttpClientInterface, CookieJar, CrawleeLogger, SendRequestOptions, SessionFingerprint } from '@crawlee/types';
 /**
  * Per-request options handed to a concrete client's `fetch` implementation.
  */
@@ -25,6 +24,14 @@ export interface CustomFetchOptions {
      * rest on a best-effort basis. Sourced from `SendRequestOptions.session.fingerprint`.
      */
     fingerprint?: SessionFingerprint;
+    /**
+     * When `true`, TLS certificate errors should be ignored for this request.
+     * Set when `SendRequestOptions.ignoreTlsErrors` is passed (e.g. from the
+     * `ignoreTlsErrors` crawler option) or when the session's proxy is a MITM
+     * proxy (`session.proxyInfo.ignoreTlsErrors`). Best-effort: clients that
+     * cannot disable TLS verification ignore it.
+     */
+    ignoreTlsErrors?: boolean;
 }
 /**
  * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
@@ -32,7 +39,7 @@ export interface CustomFetchOptions {
  * implement only the low-level network call in `fetch`.
  */
 export declare abstract class BaseHttpClient implements BaseHttpClientInterface {
-    private log?;
+    #private;
     constructor(options?: {
         logger?: CrawleeLogger;
     });

package/base-http-client.js

@@ -1,13 +1,12 @@
-import { CookieJar } from 'tough-cookie';
 /**
  * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
  * behaviors (redirect handling, proxy and cookie handling). Concrete clients
  * implement only the low-level network call in `fetch`.
  */
 export class BaseHttpClient {
-    log;
+    #log;
     constructor(options) {
-        this.log = options?.logger;
+        this.#log = options?.logger;
     }
     async applyCookies(request, cookieJar) {
         try {
@@ -27,13 +26,13 @@ export class BaseHttpClient {
                 .split(/; */)
                 .filter(Boolean)
                 .map((pair) => merged.setCookie(pair, request.url)));
-            const cookieString = merged.getCookieStringSync(request.url);
+            const cookieString = await merged.getCookieString(request.url);
             if (cookieString) {
                 request.headers.set('cookie', cookieString);
             }
         }
         catch (e) {
-            this.log?.warning(`Failed to get cookies for URL "${request.url}": ${e.message}`);
+            this.#log?.warning(`Failed to get cookies for URL "${request.url}": ${e.message}`);
         }
         return request;
     }
@@ -44,21 +43,26 @@ export class BaseHttpClient {
                 await cookieJar.setCookie(header, response.url);
             }
             catch (e) {
-                this.log?.warning(`Failed to set cookie for URL "${response.url}": ${e.message}`);
+                this.#log?.warning(`Failed to set cookie for URL "${response.url}": ${e.message}`);
             }
         }
     }
-    resolveRequestContext(options) {
+    async resolveRequestContext(options) {
         const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
-        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar();
+        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? (await this.#createDefaultCookieJar());
         const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
         return {
             proxyUrl,
-            cookieJar: cookieJar,
+            cookieJar,
             signal,
             fingerprint: options?.session?.fingerprint,
+            ignoreTlsErrors: options?.ignoreTlsErrors || options?.session?.proxyInfo?.ignoreTlsErrors,
         };
     }
+    async #createDefaultCookieJar() {
+        const { CookieJar: ToughCookieJar } = await import('tough-cookie');
+        return new ToughCookieJar();
+    }
     createAbortSignal(signal, timeoutMillis) {
         if (signal && timeoutMillis) {
             return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
@@ -104,7 +108,7 @@ export class BaseHttpClient {
         const maxRedirects = 10;
         let currentRequest = initialRequest;
         let redirectCount = 0;
-        const { proxyUrl, cookieJar, signal, fingerprint } = this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.resolveRequestContext(options);
         currentRequest = initialRequest.clone();
         while (true) {
             await this.applyCookies(currentRequest, cookieJar);
@@ -113,6 +117,7 @@ export class BaseHttpClient {
                 proxyUrl,
                 cookieJar,
                 fingerprint,
+                ignoreTlsErrors,
                 redirect: 'manual',
             });
             await this.setCookies(response, cookieJar);

package/fetch-http-client.d.ts

@@ -1,3 +1,4 @@
+import type { CrawleeLogger } from '@crawlee/types';
 import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
 /**
  * A HTTP client implementation using the native `fetch` API.
@@ -5,5 +6,9 @@ import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export declare class FetchHttpClient extends BaseHttpClient {
+    #private;
+    constructor(options?: {
+        logger?: CrawleeLogger;
+    });
     fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }

package/fetch-http-client.js

@@ -5,7 +5,16 @@ import { BaseHttpClient } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export class FetchHttpClient extends BaseHttpClient {
+    #logger;
+    constructor(options) {
+        super(options);
+        this.#logger = options?.logger;
+    }
     async fetch(request, options) {
+        if (options?.ignoreTlsErrors) {
+            this.#logger?.warningOnce('FetchHttpClient cannot disable TLS certificate verification, the `ignoreTlsErrors` option is ignored. ' +
+                'Install the optional @crawlee/impit-client dependency to make it work.');
+        }
         return fetch(request, options);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http-client",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -47,7 +47,7 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-beta.99",
+        "@crawlee/types": "4.0.0-rc.0",
         "tough-cookie": "^6.0.0"
     },
     "lerna": {
@@ -57,5 +57,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }