@crawlee/http-client 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/base-http-client.d.ts

@@ -1,5 +1,4 @@
-import type { BaseHttpClient as BaseHttpClientInterface, CrawleeLogger, SendRequestOptions, SessionFingerprint } from '@crawlee/types';
-import { CookieJar } from 'tough-cookie';
+import type { BaseHttpClient as BaseHttpClientInterface, CookieJar, CrawleeLogger, SendRequestOptions, SessionFingerprint } from '@crawlee/types';
 /**
  * Per-request options handed to a concrete client's `fetch` implementation.
  */
@@ -25,6 +24,14 @@ export interface CustomFetchOptions {
      * rest on a best-effort basis. Sourced from `SendRequestOptions.session.fingerprint`.
      */
     fingerprint?: SessionFingerprint;
+    /**
+     * When `true`, TLS certificate errors should be ignored for this request.
+     * Set when `SendRequestOptions.ignoreTlsErrors` is passed (e.g. from the
+     * `ignoreTlsErrors` crawler option) or when the session's proxy is a MITM
+     * proxy (`session.proxyInfo.ignoreTlsErrors`). Best-effort: clients that
+     * cannot disable TLS verification ignore it.
+     */
+    ignoreTlsErrors?: boolean;
 }
 /**
  * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
@@ -32,7 +39,7 @@ export interface CustomFetchOptions {
  * implement only the low-level network call in `fetch`.
  */
 export declare abstract class BaseHttpClient implements BaseHttpClientInterface {
-    private log?;
+    #private;
     constructor(options?: {
         logger?: CrawleeLogger;
     });
@@ -41,12 +48,6 @@ export declare abstract class BaseHttpClient implements BaseHttpClientInterface
      * automatic redirect following or special error handling.
      */
     protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
-    private applyCookies;
-    private setCookies;
-    private resolveRequestContext;
-    private createAbortSignal;
-    private isRedirect;
-    private buildRedirectRequest;
     /**
      * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
      */

package/base-http-client.js

@@ -1,15 +1,14 @@
-import { CookieJar } from 'tough-cookie';
 /**
  * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
  * behaviors (redirect handling, proxy and cookie handling). Concrete clients
  * implement only the low-level network call in `fetch`.
  */
 export class BaseHttpClient {
-    log;
+    #log;
     constructor(options) {
-        this.log = options?.logger;
+        this.#log = options?.logger;
     }
-    async applyCookies(request, cookieJar) {
+    async #applyCookies(request, cookieJar) {
         try {
             const requestCookies = request.headers.get('cookie') ?? '';
             if (!requestCookies) {
@@ -27,39 +26,44 @@ export class BaseHttpClient {
                 .split(/; */)
                 .filter(Boolean)
                 .map((pair) => merged.setCookie(pair, request.url)));
-            const cookieString = merged.getCookieStringSync(request.url);
+            const cookieString = await merged.getCookieString(request.url);
             if (cookieString) {
                 request.headers.set('cookie', cookieString);
             }
         }
         catch (e) {
-            this.log?.warning(`Failed to get cookies for URL "${request.url}": ${e.message}`);
+            this.#log?.warning(`Failed to get cookies for URL "${request.url}": ${e.message}`);
         }
         return request;
     }
-    async setCookies(response, cookieJar) {
+    async #setCookies(response, cookieJar) {
         const setCookieHeaders = response.headers.getSetCookie();
         for (const header of setCookieHeaders) {
             try {
                 await cookieJar.setCookie(header, response.url);
             }
             catch (e) {
-                this.log?.warning(`Failed to set cookie for URL "${response.url}": ${e.message}`);
+                this.#log?.warning(`Failed to set cookie for URL "${response.url}": ${e.message}`);
             }
         }
     }
-    resolveRequestContext(options) {
+    async #resolveRequestContext(options) {
         const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
-        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar();
-        const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
+        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? (await this.#createDefaultCookieJar());
+        const signal = this.#createAbortSignal(options?.signal, options?.timeoutMillis);
         return {
             proxyUrl,
-            cookieJar: cookieJar,
+            cookieJar,
             signal,
             fingerprint: options?.session?.fingerprint,
+            ignoreTlsErrors: options?.ignoreTlsErrors || options?.session?.proxyInfo?.ignoreTlsErrors,
         };
     }
-    createAbortSignal(signal, timeoutMillis) {
+    async #createDefaultCookieJar() {
+        const { CookieJar: ToughCookieJar } = await import('tough-cookie');
+        return new ToughCookieJar();
+    }
+    #createAbortSignal(signal, timeoutMillis) {
         if (signal && timeoutMillis) {
             return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
         }
@@ -68,11 +72,11 @@ export class BaseHttpClient {
         }
         return timeoutMillis ? AbortSignal.timeout(timeoutMillis) : undefined;
     }
-    isRedirect(response) {
+    #isRedirect(response) {
         const status = response.status;
         return status >= 300 && status < 400 && !!response.headers.get('location');
     }
-    buildRedirectRequest(currentRequest, response, initialRequest) {
+    #buildRedirectRequest(currentRequest, response, initialRequest) {
         const location = response.headers.get('location');
         const nextUrl = new URL(location, response.url ?? currentRequest.url);
         const prevMethod = (currentRequest.method ?? 'GET').toUpperCase();
@@ -89,12 +93,20 @@ export class BaseHttpClient {
         }
         const nextHeaders = new Headers();
         currentRequest.headers.forEach((value, key) => nextHeaders.set(key, value));
+        // Like native `fetch`, do not forward credentials to a different origin
+        if (nextUrl.origin !== new URL(currentRequest.url).origin) {
+            for (const header of ['authorization', 'proxy-authorization', 'cookie']) {
+                nextHeaders.delete(header);
+            }
+        }
         return new Request(nextUrl.toString(), {
             method: nextMethod,
             headers: nextHeaders,
             body: nextBody,
             credentials: currentRequest.credentials,
             redirect: 'manual',
+            // Node-specific option required for a stream body, which the replayed body always is
+            duplex: 'half',
         });
     }
     /**
@@ -104,23 +116,24 @@ export class BaseHttpClient {
         const maxRedirects = 10;
         let currentRequest = initialRequest;
         let redirectCount = 0;
-        const { proxyUrl, cookieJar, signal, fingerprint } = this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.#resolveRequestContext(options);
         currentRequest = initialRequest.clone();
         while (true) {
-            await this.applyCookies(currentRequest, cookieJar);
-            const response = await this.fetch(currentRequest, {
+            // Like `fetch`, set the jar cookies on a copy, so that the next redirect hop does not inherit them
+            const response = await this.fetch(await this.#applyCookies(new Request(currentRequest), cookieJar), {
                 signal,
                 proxyUrl,
                 cookieJar,
                 fingerprint,
+                ignoreTlsErrors,
                 redirect: 'manual',
             });
-            await this.setCookies(response, cookieJar);
-            if (this.isRedirect(response)) {
+            await this.#setCookies(response, cookieJar);
+            if (this.#isRedirect(response)) {
                 if (redirectCount++ >= maxRedirects) {
                     throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`);
                 }
-                currentRequest = this.buildRedirectRequest(currentRequest, response, initialRequest);
+                currentRequest = this.#buildRedirectRequest(currentRequest, response, initialRequest);
                 continue;
             }
             return response;

package/fetch-http-client.d.ts

@@ -1,3 +1,4 @@
+import type { CrawleeLogger } from '@crawlee/types';
 import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
 /**
  * A HTTP client implementation using the native `fetch` API.
@@ -5,5 +6,9 @@ import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export declare class FetchHttpClient extends BaseHttpClient {
-    fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+    #private;
+    constructor(options?: {
+        logger?: CrawleeLogger;
+    });
+    protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }

package/fetch-http-client.js

@@ -5,7 +5,16 @@ import { BaseHttpClient } from './base-http-client.js';
  * This implementation does not support proxying.
  */
 export class FetchHttpClient extends BaseHttpClient {
+    #logger;
+    constructor(options) {
+        super(options);
+        this.#logger = options?.logger;
+    }
     async fetch(request, options) {
+        if (options?.ignoreTlsErrors) {
+            this.#logger?.warningOnce('FetchHttpClient cannot disable TLS certificate verification, the `ignoreTlsErrors` option is ignored. ' +
+                'Install the optional @crawlee/impit-client dependency to make it work.');
+        }
         return fetch(request, options);
     }
 }

package/index.d.ts

@@ -1,3 +1,3 @@
 export { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
-export { ResponseWithUrl, type IResponseWithUrl } from './response.js';
+export { ResponseWithUrl } from './response.js';
 export { FetchHttpClient } from './fetch-http-client.js';

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/http-client",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,7 +47,7 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-beta.99",
+        "@crawlee/types": "4.0.0-rc.1",
         "tough-cookie": "^6.0.0"
     },
     "lerna": {
@@ -57,5 +57,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/response.d.ts

@@ -1,13 +1,10 @@
-export interface IResponseWithUrl extends Response {
-    url: string;
-}
 /**
  * A Response class that includes the original request URL.
  *
  * This class extends `Response` from `fetch` API and is fully compatible with this.
  */
-export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
-    url: string;
+export declare class ResponseWithUrl extends Response {
+    readonly url: string;
     constructor(body: BodyInit | null, init: ResponseInit & {
         url?: string;
     });