@crawlee/http-client 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/base-http-client.d.ts

@@ -48,12 +48,6 @@ export declare abstract class BaseHttpClient implements BaseHttpClientInterface
      * automatic redirect following or special error handling.
      */
     protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
-    private applyCookies;
-    private setCookies;
-    private resolveRequestContext;
-    private createAbortSignal;
-    private isRedirect;
-    private buildRedirectRequest;
     /**
      * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
      */

package/base-http-client.js

@@ -8,7 +8,7 @@ export class BaseHttpClient {
     constructor(options) {
         this.#log = options?.logger;
     }
-    async applyCookies(request, cookieJar) {
+    async #applyCookies(request, cookieJar) {
         try {
             const requestCookies = request.headers.get('cookie') ?? '';
             if (!requestCookies) {
@@ -36,7 +36,7 @@ export class BaseHttpClient {
         }
         return request;
     }
-    async setCookies(response, cookieJar) {
+    async #setCookies(response, cookieJar) {
         const setCookieHeaders = response.headers.getSetCookie();
         for (const header of setCookieHeaders) {
             try {
@@ -47,10 +47,10 @@ export class BaseHttpClient {
             }
         }
     }
-    async resolveRequestContext(options) {
+    async #resolveRequestContext(options) {
         const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
         const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? (await this.#createDefaultCookieJar());
-        const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
+        const signal = this.#createAbortSignal(options?.signal, options?.timeoutMillis);
         return {
             proxyUrl,
             cookieJar,
@@ -63,7 +63,7 @@ export class BaseHttpClient {
         const { CookieJar: ToughCookieJar } = await import('tough-cookie');
         return new ToughCookieJar();
     }
-    createAbortSignal(signal, timeoutMillis) {
+    #createAbortSignal(signal, timeoutMillis) {
         if (signal && timeoutMillis) {
             return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
         }
@@ -72,11 +72,11 @@ export class BaseHttpClient {
         }
         return timeoutMillis ? AbortSignal.timeout(timeoutMillis) : undefined;
     }
-    isRedirect(response) {
+    #isRedirect(response) {
         const status = response.status;
         return status >= 300 && status < 400 && !!response.headers.get('location');
     }
-    buildRedirectRequest(currentRequest, response, initialRequest) {
+    #buildRedirectRequest(currentRequest, response, initialRequest) {
         const location = response.headers.get('location');
         const nextUrl = new URL(location, response.url ?? currentRequest.url);
         const prevMethod = (currentRequest.method ?? 'GET').toUpperCase();
@@ -93,12 +93,20 @@ export class BaseHttpClient {
         }
         const nextHeaders = new Headers();
         currentRequest.headers.forEach((value, key) => nextHeaders.set(key, value));
+        // Like native `fetch`, do not forward credentials to a different origin
+        if (nextUrl.origin !== new URL(currentRequest.url).origin) {
+            for (const header of ['authorization', 'proxy-authorization', 'cookie']) {
+                nextHeaders.delete(header);
+            }
+        }
         return new Request(nextUrl.toString(), {
             method: nextMethod,
             headers: nextHeaders,
             body: nextBody,
             credentials: currentRequest.credentials,
             redirect: 'manual',
+            // Node-specific option required for a stream body, which the replayed body always is
+            duplex: 'half',
         });
     }
     /**
@@ -108,11 +116,11 @@ export class BaseHttpClient {
         const maxRedirects = 10;
         let currentRequest = initialRequest;
         let redirectCount = 0;
-        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.#resolveRequestContext(options);
         currentRequest = initialRequest.clone();
         while (true) {
-            await this.applyCookies(currentRequest, cookieJar);
-            const response = await this.fetch(currentRequest, {
+            // Like `fetch`, set the jar cookies on a copy, so that the next redirect hop does not inherit them
+            const response = await this.fetch(await this.#applyCookies(new Request(currentRequest), cookieJar), {
                 signal,
                 proxyUrl,
                 cookieJar,
@@ -120,12 +128,12 @@ export class BaseHttpClient {
                 ignoreTlsErrors,
                 redirect: 'manual',
             });
-            await this.setCookies(response, cookieJar);
-            if (this.isRedirect(response)) {
+            await this.#setCookies(response, cookieJar);
+            if (this.#isRedirect(response)) {
                 if (redirectCount++ >= maxRedirects) {
                     throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`);
                 }
-                currentRequest = this.buildRedirectRequest(currentRequest, response, initialRequest);
+                currentRequest = this.#buildRedirectRequest(currentRequest, response, initialRequest);
                 continue;
             }
             return response;

package/fetch-http-client.d.ts

@@ -10,5 +10,5 @@ export declare class FetchHttpClient extends BaseHttpClient {
     constructor(options?: {
         logger?: CrawleeLogger;
     });
-    fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+    protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }

package/index.d.ts

@@ -1,3 +1,3 @@
 export { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
-export { ResponseWithUrl, type IResponseWithUrl } from './response.js';
+export { ResponseWithUrl } from './response.js';
 export { FetchHttpClient } from './fetch-http-client.js';

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/http-client",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,7 +47,7 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.1",
         "tough-cookie": "^6.0.0"
     },
     "lerna": {
@@ -57,5 +57,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/response.d.ts

@@ -1,13 +1,10 @@
-export interface IResponseWithUrl extends Response {
-    url: string;
-}
 /**
  * A Response class that includes the original request URL.
  *
  * This class extends `Response` from `fetch` API and is fully compatible with this.
  */
-export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
-    url: string;
+export declare class ResponseWithUrl extends Response {
+    readonly url: string;
     constructor(body: BodyInit | null, init: ResponseInit & {
         url?: string;
     });