@crawlee/http-client 4.0.0-beta.199 → 4.0.0-beta.201

package/base-http-client.d.ts

@@ -48,12 +48,6 @@ export declare abstract class BaseHttpClient implements BaseHttpClientInterface
      * automatic redirect following or special error handling.
      */
     protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
-    private applyCookies;
-    private setCookies;
-    private resolveRequestContext;
-    private createAbortSignal;
-    private isRedirect;
-    private buildRedirectRequest;
     /**
      * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
      */

package/base-http-client.js

@@ -8,7 +8,7 @@ export class BaseHttpClient {
     constructor(options) {
         this.#log = options?.logger;
     }
-    async applyCookies(request, cookieJar) {
+    async #applyCookies(request, cookieJar) {
         try {
             const requestCookies = request.headers.get('cookie') ?? '';
             if (!requestCookies) {
@@ -36,7 +36,7 @@ export class BaseHttpClient {
         }
         return request;
     }
-    async setCookies(response, cookieJar) {
+    async #setCookies(response, cookieJar) {
         const setCookieHeaders = response.headers.getSetCookie();
         for (const header of setCookieHeaders) {
             try {
@@ -47,10 +47,10 @@ export class BaseHttpClient {
             }
         }
     }
-    async resolveRequestContext(options) {
+    async #resolveRequestContext(options) {
         const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
         const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? (await this.#createDefaultCookieJar());
-        const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
+        const signal = this.#createAbortSignal(options?.signal, options?.timeoutMillis);
         return {
             proxyUrl,
             cookieJar,
@@ -63,7 +63,7 @@ export class BaseHttpClient {
         const { CookieJar: ToughCookieJar } = await import('tough-cookie');
         return new ToughCookieJar();
     }
-    createAbortSignal(signal, timeoutMillis) {
+    #createAbortSignal(signal, timeoutMillis) {
         if (signal && timeoutMillis) {
             return AbortSignal.any([signal, AbortSignal.timeout(timeoutMillis)]);
         }
@@ -72,11 +72,11 @@ export class BaseHttpClient {
         }
         return timeoutMillis ? AbortSignal.timeout(timeoutMillis) : undefined;
     }
-    isRedirect(response) {
+    #isRedirect(response) {
         const status = response.status;
         return status >= 300 && status < 400 && !!response.headers.get('location');
     }
-    buildRedirectRequest(currentRequest, response, initialRequest) {
+    #buildRedirectRequest(currentRequest, response, initialRequest) {
         const location = response.headers.get('location');
         const nextUrl = new URL(location, response.url ?? currentRequest.url);
         const prevMethod = (currentRequest.method ?? 'GET').toUpperCase();
@@ -116,11 +116,11 @@ export class BaseHttpClient {
         const maxRedirects = 10;
         let currentRequest = initialRequest;
         let redirectCount = 0;
-        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint, ignoreTlsErrors } = await this.#resolveRequestContext(options);
         currentRequest = initialRequest.clone();
         while (true) {
             // Like `fetch`, set the jar cookies on a copy, so that the next redirect hop does not inherit them
-            const response = await this.fetch(await this.applyCookies(new Request(currentRequest), cookieJar), {
+            const response = await this.fetch(await this.#applyCookies(new Request(currentRequest), cookieJar), {
                 signal,
                 proxyUrl,
                 cookieJar,
@@ -128,12 +128,12 @@ export class BaseHttpClient {
                 ignoreTlsErrors,
                 redirect: 'manual',
             });
-            await this.setCookies(response, cookieJar);
-            if (this.isRedirect(response)) {
+            await this.#setCookies(response, cookieJar);
+            if (this.#isRedirect(response)) {
                 if (redirectCount++ >= maxRedirects) {
                     throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`);
                 }
-                currentRequest = this.buildRedirectRequest(currentRequest, response, initialRequest);
+                currentRequest = this.#buildRedirectRequest(currentRequest, response, initialRequest);
                 continue;
             }
             return response;

package/fetch-http-client.d.ts

@@ -10,5 +10,5 @@ export declare class FetchHttpClient extends BaseHttpClient {
     constructor(options?: {
         logger?: CrawleeLogger;
     });
-    fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+    protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }

package/index.d.ts

@@ -1,3 +1,3 @@
 export { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
-export { ResponseWithUrl, type IResponseWithUrl } from './response.js';
+export { ResponseWithUrl } from './response.js';
 export { FetchHttpClient } from './fetch-http-client.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http-client",
-    "version": "4.0.0-beta.199",
+    "version": "4.0.0-beta.201",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -47,7 +47,7 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/types": "4.0.0-beta.199",
+        "@crawlee/types": "4.0.0-beta.201",
         "tough-cookie": "^6.0.0"
     },
     "lerna": {
@@ -57,5 +57,5 @@
             }
         }
     },
-    "gitHead": "15064bef4a733bffd4db490cc683aee488787c2e"
+    "gitHead": "5b29925251e92eb69252045f4980153844f58462"
 }

package/response.d.ts

@@ -1,13 +1,10 @@
-export interface IResponseWithUrl extends Response {
-    url: string;
-}
 /**
  * A Response class that includes the original request URL.
  *
  * This class extends `Response` from `fetch` API and is fully compatible with this.
  */
-export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
-    url: string;
+export declare class ResponseWithUrl extends Response {
+    readonly url: string;
     constructor(body: BodyInit | null, init: ResponseInit & {
         url?: string;
     });