@crawlee/http 4.0.0-beta.80 → 4.0.0-beta.81

package/internals/http-crawler.d.ts

@@ -1,9 +1,8 @@
 import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
-import type { Awaitable, Dictionary, ISession } from '@crawlee/types';
+import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
-import type { RequestLike, ResponseLike } from 'content-type';
 import type { JsonValue } from 'type-fest';
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -228,14 +227,14 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * @category Crawlers
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
-    protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
-    protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
-    protected saveResponseCookies: boolean;
-    protected navigationTimeoutMillis: number;
-    protected ignoreSslErrors: boolean;
-    protected suggestResponseEncoding?: string;
-    protected forceResponseEncoding?: string;
-    protected readonly supportedMimeTypes: Set<string>;
+    private preNavigationHooks;
+    private postNavigationHooks;
+    private saveResponseCookies;
+    private navigationTimeoutMillis;
+    private ignoreSslErrors;
+    private suggestResponseEncoding?;
+    private forceResponseEncoding?;
+    private readonly supportedMimeTypes;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -337,64 +336,30 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    protected _requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response>;
+    private requestFunction;
     /**
      * Encodes and parses response according to the provided content type
      */
-    protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
-        response: Response;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        body: string;
-    } | {
-        body: Buffer<ArrayBuffer>;
-        response: Response;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-    }>;
+    private parseResponse;
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    protected _getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string): {
-        url: string;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        method: import("@crawlee/types").AllowedHttpMethods;
-        proxyUrl: string | undefined;
-        timeout: number;
-        sessionToken: ISession;
-        headers: Record<string, string> | undefined;
-        https: {
-            rejectUnauthorized: boolean;
-        };
-        body: string | undefined;
-    };
-    protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
-        encoding: BufferEncoding;
-        response: Response;
-    };
+    private getRequestOptions;
+    private encodeResponse;
     /**
      * Checks and extends supported mime types
      */
-    protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
+    private extendSupportedMimeTypes;
     /**
      * Handles timeout request
      */
-    protected _handleRequestTimeout(session: ISession): void;
+    private handleRequestTimeout;
     private _abortDownloadOfBody;
     /**
      * @internal wraps public utility for mocking purposes
      */
     private _requestAsBrowser;
 }
-interface RequestFunctionOptions {
-    request: CrawleeRequest;
-    session: ISession;
-    proxyUrl?: string;
-}
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.

package/internals/http-crawler.js

@@ -129,7 +129,7 @@ export class HttpCrawler extends BasicCrawler {
         });
         this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
+            this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
@@ -188,7 +188,7 @@ export class HttpCrawler extends BasicCrawler {
         tryCancel();
         const { request, session } = crawlingContext;
         const proxyUrl = crawlingContext.proxyInfo?.url;
-        const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
@@ -215,7 +215,7 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
-        const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
+        const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -291,15 +291,15 @@ export class HttpCrawler extends BasicCrawler {
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async _requestFunction({ request, session, proxyUrl }) {
-        const opts = this._getRequestOptions(request, session, proxyUrl);
+    async requestFunction({ request, session, proxyUrl }) {
+        const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
             return await this._requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
-                this._handleRequestTimeout(session);
-                return new Response(); // this will never happen, as _handleRequestTimeout always throws
+                this.handleRequestTimeout(session);
+                return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
                 throw new SessionError(this._getMessageFromError(e));
@@ -312,10 +312,10 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async _parseResponse(request, response) {
+    async parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
-        const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
         const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
             this.stats.registerStatusCode(status);
@@ -361,7 +361,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    _getRequestOptions(request, session, proxyUrl) {
+    getRequestOptions(request, session, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
@@ -389,7 +389,7 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    _encodeResponse(request, response, encoding) {
+    encodeResponse(request, response, encoding) {
         if (this.forceResponseEncoding) {
             encoding = this.forceResponseEncoding;
         }
@@ -425,7 +425,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
+    extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
                 this.supportedMimeTypes.add(mimeType);
@@ -443,7 +443,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    _handleRequestTimeout(session) {
+    handleRequestTimeout(session) {
         session.markBad();
         throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.81",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.80",
-        "@crawlee/core": "4.0.0-beta.80",
-        "@crawlee/http-client": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/basic": "4.0.0-beta.81",
+        "@crawlee/core": "4.0.0-beta.81",
+        "@crawlee/http-client": "4.0.0-beta.81",
+        "@crawlee/types": "4.0.0-beta.81",
+        "@crawlee/utils": "4.0.0-beta.81",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
 }