@crawlee/http 4.0.0-beta.210 → 4.0.0-beta.212

package/internals/file-download.d.ts

@@ -49,7 +49,6 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
 export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
     #private;
     constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
-    private initiateDownload;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/file-download.js

@@ -44,9 +44,9 @@ export class FileDownload extends BasicCrawler {
         });
     }
     #buildContextPipeline() {
-        return ContextPipeline.create().compose(this.initiateDownload.bind(this));
+        return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
     }
-    async initiateDownload(context, onCleanup) {
+    async #initiateDownload(context, onCleanup) {
         const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
             session: context.session,
         });

package/internals/http-crawler.d.ts

@@ -381,38 +381,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
     /** @internal */
     protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
-    private prepareHttpRequest;
     private makeHttpRequest;
-    private processHttpResponse;
-    private handleBlockedRequestByContent;
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     */
-    private requestFunction;
-    /**
-     * Encodes and parses response according to the provided content type
-     */
-    private parseResponse;
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     */
-    private getRequestOptions;
-    private encodeResponse;
-    /**
-     * Checks and extends supported mime types
-     */
-    private extendSupportedMimeTypes;
-    /**
-     * Handles timeout request
-     */
-    private handleRequestTimeout;
-    private abortDownloadOfBody;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private requestAsBrowser;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/http-crawler.js

@@ -147,7 +147,7 @@ export class HttpCrawler extends BasicCrawler {
         });
         this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this.extendSupportedMimeTypes(additionalMimeTypes);
+            this.#extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
@@ -160,7 +160,7 @@ export class HttpCrawler extends BasicCrawler {
         // members `extendContext` added at runtime.
         this.#preNavigationHooks = preNavigationHooks;
         this.#postNavigationHooks = [
-            ({ request, response }) => this.abortDownloadOfBody(request, response),
+            ({ request, response }) => this.#abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
         this.#saveResponseCookies = saveResponseCookies;
@@ -191,7 +191,7 @@ export class HttpCrawler extends BasicCrawler {
             }
             return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
         });
-        let pipeline = ContextPipeline.create().compose(this.prepareHttpRequest.bind(this));
+        let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
         for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
@@ -200,20 +200,23 @@ export class HttpCrawler extends BasicCrawler {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
-            .compose(this.processHttpResponse.bind(this))
-            .compose(this.handleBlockedRequestByContent.bind(this));
+            .compose(this.#processHttpResponse.bind(this))
+            .compose(this.#handleBlockedRequestByContent.bind(this));
     }
-    async prepareHttpRequest(crawlingContext) {
+    async #prepareHttpRequest(crawlingContext) {
         const { request } = crawlingContext;
         if (request.skipNavigation) {
             return {
                 request: new Proxy(request, {
-                    get(target, propertyName, receiver) {
+                    get(target, propertyName) {
                         if (propertyName === 'loadedUrl') {
                             throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
-                        return Reflect.get(target, propertyName, receiver);
+                        // `target` as receiver and bound methods, or `#` members throw on the proxy
+                        const value = Reflect.get(target, propertyName, target);
+                        return typeof value === 'function' ? value.bind(target) : value;
                     },
+                    set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
                 }),
                 get response() {
                     throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -231,13 +234,13 @@ export class HttpCrawler extends BasicCrawler {
         // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
         // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
         // client abort.
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
         return { request: request, response: httpResponse };
     }
-    async processHttpResponse(crawlingContext) {
+    async #processHttpResponse(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 get contentType() {
@@ -276,7 +279,7 @@ export class HttpCrawler extends BasicCrawler {
         if (remaining <= 0) {
             throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -330,7 +333,7 @@ export class HttpCrawler extends BasicCrawler {
             body: parsed.body,
         };
     }
-    async handleBlockedRequestByContent(crawlingContext) {
+    async #handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
             const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
@@ -356,14 +359,14 @@ export class HttpCrawler extends BasicCrawler {
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async requestFunction({ request, session, proxyUrl }) {
-        const opts = this.getRequestOptions(request, proxyUrl);
+    async #requestFunction({ request, session, proxyUrl }) {
+        const opts = this.#getRequestOptions(request, proxyUrl);
         try {
-            return await this.requestAsBrowser(opts, session);
+            return await this.#requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
-                this.handleRequestTimeout(session);
+                this.#handleRequestTimeout(session);
                 return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
@@ -377,14 +380,14 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async parseResponse(request, response) {
+    async #parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
         if (status >= 400 && status <= 599) {
             this.statistics.registerStatusCode(status);
         }
         if (this.isErrorStatusCode(status)) {
-            const { response: decoded } = this.encodeResponse(request, response, charset);
+            const { response: decoded } = this.#encodeResponse(request, response, charset);
             const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
@@ -412,7 +415,7 @@ export class HttpCrawler extends BasicCrawler {
                 : rawBytes.toString('utf8');
             return { response, contentType: { type, encoding: 'utf-8' }, body };
         }
-        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
+        const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
         const contentType = { type, encoding };
         if (HTML_AND_XML_MIME_TYPES.includes(type)) {
             return { response, contentType, body: await reencodedResponse.text() };
@@ -426,7 +429,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    getRequestOptions(request, proxyUrl) {
+    #getRequestOptions(request, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
@@ -443,7 +446,7 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    encodeResponse(request, response, encoding) {
+    #encodeResponse(request, response, encoding) {
         if (this.#forceResponseEncoding) {
             encoding = this.#forceResponseEncoding;
         }
@@ -479,7 +482,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    extendSupportedMimeTypes(additionalMimeTypes) {
+    #extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
                 this.#supportedMimeTypes.add(mimeType);
@@ -497,11 +500,11 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    handleRequestTimeout(session) {
+    #handleRequestTimeout(session) {
         session.markBad();
         throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    abortDownloadOfBody(request, response) {
+    #abortDownloadOfBody(request, response) {
         const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
         const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
@@ -514,7 +517,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    requestAsBrowser = async (options, session) => {
+    #requestAsBrowser = async (options, session) => {
         const opts = processHttpRequestOptions({
             ...options,
             responseType: 'text',

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.210",
+    "version": "4.0.0-beta.212",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/basic": "4.0.0-beta.210",
-        "@crawlee/core": "4.0.0-beta.210",
-        "@crawlee/http-client": "4.0.0-beta.210",
-        "@crawlee/types": "4.0.0-beta.210",
-        "@crawlee/utils": "4.0.0-beta.210",
+        "@crawlee/basic": "4.0.0-beta.212",
+        "@crawlee/core": "4.0.0-beta.212",
+        "@crawlee/http-client": "4.0.0-beta.212",
+        "@crawlee/types": "4.0.0-beta.212",
+        "@crawlee/utils": "4.0.0-beta.212",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "037dff5d651d88e96749cc79d17bc6d1e154f4e1"
+    "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
 }