@crawlee/http 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/http-crawler.js

@@ -1,12 +1,11 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
-import { getCookiesFromResponse } from '@crawlee/core';
+import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
-import contentTypeParser from 'content-type';
+import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import iconv from 'iconv-lite';
-import ow from 'ow';
+import { z } from 'zod';
 import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
 import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
@@ -16,7 +15,8 @@ const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
 /**
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
- * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
  *
  * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
  * wholesale, tuning included, so spread these options in if you want to keep it:
@@ -109,67 +109,67 @@ export class HttpCrawler extends BasicCrawler {
     // extension-aware for consumer DX, but internally the pipeline composes hooks against the
     // concrete crawling context, which does not statically carry `ContextExtension`. The members
     // added by `extendContext` are present at runtime regardless.
-    preNavigationHooks;
-    postNavigationHooks;
-    saveResponseCookies;
-    navigationTimeoutMillis;
-    ignoreSslErrors;
-    suggestResponseEncoding;
-    forceResponseEncoding;
-    supportedMimeTypes;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    #navigationTimeoutMillis;
+    #ignoreTlsErrors;
+    #suggestResponseEncoding;
+    #forceResponseEncoding;
+    #supportedMimeTypes;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
-        navigationTimeoutSecs: ow.optional.number,
-        ignoreSslErrors: ow.optional.boolean,
-        additionalMimeTypes: ow.optional.array.ofType(ow.string),
-        suggestResponseEncoding: ow.optional.string,
-        forceResponseEncoding: ow.optional.string,
-        saveResponseCookies: ow.optional.boolean,
-        preNavigationHooks: ow.optional.array,
-        postNavigationHooks: ow.optional.array,
+        navigationTimeoutSecs: schemas.anyNumber.default(30),
+        ignoreTlsErrors: z.boolean().default(true),
+        additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
+        suggestResponseEncoding: z.string().optional(),
+        forceResponseEncoding: z.string().optional(),
+        saveResponseCookies: z.boolean().default(true),
+        preNavigationHooks: schemas.anyArray.default(() => []),
+        postNavigationHooks: schemas.anyArray.default(() => []),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
+        const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks, 
         // BasicCrawler
-        contextPipelineBuilder, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
         super({
             ...basicCrawlerOptions,
             contextPipelineBuilder: contextPipelineBuilder ??
                 (() => this.buildContextPipeline()),
         });
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this.extendSupportedMimeTypes(additionalMimeTypes);
+            this.#extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#ignoreTlsErrors = ignoreTlsErrors;
+        this.#suggestResponseEncoding = suggestResponseEncoding;
+        this.#forceResponseEncoding = forceResponseEncoding;
         // Cast away the extension-aware option types to the base internal storage types (see the field
         // declarations above). This is sound - the hooks only ever receive the base context plus the
         // members `extendContext` added at runtime.
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = [
+            ({ request, response }) => this.#abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        this.saveResponseCookies = saveResponseCookies;
+        this.#saveResponseCookies = saveResponseCookies;
     }
+    /** @internal */
     getNavigationTimeoutMillis() {
-        return this.navigationTimeoutMillis;
+        return this.#navigationTimeoutMillis;
     }
-    /**
-     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
-     * concurrency shortcuts on top. Not called for a supplied
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
-     * keep the tuning.
-     */
+    /** @internal */
     createDefaultConcurrencySystem(options) {
         return super.createDefaultConcurrencySystem({
             ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
@@ -179,45 +179,44 @@ export class HttpCrawler extends BasicCrawler {
     buildContextPipeline() {
         // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
         // the response-derived members, so the guarded action is bypassed and the context left untouched.
-        const skipGuard = (action) => ({
-            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
-        });
+        const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
         // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
         // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
         // window the navigation uses instead of each step being timed on its own.
-        const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;
+        const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
         const windowGuard = (step) => skipGuard(async (ctx) => {
-            const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+            const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
             if (remaining <= 0) {
                 throw new TimeoutError(navigationTimedOut);
             }
             return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
         });
-        let pipeline = ContextPipeline.create().compose({
-            action: this.prepareHttpRequest.bind(this),
-        });
-        for (const hook of this.preNavigationHooks) {
+        let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
+        for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
         let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
-        for (const hook of this.postNavigationHooks) {
+        for (const hook of this.#postNavigationHooks) {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
-            .compose({ action: this.processHttpResponse.bind(this) })
-            .compose({ action: this.handleBlockedRequestByContent.bind(this) });
+            .compose(this.#processHttpResponse.bind(this))
+            .compose(this.#handleBlockedRequestByContent.bind(this));
     }
-    async prepareHttpRequest(crawlingContext) {
+    async #prepareHttpRequest(crawlingContext) {
         const { request } = crawlingContext;
         if (request.skipNavigation) {
             return {
                 request: new Proxy(request, {
-                    get(target, propertyName, receiver) {
+                    get(target, propertyName) {
                         if (propertyName === 'loadedUrl') {
                             throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
-                        return Reflect.get(target, propertyName, receiver);
+                        // `target` as receiver and bound methods, or `#` members throw on the proxy
+                        const value = Reflect.get(target, propertyName, target);
+                        return typeof value === 'function' ? value.bind(target) : value;
                     },
+                    set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
                 }),
                 get response() {
                     throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -227,6 +226,7 @@ export class HttpCrawler extends BasicCrawler {
         request.state = RequestState.BEFORE_NAV;
         return {};
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async makeHttpRequest(crawlingContext) {
         tryCancel();
         const { request, session } = crawlingContext;
@@ -234,13 +234,13 @@ export class HttpCrawler extends BasicCrawler {
         // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
         // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
         // client abort.
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)), `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
         return { request: request, response: httpResponse };
     }
-    async processHttpResponse(crawlingContext) {
+    async #processHttpResponse(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 get contentType() {
@@ -261,40 +261,55 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
+        // Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
+        // error is still a rate limit the domain should back off from.
+        if (crawlingContext.response.status === 429) {
+            const retryAfter = crawlingContext.response.headers.get('retry-after');
+            if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                // This is the one path that never reads the body, so cancel it to release the connection
+                // rather than leaving it to the garbage collector.
+                await crawlingContext.response.body?.cancel().catch(() => { });
+                throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+            }
+        }
         // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
         // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
         // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
-        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
         if (remaining <= 0) {
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
+        const loadBody = async () => {
+            const { load } = await import('cheerio/slim');
+            return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
+        };
         const waitForSelector = async (selector, _timeoutMs) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body.toString());
+            const $ = await loadBody();
             if ($(selector).get().length === 0) {
                 throw new Error(`Selector '${selector}' not found.`);
             }
         };
         const parseWithCheerio = async (selector, timeoutMs) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body.toString());
+            const $ = await loadBody();
             if (selector) {
                 await crawlingContext.waitForSelector(selector, timeoutMs);
             }
             return $;
         };
-        this._throwOnBlockedRequest(response.status);
-        if (this.saveResponseCookies) {
+        this.throwOnBlockedRequest(response.status);
+        if (this.#saveResponseCookies) {
             try {
                 for (const cookie of getCookiesFromResponse(response)) {
                     if (!cookie)
                         continue;
                     try {
-                        crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });
+                        await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+                            ignoreError: false,
+                        });
                     }
                     catch (e) {
                         this.log.debug(`Could not set cookie: ${e.message}`);
@@ -318,15 +333,15 @@ export class HttpCrawler extends BasicCrawler {
             body: parsed.body,
         };
     }
-    async handleBlockedRequestByContent(crawlingContext) {
+    async #handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
+            const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
         return {};
     }
-    async isRequestBlocked(crawlingContext) {
+    async #isRequestBlocked(crawlingContext) {
         if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
             const $ = await crawlingContext.parseWithCheerio();
             const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
@@ -344,18 +359,18 @@ export class HttpCrawler extends BasicCrawler {
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async requestFunction({ request, session, proxyUrl }) {
-        const opts = this.getRequestOptions(request, session, proxyUrl);
+    async #requestFunction({ request, session, proxyUrl }) {
+        const opts = this.#getRequestOptions(request, proxyUrl);
         try {
-            return await this._requestAsBrowser(opts, session);
+            return await this.#requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
-                this.handleRequestTimeout(session);
+                this.#handleRequestTimeout(session);
                 return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
-                throw new SessionError(this._getMessageFromError(e));
+                throw new SessionError(this.getMessageFromError(e));
             }
             else {
                 throw e;
@@ -365,16 +380,15 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async parseResponse(request, response) {
+    async #parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
-        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
-        const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
-            this.stats.registerStatusCode(status);
+            this.statistics.registerStatusCode(status);
         }
         if (this.isErrorStatusCode(status)) {
-            const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
+            const { response: decoded } = this.#encodeResponse(request, response, charset);
+            const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -390,64 +404,54 @@ export class HttpCrawler extends BasicCrawler {
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            if (!charset && !this.forceResponseEncoding) {
-                const rawBytes = Buffer.from(await response.arrayBuffer());
-                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
-                const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
-                const body = iconv.encodingExists(charsetToUse)
-                    ? iconv.decode(rawBytes, charsetToUse)
-                    : rawBytes.toString('utf8');
-                return { response, contentType: { type, encoding: 'utf-8' }, body };
-            }
-            return { response, contentType, body: await reencodedResponse.text() };
+        if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
+            // The charset comes from the document itself, so the raw bytes are what we need -
+            // decoding them through `encodeResponse` first would consume the body for nothing.
+            const rawBytes = Buffer.from(await response.arrayBuffer());
+            const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+            const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
+            const body = iconv.encodingExists(charsetToUse)
+                ? iconv.decode(rawBytes, charsetToUse)
+                : rawBytes.toString('utf8');
+            return { response, contentType: { type, encoding: 'utf-8' }, body };
         }
-        else {
-            const body = Buffer.from(await reencodedResponse.bytes());
-            return {
-                body,
-                response,
-                contentType,
-            };
+        const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
+        const contentType = { type, encoding };
+        if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            return { response, contentType, body: await reencodedResponse.text() };
         }
+        return {
+            body: Buffer.from(await reencodedResponse.bytes()),
+            response,
+            contentType,
+        };
     }
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    getRequestOptions(request, session, proxyUrl) {
+    #getRequestOptions(request, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: this.navigationTimeoutMillis,
-            sessionToken: session,
+            timeout: this.#navigationTimeoutMillis,
             headers: request.headers,
-            https: {
-                rejectUnauthorized: !this.ignoreSslErrors,
-            },
             body: undefined,
         };
         if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
-            requestOptions.headers.Cookie = this._getCookieHeaderFromRequest(request);
+            requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
             delete requestOptions.headers.cookie;
         }
-        // Disable SSL verification for MITM proxies
-        if (session.proxyInfo?.ignoreTlsErrors) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
-        }
         if (/PATCH|POST|PUT/.test(request.method))
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
+    #encodeResponse(request, response, encoding) {
+        if (this.#forceResponseEncoding) {
+            encoding = this.#forceResponseEncoding;
         }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
+        else if (!encoding && this.#suggestResponseEncoding) {
+            encoding = this.#suggestResponseEncoding;
         }
         // Fall back to utf-8 if we still don't have encoding.
         const utf8 = 'utf8';
@@ -478,42 +482,40 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    extendSupportedMimeTypes(additionalMimeTypes) {
+    #extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
-                this.supportedMimeTypes.add(mimeType);
+                this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
-            try {
-                const parsedType = contentTypeParser.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
-            }
-            catch (err) {
+            const parsedType = parseContentType(mimeType);
+            if (!isTypeValid(parsedType.type)) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
             }
+            this.#supportedMimeTypes.add(parsedType.type);
         }
     }
     /**
      * Handles timeout request
      */
-    handleRequestTimeout(session) {
+    #handleRequestTimeout(session) {
         session.markBad();
-        throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    _abortDownloadOfBody(request, response) {
+    #abortDownloadOfBody(request, response) {
         const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
         const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
-        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
+        if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
-                `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+                `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
         }
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    _requestAsBrowser = async (options, session) => {
+    #requestAsBrowser = async (options, session) => {
         const opts = processHttpRequestOptions({
             ...options,
             responseType: 'text',
@@ -521,7 +523,7 @@ export class HttpCrawler extends BasicCrawler {
         // When saveResponseCookies is false, the response cookies must not mutate the
         // session jar. Reads still go through the session (so session.setCookie() in pre-nav
         // hooks keeps working) but a per-request clone is passed in so writes are discarded.
-        const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
         // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
         // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
         // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
@@ -537,8 +539,10 @@ export class HttpCrawler extends BasicCrawler {
         }), {
             session,
             cookieJar,
+            proxyUrl: opts.proxyUrl,
             signal: cancelSignal,
             timeoutMillis: cancelSignal ? undefined : opts.timeout,
+            ignoreTlsErrors: this.#ignoreTlsErrors,
         });
         return response;
     };

package/internals/utils.js

@@ -1,9 +1,10 @@
 import { extname } from 'node:path';
 import { Readable } from 'node:stream';
-import { applySearchParams } from '@crawlee/utils';
-import contentTypeParser from 'content-type';
+import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import mime from 'mime-types';
-import ow, { ObjectPredicate } from 'ow';
+import { z } from 'zod';
+const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
 /**
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
  */
@@ -54,25 +55,20 @@ export function extractCharsetFromHtmlBytes(bytes) {
  * @param response HTTP response object
  */
 export function parseContentTypeFromResponse(response) {
-    ow(response, ow.object.partialShape({
-        url: ow.string.url,
-        headers: new ObjectPredicate(),
-    }));
+    parseArgument(response, responseWithUrlSchema);
     const { url, headers } = response;
     let parsedContentType;
     if (headers.get('content-type')) {
-        try {
-            parsedContentType = contentTypeParser.parse(headers.get('content-type'));
-        }
-        catch {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
+        const parsed = parseContentType(headers.get('content-type'));
+        // Can not parse content type from Content-Type header. Try to parse it from file extension.
+        if (isTypeValid(parsed.type))
+            parsedContentType = parsed;
     }
     // Parse content type from file extension as fallback
     if (!parsedContentType) {
         const parsedUrl = new URL(url);
         const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
+        parsedContentType = parseContentType(contentTypeFromExtname);
     }
     return {
         type: parsedContentType.type,

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,21 +47,18 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.4",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.99",
-        "@crawlee/core": "4.0.0-beta.99",
-        "@crawlee/http-client": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
-        "@types/content-type": "^1.1.8",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/basic": "4.0.0-rc.1",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "cheerio": "^1.0.0",
-        "content-type": "^1.0.5",
+        "content-type": "^3.1.1",
         "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
-        "ow": "^2.0.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "type-fest": "^4.41.0",
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -70,5 +67,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }