@crawlee/http 4.0.0-beta.99 → 4.0.0-rc.0

package/internals/http-crawler.d.ts

@@ -2,8 +2,9 @@ import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions,
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import type { JsonValue } from 'type-fest';
+import { z } from 'zod';
 /**
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
  * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
@@ -19,7 +20,7 @@ export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySyste
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
-export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {
+export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * Timeout for the whole navigation phase, given in seconds. A single window shared by the
      * `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
@@ -29,9 +30,14 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      */
     navigationTimeoutSecs?: number;
     /**
-     * If set to true, SSL certificate errors will be ignored.
+     * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
+     * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every navigation request, so custom
+     * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
+     * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
+     *
+     * @default true
      */
-    ignoreSslErrors?: boolean;
+    ignoreTlsErrors?: boolean;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
      * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
@@ -53,7 +59,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * ]
      * ```
      */
-    preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];
+    preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
@@ -241,107 +247,122 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * ```
  * @category Crawlers
  */
-export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
-    private preNavigationHooks;
-    private postNavigationHooks;
-    private saveResponseCookies;
-    private navigationTimeoutMillis;
-    private ignoreSslErrors;
-    private suggestResponseEncoding?;
-    private forceResponseEncoding?;
-    private readonly supportedMimeTypes;
+export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalMimeTypes: import("ow").ArrayPredicate<string>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        blockedStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
+        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
+        forceResponseEncoding: z.ZodOptional<z.ZodString>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
     };
+    protected static optionsSchema: z.ZodObject<{
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
+        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
+        forceResponseEncoding: z.ZodOptional<z.ZodString>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
     protected getNavigationTimeoutMillis(): number;
     /**
      * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
@@ -379,11 +400,11 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * Handles timeout request
      */
     private handleRequestTimeout;
-    private _abortDownloadOfBody;
+    private abortDownloadOfBody;
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _requestAsBrowser;
+    private requestAsBrowser;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/http-crawler.js

@@ -1,12 +1,12 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
 import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
-import { getCookiesFromResponse } from '@crawlee/core';
+import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
+import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
-import ow from 'ow';
+import { z } from 'zod';
 import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
 import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
@@ -109,60 +109,60 @@ export class HttpCrawler extends BasicCrawler {
     // extension-aware for consumer DX, but internally the pipeline composes hooks against the
     // concrete crawling context, which does not statically carry `ContextExtension`. The members
     // added by `extendContext` are present at runtime regardless.
-    preNavigationHooks;
-    postNavigationHooks;
-    saveResponseCookies;
-    navigationTimeoutMillis;
-    ignoreSslErrors;
-    suggestResponseEncoding;
-    forceResponseEncoding;
-    supportedMimeTypes;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    #navigationTimeoutMillis;
+    #ignoreTlsErrors;
+    #suggestResponseEncoding;
+    #forceResponseEncoding;
+    #supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
-        navigationTimeoutSecs: ow.optional.number,
-        ignoreSslErrors: ow.optional.boolean,
-        additionalMimeTypes: ow.optional.array.ofType(ow.string),
-        suggestResponseEncoding: ow.optional.string,
-        forceResponseEncoding: ow.optional.string,
-        saveResponseCookies: ow.optional.boolean,
-        preNavigationHooks: ow.optional.array,
-        postNavigationHooks: ow.optional.array,
+        navigationTimeoutSecs: schemas.anyNumber.default(30),
+        ignoreTlsErrors: z.boolean().default(true),
+        additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
+        suggestResponseEncoding: z.string().optional(),
+        forceResponseEncoding: z.string().optional(),
+        saveResponseCookies: z.boolean().default(true),
+        preNavigationHooks: schemas.anyArray.default(() => []),
+        postNavigationHooks: schemas.anyArray.default(() => []),
     };
+    static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
+        const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks, 
         // BasicCrawler
-        contextPipelineBuilder, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
         super({
             ...basicCrawlerOptions,
             contextPipelineBuilder: contextPipelineBuilder ??
                 (() => this.buildContextPipeline()),
         });
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
             this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#ignoreTlsErrors = ignoreTlsErrors;
+        this.#suggestResponseEncoding = suggestResponseEncoding;
+        this.#forceResponseEncoding = forceResponseEncoding;
         // Cast away the extension-aware option types to the base internal storage types (see the field
         // declarations above). This is sound - the hooks only ever receive the base context plus the
         // members `extendContext` added at runtime.
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = [
+            ({ request, response }) => this.abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        this.saveResponseCookies = saveResponseCookies;
+        this.#saveResponseCookies = saveResponseCookies;
     }
     getNavigationTimeoutMillis() {
-        return this.navigationTimeoutMillis;
+        return this.#navigationTimeoutMillis;
     }
     /**
      * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
@@ -185,9 +185,9 @@ export class HttpCrawler extends BasicCrawler {
         // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
         // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
         // window the navigation uses instead of each step being timed on its own.
-        const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;
+        const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
         const windowGuard = (step) => skipGuard(async (ctx) => {
-            const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+            const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
             if (remaining <= 0) {
                 throw new TimeoutError(navigationTimedOut);
             }
@@ -196,11 +196,11 @@ export class HttpCrawler extends BasicCrawler {
         let pipeline = ContextPipeline.create().compose({
             action: this.prepareHttpRequest.bind(this),
         });
-        for (const hook of this.preNavigationHooks) {
+        for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
         let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
-        for (const hook of this.postNavigationHooks) {
+        for (const hook of this.#postNavigationHooks) {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
@@ -234,7 +234,7 @@ export class HttpCrawler extends BasicCrawler {
         // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
         // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
         // client abort.
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)), `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
@@ -261,14 +261,25 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
+        // Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
+        // error is still a rate limit the domain should back off from.
+        if (crawlingContext.response.status === 429) {
+            const retryAfter = crawlingContext.response.headers.get('retry-after');
+            if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                // This is the one path that never reads the body, so cancel it to release the connection
+                // rather than leaving it to the garbage collector.
+                await crawlingContext.response.body?.cancel().catch(() => { });
+                throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+            }
+        }
         // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
         // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
         // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
-        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
         if (remaining <= 0) {
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -287,14 +298,16 @@ export class HttpCrawler extends BasicCrawler {
             }
             return $;
         };
-        this._throwOnBlockedRequest(response.status);
-        if (this.saveResponseCookies) {
+        this.throwOnBlockedRequest(response.status);
+        if (this.#saveResponseCookies) {
             try {
                 for (const cookie of getCookiesFromResponse(response)) {
                     if (!cookie)
                         continue;
                     try {
-                        crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });
+                        await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+                            ignoreError: false,
+                        });
                     }
                     catch (e) {
                         this.log.debug(`Could not set cookie: ${e.message}`);
@@ -347,7 +360,7 @@ export class HttpCrawler extends BasicCrawler {
     async requestFunction({ request, session, proxyUrl }) {
         const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
-            return await this._requestAsBrowser(opts, session);
+            return await this.requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
@@ -355,7 +368,7 @@ export class HttpCrawler extends BasicCrawler {
                 return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
-                throw new SessionError(this._getMessageFromError(e));
+                throw new SessionError(this.getMessageFromError(e));
             }
             else {
                 throw e;
@@ -371,7 +384,7 @@ export class HttpCrawler extends BasicCrawler {
         const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
         const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
-            this.stats.registerStatusCode(status);
+            this.statistics.registerStatusCode(status);
         }
         if (this.isErrorStatusCode(status)) {
             const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
@@ -391,10 +404,10 @@ export class HttpCrawler extends BasicCrawler {
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
         else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            if (!charset && !this.forceResponseEncoding) {
+            if (!charset && !this.#forceResponseEncoding) {
                 const rawBytes = Buffer.from(await response.arrayBuffer());
                 const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
-                const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+                const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
                 const body = iconv.encodingExists(charsetToUse)
                     ? iconv.decode(rawBytes, charsetToUse)
                     : rawBytes.toString('utf8');
@@ -419,35 +432,25 @@ export class HttpCrawler extends BasicCrawler {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: this.navigationTimeoutMillis,
+            timeout: this.#navigationTimeoutMillis,
             sessionToken: session,
             headers: request.headers,
-            https: {
-                rejectUnauthorized: !this.ignoreSslErrors,
-            },
             body: undefined,
         };
         if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
-            requestOptions.headers.Cookie = this._getCookieHeaderFromRequest(request);
+            requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
             delete requestOptions.headers.cookie;
         }
-        // Disable SSL verification for MITM proxies
-        if (session.proxyInfo?.ignoreTlsErrors) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
-        }
         if (/PATCH|POST|PUT/.test(request.method))
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
     encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
+        if (this.#forceResponseEncoding) {
+            encoding = this.#forceResponseEncoding;
         }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
+        else if (!encoding && this.#suggestResponseEncoding) {
+            encoding = this.#suggestResponseEncoding;
         }
         // Fall back to utf-8 if we still don't have encoding.
         const utf8 = 'utf8';
@@ -481,12 +484,12 @@ export class HttpCrawler extends BasicCrawler {
     extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
-                this.supportedMimeTypes.add(mimeType);
+                this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
             try {
                 const parsedType = contentTypeParser.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
+                this.#supportedMimeTypes.add(parsedType.type);
             }
             catch (err) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
@@ -498,22 +501,22 @@ export class HttpCrawler extends BasicCrawler {
      */
     handleRequestTimeout(session) {
         session.markBad();
-        throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    _abortDownloadOfBody(request, response) {
+    abortDownloadOfBody(request, response) {
         const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
         const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
-        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
+        if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
-                `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+                `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
         }
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    _requestAsBrowser = async (options, session) => {
+    requestAsBrowser = async (options, session) => {
         const opts = processHttpRequestOptions({
             ...options,
             responseType: 'text',
@@ -521,7 +524,7 @@ export class HttpCrawler extends BasicCrawler {
         // When saveResponseCookies is false, the response cookies must not mutate the
         // session jar. Reads still go through the session (so session.setCookie() in pre-nav
         // hooks keeps working) but a per-request clone is passed in so writes are discarded.
-        const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
         // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
         // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
         // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
@@ -539,6 +542,7 @@ export class HttpCrawler extends BasicCrawler {
             cookieJar,
             signal: cancelSignal,
             timeoutMillis: cancelSignal ? undefined : opts.timeout,
+            ignoreTlsErrors: this.#ignoreTlsErrors,
         });
         return response;
     };

package/internals/utils.js

@@ -1,9 +1,10 @@
 import { extname } from 'node:path';
 import { Readable } from 'node:stream';
-import { applySearchParams } from '@crawlee/utils';
+import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import mime from 'mime-types';
-import ow, { ObjectPredicate } from 'ow';
+import { z } from 'zod';
+const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
 /**
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
  */
@@ -54,10 +55,7 @@ export function extractCharsetFromHtmlBytes(bytes) {
  * @param response HTTP response object
  */
 export function parseContentTypeFromResponse(response) {
-    ow(response, ow.object.partialShape({
-        url: ow.string.url,
-        headers: new ObjectPredicate(),
-    }));
+    parseArgument(response, responseWithUrlSchema);
     const { url, headers } = response;
     let parsedContentType;
     if (headers.get('content-type')) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,19 +49,19 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.99",
-        "@crawlee/core": "4.0.0-beta.99",
-        "@crawlee/http-client": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
+        "@crawlee/basic": "4.0.0-rc.0",
+        "@crawlee/core": "4.0.0-rc.0",
+        "@crawlee/http-client": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/utils": "4.0.0-rc.0",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
         "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
-        "ow": "^2.0.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "type-fest": "^4.41.0",
+        "zod": "^4.4.3"
     },
     "lerna": {
         "command": {
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }