@crawlee/http 4.0.0-beta.2 → 4.0.0-beta.200

package/internals/http-crawler.d.ts

@@ -1,72 +1,87 @@
-import type { IncomingMessage } from 'node:http';
-import type { Readable } from 'node:stream';
-import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterRoutes, Session } from '@crawlee/basic';
-import { BasicCrawler, Configuration, CrawlerExtension } from '@crawlee/basic';
-import type { HttpResponse } from '@crawlee/core';
+import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
-import type { RequestLike, ResponseLike } from 'content-type';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { Method, OptionsInit } from 'got-scraping';
-import { ObjectPredicate } from 'ow';
+import type { CheerioAPI } from 'cheerio';
 import type { JsonValue } from 'type-fest';
+import { z } from 'zod';
 /**
- * TODO exists for BC within HttpCrawler - replace completely with StreamingHttpResponse in 4.0
- * @internal
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
  */
-export type PlainResponse = Omit<HttpResponse, 'body'> & IncomingMessage & {
-    body?: unknown;
-};
+export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
-export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext> extends BasicCrawlerOptions<Context> {
+JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
+export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
-     * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
+     * Timeout for the whole navigation phase, given in seconds. A single window shared by the
+     * `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
+     * so a slow hook eats into the same budget the navigation uses. Separate from the
+     * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
+     * request handler.
      */
     navigationTimeoutSecs?: number;
     /**
-     * If set to true, SSL certificate errors will be ignored.
-     */
-    ignoreSslErrors?: boolean;
-    /**
-     * If set, this crawler will be configured for all connections to use
-     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
-     * For more information, see the [documentation](https://docs.apify.com/proxy).
+     * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
+     * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every navigation request, so custom
+     * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
+     * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
+     *
+     * @default true
      */
-    proxyConfiguration?: ProxyConfiguration;
+    ignoreTlsErrors?: boolean;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
-     * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
+     * which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
+     *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context,
+     * allowing the hook to override context members for subsequent hooks and pipeline stages.
+     *
+     * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
+     * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
+     * This means the members added by `extendContext` are already available here, but navigation-dependent
+     * members (e.g. `response`, `body`, `$`) are not.
      * Example:
      * ```
      * preNavigationHooks: [
-     *     async (crawlingContext, gotOptions) => {
+     *     async (crawlingContext) => {
      *         // ...
      *     },
      * ]
      * ```
-     *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@link PrePageCreateHook}
      */
-    preNavigationHooks?: InternalHttpHook<Context>[];
+    preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
+     *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context,
+     * which is useful for overriding the `response` after solving a challenge or re-fetching the resource.
      * Example:
      * ```
      * postNavigationHooks: [
      *     async (crawlingContext) => {
-     *         // ...
+     *         if (await needsRevalidation(crawlingContext)) {
+     *             return { response: await refetch(crawlingContext.request) };
+     *         }
      *     },
      * ]
      * ```
      */
-    postNavigationHooks?: InternalHttpHook<Context>[];
+    postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
     /**
      * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
-     * you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
+     * you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+     * and `application/json` MIME types are supported.
      */
     additionalMimeTypes?: string[];
     /**
@@ -92,35 +107,26 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      */
     forceResponseEncoding?: string;
     /**
-     * Automatically saves cookies to Session. Works only if Session Pool is used.
+     * Automatically saves cookies to Session. Enabled by default.
      *
      * It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
      * It passes the "Cookie" header to the request with the session cookies.
      */
-    persistCookiesPerSession?: boolean;
+    saveResponseCookies?: boolean;
+}
+export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
+interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     /**
-     * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
-     * By default, status codes >= 500 trigger errors.
+     * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
      */
-    ignoreHttpErrorStatusCodes?: number[];
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     /**
-     * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
-     * By default, status codes >= 500 trigger errors.
+     * The HTTP response object containing status code, headers, and other response metadata.
      */
-    additionalHttpErrorStatusCodes?: number[];
+    response: Response;
 }
-/**
- * @internal
- */
-export type InternalHttpHook<Context> = (crawlingContext: Context, gotOptions: OptionsInit) => Awaitable<void>;
-export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
-/**
- * @internal
- */
 export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
-Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
+JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
     /**
      * The request body of the web page.
      * The type depends on the `Content-Type` header of the web page:
@@ -139,9 +145,12 @@ Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
         type: string;
         encoding: BufferEncoding;
     };
-    response: PlainResponse;
     /**
-     * Wait for an element matching the selector to appear. Timeout is ignored.
+     * Wait for an element matching the selector to appear.
+     *
+     * `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
+     * nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
+     * the selector instead, with `timeoutMs` defaulting to 5s.
      *
      * **Example usage:**
      * ```ts
@@ -155,7 +164,12 @@ Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
     waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
     /**
      * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
-     * When provided with the `selector` argument, it will throw if it's not available.
+     * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
+     * the same return type as the `$` context property, so use it only if you are abstracting your workflow to
+     * support different context types in one handler.
+     *
+     * When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
+     * {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
      *
      * **Example usage:**
      * ```ts
@@ -165,9 +179,9 @@ Crawler = HttpCrawler<any>> extends CrawlingContext<Crawler, UserData> {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
 }
-export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData, HttpCrawler<HttpCrawlingContext<UserData, JSONData>>> {
+export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
 }
 export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
@@ -182,38 +196,40 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  *
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```javascript
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, this crawler only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
  * For details, see {@link HttpCrawlerOptions.requestHandler}.
  *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -238,248 +254,165 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * ```
  * @category Crawlers
  */
-export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, HttpCrawler<Context>>> extends BasicCrawler<Context> {
-    readonly config: Configuration;
+export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
+     * @internal
      */
-    proxyConfiguration?: ProxyConfiguration;
-    protected userRequestHandlerTimeoutMillis: number;
-    protected preNavigationHooks: InternalHttpHook<Context>[];
-    protected postNavigationHooks: InternalHttpHook<Context>[];
-    protected persistCookiesPerSession: boolean;
-    protected navigationTimeoutMillis: number;
-    protected ignoreSslErrors: boolean;
-    protected suggestResponseEncoding?: string;
-    protected forceResponseEncoding?: string;
-    protected additionalHttpErrorStatusCodes: Set<number>;
-    protected ignoreHttpErrorStatusCodes: Set<number>;
-    protected readonly supportedMimeTypes: Set<string>;
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalMimeTypes: import("ow").ArrayPredicate<string>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
+        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
+        forceResponseEncoding: z.ZodOptional<z.ZodString>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
     };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
+        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
+        forceResponseEncoding: z.ZodOptional<z.ZodString>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+    }, z.core.$strict>;
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options?: HttpCrawlerOptions<Context>, config?: Configuration);
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension: CrawlerExtension): void;
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    protected _runRequestHandler(crawlingContext: Context): Promise<void>;
-    protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
-    protected _handleNavigation(crawlingContext: Context): Promise<void>;
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    protected _applyCookies({ session, request }: CrawlingContext, gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string): void;
+    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+    /** @internal */
+    protected getNavigationTimeoutMillis(): number;
+    /** @internal */
+    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
+    private prepareHttpRequest;
+    private makeHttpRequest;
+    private processHttpResponse;
+    private handleBlockedRequestByContent;
     /**
      * Function to make the HTTP request. It performs optimizations
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    protected _requestFunction({ request, session, proxyUrl, gotOptions, }: RequestFunctionOptions): Promise<PlainResponse>;
+    private requestFunction;
     /**
      * Encodes and parses response according to the provided content type
      */
-    protected _parseResponse(request: Request, responseStream: IncomingMessage, crawlingContext: Context): Promise<(Partial<Context> & {
-        isXml: boolean;
-        response: IncomingMessage;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-    }) | {
-        body: Buffer<ArrayBufferLike>;
-        response: IncomingMessage;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        enqueueLinks: () => Promise<{
-            processedRequests: never[];
-            unprocessedRequests: never[];
-        }>;
-    }>;
-    protected _parseHTML(response: IncomingMessage, _isXml: boolean, _crawlingContext: Context): Promise<Partial<Context>>;
+    private parseResponse;
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        body?: string | Buffer | Readable | Generator | AsyncGenerator | import("form-data-encoder").FormDataLike | undefined;
-        json?: unknown;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        request?: import("got-scraping").RequestFunction | undefined;
-        url?: string | URL | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        headers?: import("got-scraping").Headers | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        agent?: import("got-scraping").Agents | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        h2session?: import("http2").ClientHttp2Session | undefined;
-        decompress?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        timeout?: import("got-scraping").Delays | undefined;
-        prefixUrl?: string | URL | undefined;
-        form?: Record<string, any> | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cookieJar?: import("got-scraping").PromiseCookieJar | import("got-scraping").ToughCookieJar | undefined;
-        signal?: AbortSignal | undefined;
-        ignoreInvalidCookies?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        searchParams?: string | import("got-scraping").SearchParameters | URLSearchParams | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsLookup?: import("cacheable-lookup").default["lookup"] | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsCache?: import("cacheable-lookup").default | boolean | undefined;
-        context?: Record<string, unknown> | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        followRedirect?: boolean | ((response: import("got-scraping").PlainResponse) => boolean) | undefined;
-        maxRedirects?: number | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cache?: string | import("cacheable-request").StorageAdapter | boolean | undefined;
-        throwHttpErrors?: boolean | undefined;
-        username?: string | undefined;
-        password?: string | undefined;
-        http2?: boolean | undefined;
-        allowGetBody?: boolean | undefined;
-        methodRewriting?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        dnsLookupIpVersion?: import("got-scraping").DnsLookupIpVersion;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        parseJson?: import("got-scraping").ParseJsonFunction | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        stringifyJson?: import("got-scraping").StringifyJsonFunction | undefined;
-        localAddress?: string | undefined;
-        method?: Method | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        createConnection?: import("got-scraping").CreateConnectionFunction | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        cacheOptions?: import("got-scraping").CacheOptions | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        https?: import("got-scraping").HttpsOptions | undefined;
-        encoding?: BufferEncoding | undefined;
-        resolveBodyOnly?: boolean | undefined;
-        isStream?: boolean | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        responseType?: import("got-scraping").ResponseType | undefined;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        pagination?: import("got-scraping").PaginationOptions<unknown, unknown> | undefined;
-        setHost?: boolean | undefined;
-        maxHeaderSize?: number | undefined;
-        enableUnixSockets?: boolean | undefined;
-    } & {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        hooks?: Partial<import("got-scraping").Hooks>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retry?: Partial<import("got-scraping").RetryOptions>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-    } & import("got-scraping").Context & Required<Pick<OptionsInit, "url">> & {
-        isStream: true;
-    };
-    protected _encodeResponse(request: Request, response: IncomingMessage, encoding: BufferEncoding): {
-        encoding: BufferEncoding;
-        response: IncomingMessage;
-    };
+    private getRequestOptions;
+    private encodeResponse;
     /**
      * Checks and extends supported mime types
      */
-    protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
+    private extendSupportedMimeTypes;
     /**
      * Handles timeout request
      */
-    protected _handleRequestTimeout(session?: Session): void;
-    private _abortDownloadOfBody;
+    private handleRequestTimeout;
+    private abortDownloadOfBody;
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _requestAsBrowser;
-}
-interface RequestFunctionOptions {
-    request: Request;
-    session?: Session;
-    proxyUrl?: string;
-    gotOptions: OptionsInit;
+    private requestAsBrowser;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.
@@ -505,7 +438,7 @@ interface RequestFunctionOptions {
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 export {};
-//# sourceMappingURL=http-crawler.d.ts.map