@crawlee/http 4.0.0-beta.25 → 4.0.0-beta.251

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,25 +1,56 @@
1
- import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
2
- import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
3
- import type { LoadedRequest } from '@crawlee/core';
1
+ import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
2
+ import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
4
3
  import type { Awaitable, Dictionary } from '@crawlee/types';
5
- import { type CheerioRoot } from '@crawlee/utils';
6
- import type { RequestLike, ResponseLike } from 'content-type';
4
+ import type { Cheerio, CheerioAPI } from 'cheerio';
5
+ import type { AnyNode } from 'domhandler';
7
6
  import type { JsonValue } from 'type-fest';
7
+ import { z } from 'zod';
8
+ /**
9
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
10
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
11
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
12
+ *
13
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
14
+ * wholesale, tuning included, so spread these options in if you want to keep it:
15
+ *
16
+ * ```typescript
17
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
18
+ * ```
19
+ */
20
+ export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;
8
21
  export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
- JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
10
- export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
22
+ JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
23
+ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
24
+ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
11
25
  /**
12
- * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
26
+ * Timeout for the whole navigation phase, given in seconds. A single window shared by the
27
+ * `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
28
+ * so a slow hook eats into the same budget the navigation uses. Separate from the
29
+ * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
30
+ * request handler.
13
31
  */
14
32
  navigationTimeoutSecs?: number;
15
33
  /**
16
- * If set to true, SSL certificate errors will be ignored.
34
+ * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
35
+ * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every navigation request, so custom
36
+ * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
37
+ * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
38
+ *
39
+ * @default false
17
40
  */
18
- ignoreSslErrors?: boolean;
41
+ ignoreTlsErrors?: boolean;
19
42
  /**
20
43
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
21
44
  * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
22
45
  * which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
46
+ *
47
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
48
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
49
+ *
50
+ * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
51
+ * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
52
+ * This means the members added by `extendContext` are already available here, but navigation-dependent
53
+ * members (e.g. `response`, `body`, `$`) are not.
23
54
  * Example:
24
55
  * ```
25
56
  * preNavigationHooks: [
@@ -28,27 +59,30 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
28
59
  * },
29
60
  * ]
30
61
  * ```
31
- *
32
- * Modyfing `pageOptions` is supported only in Playwright incognito.
33
- * See {@link PrePageCreateHook}
34
62
  */
35
- preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
63
+ preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
36
64
  /**
37
65
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
38
66
  * The function accepts `crawlingContext` as the only parameter.
67
+ *
68
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
69
+ * which is useful for overriding the `response` after solving a challenge or re-fetching the resource.
39
70
  * Example:
40
71
  * ```
41
72
  * postNavigationHooks: [
42
73
  * async (crawlingContext) => {
43
- * // ...
74
+ * if (await needsRevalidation(crawlingContext)) {
75
+ * return { response: await refetch(crawlingContext.request) };
76
+ * }
44
77
  * },
45
78
  * ]
46
79
  * ```
47
80
  */
48
- postNavigationHooks?: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
81
+ postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
49
82
  /**
50
83
  * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
51
- * you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
84
+ * you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
85
+ * and `application/json` MIME types are supported.
52
86
  */
53
87
  additionalMimeTypes?: string[];
54
88
  /**
@@ -74,44 +108,26 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
74
108
  */
75
109
  forceResponseEncoding?: string;
76
110
  /**
77
- * Automatically saves cookies to Session. Works only if Session Pool is used.
111
+ * Automatically saves cookies to Session. Enabled by default.
78
112
  *
79
113
  * It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
80
114
  * It passes the "Cookie" header to the request with the session cookies.
81
115
  */
82
- persistCookiesPerSession?: boolean;
83
- /**
84
- * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
85
- * By default, status codes >= 500 trigger errors.
86
- */
87
- ignoreHttpErrorStatusCodes?: number[];
88
- /**
89
- * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
90
- * By default, status codes >= 500 trigger errors.
91
- */
92
- additionalHttpErrorStatusCodes?: number[];
116
+ saveResponseCookies?: boolean;
93
117
  }
94
- /**
95
- * @internal
96
- */
97
- export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
98
- export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
99
- JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
100
- interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
118
+ export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
119
+ interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
101
120
  /**
102
121
  * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
103
122
  */
104
- request: LoadedRequest<CrawleeRequest<UserData>>;
123
+ request: LoadedRequest<CrawlingRequest<UserData>>;
105
124
  /**
106
125
  * The HTTP response object containing status code, headers, and other response metadata.
107
126
  */
108
127
  response: Response;
109
128
  }
110
- /**
111
- * @internal
112
- */
113
129
  export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
114
- JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
130
+ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
115
131
  /**
116
132
  * The request body of the web page.
117
133
  * The type depends on the `Content-Type` header of the web page:
@@ -131,7 +147,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
131
147
  encoding: BufferEncoding;
132
148
  };
133
149
  /**
134
- * Wait for an element matching the selector to appear. Timeout is ignored.
150
+ * Wait for an element matching the selector to appear.
151
+ *
152
+ * `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
153
+ * nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
154
+ * the selector instead, with `timeoutMs` defaulting to 5s.
135
155
  *
136
156
  * **Example usage:**
137
157
  * ```ts
@@ -143,9 +163,25 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
143
163
  * ```
144
164
  */
145
165
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
166
+ /**
167
+ * Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
168
+ * elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
169
+ * matches.
170
+ */
171
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
172
+ /**
173
+ * Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
174
+ * match.
175
+ */
176
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
146
177
  /**
147
178
  * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
148
- * When provided with the `selector` argument, it will throw if it's not available.
179
+ * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
180
+ * the same return type as the `$` context property, so use it only if you are abstracting your workflow to
181
+ * support different context types in one handler.
182
+ *
183
+ * When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
184
+ * {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
149
185
  *
150
186
  * **Example usage:**
151
187
  * ```ts
@@ -155,9 +191,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
155
191
  * });
156
192
  * ```
157
193
  */
158
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
194
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
159
195
  }
160
196
  export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
197
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
198
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
161
199
  }
162
200
  export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
163
201
  JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
@@ -172,13 +210,15 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
172
210
  *
173
211
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
174
212
  *
175
- * The source URLs are represented using {@link Request} objects that are fed from
176
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
177
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
213
+ * The source URLs are represented using {@link Request} objects that are fed from the
214
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
215
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
216
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
217
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
218
+ * result as `requestManager`.
178
219
  *
179
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
180
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
181
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
220
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
221
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
182
222
  *
183
223
  * The crawler finishes when there are no more {@link Request} objects to crawl.
184
224
  *
@@ -192,18 +232,18 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
192
232
  * ]
193
233
  * ```
194
234
  *
195
- * By default, this crawler only processes web pages with the `text/html`
196
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
235
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
236
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
197
237
  * and skips pages with other content types. If you want the crawler to process other content types,
198
238
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
199
239
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
200
240
  * For details, see {@link HttpCrawlerOptions.requestHandler}.
201
241
  *
202
- * New requests are only dispatched when there is enough free CPU and memory available,
203
- * using the functionality provided by the {@link AutoscaledPool} class.
204
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
205
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
206
- * {@link AutoscaledPool} options are available directly in the constructor.
242
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
243
+ * {@link ConcurrencySystem}.
244
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
245
+ * constructor, or, for finer control, by injecting a pre-configured
246
+ * {@link ConcurrencySystem|`concurrencySystem`}.
207
247
  *
208
248
  * **Example usage:**
209
249
  *
@@ -228,181 +268,134 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
228
268
  * ```
229
269
  * @category Crawlers
230
270
  */
231
- export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
232
- readonly config: Configuration;
233
- protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
234
- protected postNavigationHooks: ((crawlingContext: CrawlingContextWithReponse) => Awaitable<void>)[];
235
- protected persistCookiesPerSession: boolean;
236
- protected navigationTimeoutMillis: number;
237
- protected ignoreSslErrors: boolean;
238
- protected suggestResponseEncoding?: string;
239
- protected forceResponseEncoding?: string;
240
- protected additionalHttpErrorStatusCodes: Set<number>;
241
- protected ignoreHttpErrorStatusCodes: Set<number>;
242
- protected readonly supportedMimeTypes: Set<string>;
271
+ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
272
+ #private;
273
+ /**
274
+ * @internal
275
+ */
243
276
  protected static optionsShape: {
244
- // @ts-ignore optional peer dependency or compatibility with es2022
245
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
246
- // @ts-ignore optional peer dependency or compatibility with es2022
247
- ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
248
- // @ts-ignore optional peer dependency or compatibility with es2022
249
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
250
- // @ts-ignore optional peer dependency or compatibility with es2022
251
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
252
- // @ts-ignore optional peer dependency or compatibility with es2022
253
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
254
- // @ts-ignore optional peer dependency or compatibility with es2022
255
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
258
- // @ts-ignore optional peer dependency or compatibility with es2022
259
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
260
- // @ts-ignore optional peer dependency or compatibility with es2022
261
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
262
- // @ts-ignore optional peer dependency or compatibility with es2022
263
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
266
- // @ts-ignore optional peer dependency or compatibility with es2022
267
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
268
- // @ts-ignore optional peer dependency or compatibility with es2022
269
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
270
- // @ts-ignore optional peer dependency or compatibility with es2022
271
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
272
- // @ts-ignore optional peer dependency or compatibility with es2022
273
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
274
- // @ts-ignore optional peer dependency or compatibility with es2022
275
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
276
- // @ts-ignore optional peer dependency or compatibility with es2022
277
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
278
- // @ts-ignore optional peer dependency or compatibility with es2022
279
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
280
- // @ts-ignore optional peer dependency or compatibility with es2022
281
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
282
- // @ts-ignore optional peer dependency or compatibility with es2022
283
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
284
- // @ts-ignore optional peer dependency or compatibility with es2022
285
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
286
- // @ts-ignore optional peer dependency or compatibility with es2022
287
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
288
- // @ts-ignore optional peer dependency or compatibility with es2022
289
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
290
- // @ts-ignore optional peer dependency or compatibility with es2022
291
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
292
- // @ts-ignore optional peer dependency or compatibility with es2022
293
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
294
- // @ts-ignore optional peer dependency or compatibility with es2022
295
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
296
- // @ts-ignore optional peer dependency or compatibility with es2022
297
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
298
- // @ts-ignore optional peer dependency or compatibility with es2022
299
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
300
- // @ts-ignore optional peer dependency or compatibility with es2022
301
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
302
- // @ts-ignore optional peer dependency or compatibility with es2022
303
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
304
- // @ts-ignore optional peer dependency or compatibility with es2022
305
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
306
- // @ts-ignore optional peer dependency or compatibility with es2022
307
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
308
- // @ts-ignore optional peer dependency or compatibility with es2022
309
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
310
- // @ts-ignore optional peer dependency or compatibility with es2022
311
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
312
- // @ts-ignore optional peer dependency or compatibility with es2022
313
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
314
- // @ts-ignore optional peer dependency or compatibility with es2022
315
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
316
- // @ts-ignore optional peer dependency or compatibility with es2022
317
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
318
- // @ts-ignore optional peer dependency or compatibility with es2022
319
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
320
- // @ts-ignore optional peer dependency or compatibility with es2022
321
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
322
- // @ts-ignore optional peer dependency or compatibility with es2022
323
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
324
- // @ts-ignore optional peer dependency or compatibility with es2022
325
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
277
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
278
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
279
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
280
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
281
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
282
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
283
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
284
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
285
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
286
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
287
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
288
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
289
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
290
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
291
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
292
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
293
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
294
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
295
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
296
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
297
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
298
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
299
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
300
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
301
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
302
+ requestQueue: z.ZodOptional<z.ZodEnum<{
303
+ deferred: "deferred";
304
+ writeThrough: "writeThrough";
305
+ }>>;
306
+ }, z.core.$strict>]>>;
307
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
308
+ // @ts-ignore optional peer dependency or compatibility with es2022
309
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
310
+ // @ts-ignore optional peer dependency or compatibility with es2022
311
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
312
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
313
+ // @ts-ignore optional peer dependency or compatibility with es2022
314
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
315
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
316
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
317
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
318
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
319
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
320
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
321
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
322
+ id: z.ZodOptional<z.ZodString>;
323
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
324
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
325
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
326
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
327
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
328
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
329
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
330
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
326
331
  };
332
+ /** @internal */
333
+ protected static optionsSchema: z.ZodObject<{
334
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
335
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
336
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
337
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
338
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
339
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
340
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
341
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
342
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
343
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
344
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
345
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
346
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
347
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
348
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
349
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
350
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
351
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
352
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
353
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
354
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
355
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
356
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
357
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
358
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
359
+ requestQueue: z.ZodOptional<z.ZodEnum<{
360
+ deferred: "deferred";
361
+ writeThrough: "writeThrough";
362
+ }>>;
363
+ }, z.core.$strict>]>>;
364
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
365
+ // @ts-ignore optional peer dependency or compatibility with es2022
366
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
367
+ // @ts-ignore optional peer dependency or compatibility with es2022
368
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
369
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
370
+ // @ts-ignore optional peer dependency or compatibility with es2022
371
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
372
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
373
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
374
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
375
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
376
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
377
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
378
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
379
+ id: z.ZodOptional<z.ZodString>;
380
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
381
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
382
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
383
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
384
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
385
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
386
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
387
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
388
+ }, z.core.$strict>;
327
389
  /**
328
390
  * All `HttpCrawlerOptions` parameters are passed via an options object.
329
391
  */
330
- constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>, config?: Configuration);
392
+ constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
393
+ /** @internal */
394
+ protected getNavigationTimeoutMillis(): number;
395
+ /** @internal */
396
+ protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
331
397
  protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
332
398
  private makeHttpRequest;
333
- private processHttpResponse;
334
- private handleBlockedRequestByContent;
335
- protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
336
- /**
337
- * Returns the `Cookie` header value based on the current context and
338
- * any changes that occurred in the navigation hooks.
339
- */
340
- protected _applyCookies({ session, request }: CrawlingContext, preHookCookies: string, postHookCookies: string): string;
341
- /**
342
- * Function to make the HTTP request. It performs optimizations
343
- * on the request such as only downloading the request body if the
344
- * received content type matches text/html, application/xml, application/xhtml+xml.
345
- */
346
- protected _requestFunction({ request, session, proxyUrl, cookieString, }: RequestFunctionOptions): Promise<Response>;
347
- /**
348
- * Encodes and parses response according to the provided content type
349
- */
350
- protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
351
- response: Response;
352
- contentType: {
353
- type: string;
354
- encoding: BufferEncoding;
355
- };
356
- body: string;
357
- } | {
358
- body: Buffer<ArrayBuffer>;
359
- response: Response;
360
- contentType: {
361
- type: string;
362
- encoding: BufferEncoding;
363
- };
364
- }>;
365
- /**
366
- * Combines the provided `requestOptions` with mandatory (non-overridable) values.
367
- */
368
- protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string): {
369
- url: string;
370
- // @ts-ignore optional peer dependency or compatibility with es2022
371
- method: import("@crawlee/types").AllowedHttpMethods;
372
- proxyUrl: string | undefined;
373
- timeout: number;
374
- // @ts-ignore optional peer dependency or compatibility with es2022
375
- cookieJar: import("tough-cookie").CookieJar | undefined;
376
- sessionToken: Session | undefined;
377
- headers: Record<string, string> | undefined;
378
- https: {
379
- rejectUnauthorized: boolean;
380
- };
381
- body: string | undefined;
382
- };
383
- protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
384
- encoding: BufferEncoding;
385
- response: Response;
386
- };
387
- /**
388
- * Checks and extends supported mime types
389
- */
390
- protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
391
- /**
392
- * Handles timeout request
393
- */
394
- protected _handleRequestTimeout(session?: Session): void;
395
- private _abortDownloadOfBody;
396
- /**
397
- * @internal wraps public utility for mocking purposes
398
- */
399
- private _requestAsBrowser;
400
- }
401
- interface RequestFunctionOptions {
402
- request: CrawleeRequest;
403
- session?: Session;
404
- proxyUrl?: string;
405
- cookieString?: string;
406
399
  }
407
400
  /**
408
401
  * Creates new {@link Router} instance that works based on request labels.
@@ -428,7 +421,7 @@ interface RequestFunctionOptions {
428
421
  * await crawler.run();
429
422
  * ```
430
423
  */
431
- // @ts-ignore optional peer dependency or compatibility with es2022
432
- export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
424
+ export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
425
+ export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
426
+ export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
433
427
  export {};
434
- //# sourceMappingURL=http-crawler.d.ts.map