@crawlee/http 4.0.0-beta.26 → 4.0.0-beta.261
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +14 -51
- package/internals/file-download.js +20 -116
- package/internals/http-crawler.d.ts +221 -228
- package/internals/http-crawler.js +282 -233
- package/internals/utils.d.ts +4 -4
- package/internals/utils.js +18 -47
- package/package.json +17 -18
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/file-download.d.ts.map +0 -1
- package/internals/file-download.js.map +0 -1
- package/internals/http-crawler.d.ts.map +0 -1
- package/internals/http-crawler.js.map +0 -1
- package/internals/utils.d.ts.map +0 -1
- package/internals/utils.js.map +0 -1
|
@@ -1,25 +1,56 @@
|
|
|
1
|
-
import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest,
|
|
2
|
-
import { BasicCrawler,
|
|
3
|
-
import type { LoadedRequest } from '@crawlee/core';
|
|
1
|
+
import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
|
|
2
|
+
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
4
3
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
5
|
-
import {
|
|
6
|
-
import type {
|
|
4
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
5
|
+
import type { AnyNode } from 'domhandler';
|
|
7
6
|
import type { JsonValue } from 'type-fest';
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
/**
|
|
9
|
+
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
10
|
+
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
|
|
11
|
+
* concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
|
|
12
|
+
*
|
|
13
|
+
* A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
|
|
14
|
+
* wholesale, tuning included, so spread these options in if you want to keep it:
|
|
15
|
+
*
|
|
16
|
+
* ```typescript
|
|
17
|
+
* new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;
|
|
8
21
|
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
9
|
-
JSONData extends JsonValue = any
|
|
10
|
-
|
|
22
|
+
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
23
|
+
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
|
|
24
|
+
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
11
25
|
/**
|
|
12
|
-
* Timeout
|
|
26
|
+
* Timeout for the whole navigation phase, given in seconds. A single window shared by the
|
|
27
|
+
* `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
|
|
28
|
+
* so a slow hook eats into the same budget the navigation uses. Separate from the
|
|
29
|
+
* {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
|
|
30
|
+
* request handler.
|
|
13
31
|
*/
|
|
14
32
|
navigationTimeoutSecs?: number;
|
|
15
33
|
/**
|
|
16
|
-
* If set to true
|
|
34
|
+
* If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
|
|
35
|
+
* {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every navigation request, so custom
|
|
36
|
+
* {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
|
|
37
|
+
* clients do; the native fetch fallback cannot disable TLS verification and warns instead).
|
|
38
|
+
*
|
|
39
|
+
* @default false
|
|
17
40
|
*/
|
|
18
|
-
|
|
41
|
+
ignoreTlsErrors?: boolean;
|
|
19
42
|
/**
|
|
20
43
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
21
44
|
* or browser properties before navigation. The function accepts one parameter `crawlingContext`,
|
|
22
45
|
* which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
|
|
46
|
+
*
|
|
47
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
48
|
+
* allowing the hook to override context members for subsequent hooks and pipeline stages.
|
|
49
|
+
*
|
|
50
|
+
* The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
|
|
51
|
+
* `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
|
|
52
|
+
* This means the members added by `extendContext` are already available here, but navigation-dependent
|
|
53
|
+
* members (e.g. `response`, `body`, `$`) are not.
|
|
23
54
|
* Example:
|
|
24
55
|
* ```
|
|
25
56
|
* preNavigationHooks: [
|
|
@@ -28,27 +59,30 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
28
59
|
* },
|
|
29
60
|
* ]
|
|
30
61
|
* ```
|
|
31
|
-
*
|
|
32
|
-
* Modyfing `pageOptions` is supported only in Playwright incognito.
|
|
33
|
-
* See {@link PrePageCreateHook}
|
|
34
62
|
*/
|
|
35
|
-
preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
|
|
63
|
+
preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
|
|
36
64
|
/**
|
|
37
65
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
38
66
|
* The function accepts `crawlingContext` as the only parameter.
|
|
67
|
+
*
|
|
68
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
69
|
+
* which is useful for overriding the `response` after solving a challenge or re-fetching the resource.
|
|
39
70
|
* Example:
|
|
40
71
|
* ```
|
|
41
72
|
* postNavigationHooks: [
|
|
42
73
|
* async (crawlingContext) => {
|
|
43
|
-
*
|
|
74
|
+
* if (await needsRevalidation(crawlingContext)) {
|
|
75
|
+
* return { response: await refetch(crawlingContext.request) };
|
|
76
|
+
* }
|
|
44
77
|
* },
|
|
45
78
|
* ]
|
|
46
79
|
* ```
|
|
47
80
|
*/
|
|
48
|
-
postNavigationHooks?:
|
|
81
|
+
postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
|
|
49
82
|
/**
|
|
50
83
|
* An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
|
|
51
|
-
* you want the crawler to load and process. By default, only `text/html
|
|
84
|
+
* you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
85
|
+
* and `application/json` MIME types are supported.
|
|
52
86
|
*/
|
|
53
87
|
additionalMimeTypes?: string[];
|
|
54
88
|
/**
|
|
@@ -74,44 +108,26 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
74
108
|
*/
|
|
75
109
|
forceResponseEncoding?: string;
|
|
76
110
|
/**
|
|
77
|
-
* Automatically saves cookies to Session.
|
|
111
|
+
* Automatically saves cookies to Session. Enabled by default.
|
|
78
112
|
*
|
|
79
113
|
* It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
|
|
80
114
|
* It passes the "Cookie" header to the request with the session cookies.
|
|
81
115
|
*/
|
|
82
|
-
|
|
83
|
-
/**
|
|
84
|
-
* An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
|
|
85
|
-
* By default, status codes >= 500 trigger errors.
|
|
86
|
-
*/
|
|
87
|
-
ignoreHttpErrorStatusCodes?: number[];
|
|
88
|
-
/**
|
|
89
|
-
* An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
|
|
90
|
-
* By default, status codes >= 500 trigger errors.
|
|
91
|
-
*/
|
|
92
|
-
additionalHttpErrorStatusCodes?: number[];
|
|
116
|
+
saveResponseCookies?: boolean;
|
|
93
117
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
*/
|
|
97
|
-
export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
|
|
98
|
-
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
99
|
-
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
|
|
100
|
-
interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
118
|
+
export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
|
|
119
|
+
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
101
120
|
/**
|
|
102
121
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
103
122
|
*/
|
|
104
|
-
request: LoadedRequest<
|
|
123
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
105
124
|
/**
|
|
106
125
|
* The HTTP response object containing status code, headers, and other response metadata.
|
|
107
126
|
*/
|
|
108
127
|
response: Response;
|
|
109
128
|
}
|
|
110
|
-
/**
|
|
111
|
-
* @internal
|
|
112
|
-
*/
|
|
113
129
|
export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
114
|
-
JSONData extends JsonValue = any> extends
|
|
130
|
+
JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
|
|
115
131
|
/**
|
|
116
132
|
* The request body of the web page.
|
|
117
133
|
* The type depends on the `Content-Type` header of the web page:
|
|
@@ -131,7 +147,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
|
|
|
131
147
|
encoding: BufferEncoding;
|
|
132
148
|
};
|
|
133
149
|
/**
|
|
134
|
-
* Wait for an element matching the selector to appear.
|
|
150
|
+
* Wait for an element matching the selector to appear.
|
|
151
|
+
*
|
|
152
|
+
* `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
|
|
153
|
+
* nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
|
|
154
|
+
* the selector instead, with `timeoutMs` defaulting to 5s.
|
|
135
155
|
*
|
|
136
156
|
* **Example usage:**
|
|
137
157
|
* ```ts
|
|
@@ -143,9 +163,25 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
|
|
|
143
163
|
* ```
|
|
144
164
|
*/
|
|
145
165
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
|
|
168
|
+
* elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
|
|
169
|
+
* matches.
|
|
170
|
+
*/
|
|
171
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
|
|
172
|
+
/**
|
|
173
|
+
* Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
|
|
174
|
+
* match.
|
|
175
|
+
*/
|
|
176
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
|
|
146
177
|
/**
|
|
147
178
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
148
|
-
*
|
|
179
|
+
* This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
|
|
180
|
+
* the same return type as the `$` context property, so use it only if you are abstracting your workflow to
|
|
181
|
+
* support different context types in one handler.
|
|
182
|
+
*
|
|
183
|
+
* When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
|
|
184
|
+
* {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
|
|
149
185
|
*
|
|
150
186
|
* **Example usage:**
|
|
151
187
|
* ```ts
|
|
@@ -155,9 +191,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithReponse<UserData> {
|
|
|
155
191
|
* });
|
|
156
192
|
* ```
|
|
157
193
|
*/
|
|
158
|
-
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<
|
|
194
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
159
195
|
}
|
|
160
196
|
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
|
197
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
198
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
161
199
|
}
|
|
162
200
|
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
163
201
|
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
|
|
@@ -172,13 +210,15 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
172
210
|
*
|
|
173
211
|
* This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
|
|
174
212
|
*
|
|
175
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
176
|
-
* {@link
|
|
177
|
-
*
|
|
213
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
214
|
+
* {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
|
|
215
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
216
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
217
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
218
|
+
* result as `requestManager`.
|
|
178
219
|
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
220
|
+
* > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
|
|
221
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
182
222
|
*
|
|
183
223
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
184
224
|
*
|
|
@@ -192,18 +232,18 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
192
232
|
* ]
|
|
193
233
|
* ```
|
|
194
234
|
*
|
|
195
|
-
* By default, this crawler only processes web pages with the `text/html`
|
|
196
|
-
* and `application/
|
|
235
|
+
* By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
236
|
+
* and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
|
|
197
237
|
* and skips pages with other content types. If you want the crawler to process other content types,
|
|
198
238
|
* use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
|
|
199
239
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
200
240
|
* For details, see {@link HttpCrawlerOptions.requestHandler}.
|
|
201
241
|
*
|
|
202
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
* {@link
|
|
242
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
243
|
+
* {@link ConcurrencySystem}.
|
|
244
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
245
|
+
* constructor, or, for finer control, by injecting a pre-configured
|
|
246
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
207
247
|
*
|
|
208
248
|
* **Example usage:**
|
|
209
249
|
*
|
|
@@ -228,181 +268,134 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
228
268
|
* ```
|
|
229
269
|
* @category Crawlers
|
|
230
270
|
*/
|
|
231
|
-
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
protected navigationTimeoutMillis: number;
|
|
237
|
-
protected ignoreSslErrors: boolean;
|
|
238
|
-
protected suggestResponseEncoding?: string;
|
|
239
|
-
protected forceResponseEncoding?: string;
|
|
240
|
-
protected additionalHttpErrorStatusCodes: Set<number>;
|
|
241
|
-
protected ignoreHttpErrorStatusCodes: Set<number>;
|
|
242
|
-
protected readonly supportedMimeTypes: Set<string>;
|
|
271
|
+
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
272
|
+
#private;
|
|
273
|
+
/**
|
|
274
|
+
* @internal
|
|
275
|
+
*/
|
|
243
276
|
protected static optionsShape: {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
299
|
-
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
300
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
301
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
302
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
303
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
304
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
305
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
306
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
307
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
308
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
309
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
310
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
311
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
312
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
313
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
314
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
315
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
316
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
317
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
318
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
319
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
320
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
321
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
322
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
323
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
324
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
325
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
277
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
278
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
279
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
280
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
281
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
282
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
283
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
284
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
285
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
286
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
287
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
288
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
289
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
290
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
291
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
292
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
293
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
294
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
295
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
296
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
297
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
298
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
299
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
300
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
301
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
302
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
303
|
+
deferred: "deferred";
|
|
304
|
+
writeThrough: "writeThrough";
|
|
305
|
+
}>>;
|
|
306
|
+
}, z.core.$strict>]>>;
|
|
307
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
308
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
309
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
310
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
311
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
312
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
313
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
314
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
315
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
316
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
317
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
318
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
319
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
320
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
321
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
322
|
+
id: z.ZodOptional<z.ZodString>;
|
|
323
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
324
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
325
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
326
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
327
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
328
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
329
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
330
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
326
331
|
};
|
|
332
|
+
/** @internal */
|
|
333
|
+
protected static optionsSchema: z.ZodObject<{
|
|
334
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
335
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
336
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
337
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
338
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
339
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
340
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
341
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
342
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
343
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
344
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
345
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
346
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
347
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
348
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
349
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
350
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
351
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
352
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
353
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
354
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
355
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
356
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
357
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
358
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
359
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
360
|
+
deferred: "deferred";
|
|
361
|
+
writeThrough: "writeThrough";
|
|
362
|
+
}>>;
|
|
363
|
+
}, z.core.$strict>]>>;
|
|
364
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
365
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
366
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
367
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
368
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
369
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
370
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
371
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
372
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
373
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
374
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
375
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
376
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
377
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
378
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
379
|
+
id: z.ZodOptional<z.ZodString>;
|
|
380
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
381
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
382
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
383
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
384
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
385
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
386
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
387
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
388
|
+
}, z.core.$strict>;
|
|
327
389
|
/**
|
|
328
390
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
329
391
|
*/
|
|
330
|
-
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context
|
|
392
|
+
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
|
393
|
+
/** @internal */
|
|
394
|
+
protected getNavigationTimeoutMillis(): number;
|
|
395
|
+
/** @internal */
|
|
396
|
+
protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
|
|
331
397
|
protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
|
|
332
398
|
private makeHttpRequest;
|
|
333
|
-
private processHttpResponse;
|
|
334
|
-
private handleBlockedRequestByContent;
|
|
335
|
-
protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
|
|
336
|
-
/**
|
|
337
|
-
* Returns the `Cookie` header value based on the current context and
|
|
338
|
-
* any changes that occurred in the navigation hooks.
|
|
339
|
-
*/
|
|
340
|
-
protected _applyCookies({ session, request }: CrawlingContext, preHookCookies: string, postHookCookies: string): string;
|
|
341
|
-
/**
|
|
342
|
-
* Function to make the HTTP request. It performs optimizations
|
|
343
|
-
* on the request such as only downloading the request body if the
|
|
344
|
-
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
345
|
-
*/
|
|
346
|
-
protected _requestFunction({ request, session, proxyUrl, cookieString, }: RequestFunctionOptions): Promise<Response>;
|
|
347
|
-
/**
|
|
348
|
-
* Encodes and parses response according to the provided content type
|
|
349
|
-
*/
|
|
350
|
-
protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
|
|
351
|
-
response: Response;
|
|
352
|
-
contentType: {
|
|
353
|
-
type: string;
|
|
354
|
-
encoding: BufferEncoding;
|
|
355
|
-
};
|
|
356
|
-
body: string;
|
|
357
|
-
} | {
|
|
358
|
-
body: Buffer<ArrayBuffer>;
|
|
359
|
-
response: Response;
|
|
360
|
-
contentType: {
|
|
361
|
-
type: string;
|
|
362
|
-
encoding: BufferEncoding;
|
|
363
|
-
};
|
|
364
|
-
}>;
|
|
365
|
-
/**
|
|
366
|
-
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
367
|
-
*/
|
|
368
|
-
protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string): {
|
|
369
|
-
url: string;
|
|
370
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
371
|
-
method: import("@crawlee/types").AllowedHttpMethods;
|
|
372
|
-
proxyUrl: string | undefined;
|
|
373
|
-
timeout: number;
|
|
374
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
375
|
-
cookieJar: import("tough-cookie").CookieJar | undefined;
|
|
376
|
-
sessionToken: Session | undefined;
|
|
377
|
-
headers: Record<string, string> | undefined;
|
|
378
|
-
https: {
|
|
379
|
-
rejectUnauthorized: boolean;
|
|
380
|
-
};
|
|
381
|
-
body: string | undefined;
|
|
382
|
-
};
|
|
383
|
-
protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
|
|
384
|
-
encoding: BufferEncoding;
|
|
385
|
-
response: Response;
|
|
386
|
-
};
|
|
387
|
-
/**
|
|
388
|
-
* Checks and extends supported mime types
|
|
389
|
-
*/
|
|
390
|
-
protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
|
|
391
|
-
/**
|
|
392
|
-
* Handles timeout request
|
|
393
|
-
*/
|
|
394
|
-
protected _handleRequestTimeout(session?: Session): void;
|
|
395
|
-
private _abortDownloadOfBody;
|
|
396
|
-
/**
|
|
397
|
-
* @internal wraps public utility for mocking purposes
|
|
398
|
-
*/
|
|
399
|
-
private _requestAsBrowser;
|
|
400
|
-
}
|
|
401
|
-
interface RequestFunctionOptions {
|
|
402
|
-
request: CrawleeRequest;
|
|
403
|
-
session?: Session;
|
|
404
|
-
proxyUrl?: string;
|
|
405
|
-
cookieString?: string;
|
|
406
399
|
}
|
|
407
400
|
/**
|
|
408
401
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -428,7 +421,7 @@ interface RequestFunctionOptions {
|
|
|
428
421
|
* await crawler.run();
|
|
429
422
|
* ```
|
|
430
423
|
*/
|
|
431
|
-
|
|
432
|
-
export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData
|
|
424
|
+
export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
425
|
+
export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
426
|
+
export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
433
427
|
export {};
|
|
434
|
-
//# sourceMappingURL=http-crawler.d.ts.map
|