@crawlee/http 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +131 -64
- package/internals/file-download.js +39 -122
- package/internals/http-crawler.d.ts +57 -66
- package/internals/http-crawler.js +97 -86
- package/internals/utils.d.ts +0 -5
- package/internals/utils.js +7 -44
- package/package.json +15 -17
|
@@ -1,67 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { BasicCrawler } from '@crawlee/basic';
|
|
1
|
+
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
2
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
3
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import { Router } from '../index.js';
|
|
5
6
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
6
|
-
const kBodyDrained = Symbol('bodyDrained');
|
|
7
|
-
/**
|
|
8
|
-
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
9
|
-
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
10
|
-
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
11
|
-
* it will throw an error.
|
|
12
|
-
*
|
|
13
|
-
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
14
|
-
* @returns Transform stream that monitors the speed of the incoming data.
|
|
15
|
-
*/
|
|
16
|
-
export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
|
|
17
|
-
let snapshots = [];
|
|
18
|
-
const checkInterval = setInterval(() => {
|
|
19
|
-
const now = Date.now();
|
|
20
|
-
snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
|
|
21
|
-
const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
|
|
22
|
-
const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
|
|
23
|
-
if (totalBytes / 1024 / elapsed < minSpeedKbps) {
|
|
24
|
-
clearInterval(checkInterval);
|
|
25
|
-
stream.emit('error', new Error(`Stream speed too slow, aborting...`));
|
|
26
|
-
}
|
|
27
|
-
}, checkProgressIntervalMs);
|
|
28
|
-
const stream = new Transform({
|
|
29
|
-
transform: (chunk, _, callback) => {
|
|
30
|
-
snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
|
|
31
|
-
callback(null, chunk);
|
|
32
|
-
},
|
|
33
|
-
final: (callback) => {
|
|
34
|
-
clearInterval(checkInterval);
|
|
35
|
-
callback();
|
|
36
|
-
},
|
|
37
|
-
});
|
|
38
|
-
return stream;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Creates a transform stream that logs the progress of the incoming data.
|
|
42
|
-
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
43
|
-
*
|
|
44
|
-
* Can be used e.g. to log the progress of a download.
|
|
45
|
-
* @returns Transform stream logging the progress of the incoming data.
|
|
46
|
-
*/
|
|
47
|
-
export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
|
|
48
|
-
let transferredBytes = 0;
|
|
49
|
-
let lastLogTime = Date.now();
|
|
50
|
-
return new Transform({
|
|
51
|
-
transform: (chunk, _, callback) => {
|
|
52
|
-
transferredBytes += chunk.length;
|
|
53
|
-
if (Date.now() - lastLogTime > loggingInterval) {
|
|
54
|
-
lastLogTime = Date.now();
|
|
55
|
-
logTransferredBytes(transferredBytes);
|
|
56
|
-
}
|
|
57
|
-
callback(null, chunk);
|
|
58
|
-
},
|
|
59
|
-
flush: (callback) => {
|
|
60
|
-
logTransferredBytes(transferredBytes);
|
|
61
|
-
callback();
|
|
62
|
-
},
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
7
|
/**
|
|
66
8
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
67
9
|
*
|
|
@@ -69,32 +11,22 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
69
11
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
70
12
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
71
13
|
*
|
|
72
|
-
* `
|
|
14
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
73
15
|
*
|
|
74
|
-
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link
|
|
16
|
+
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
75
17
|
*
|
|
76
|
-
* > The {@link
|
|
18
|
+
* > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
77
19
|
*
|
|
78
20
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
79
21
|
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* ```
|
|
83
|
-
* preNavigationHooks: [
|
|
84
|
-
* (crawlingContext) => {
|
|
85
|
-
* // ...
|
|
86
|
-
* },
|
|
87
|
-
* ]
|
|
88
|
-
* ```
|
|
89
|
-
*
|
|
90
|
-
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
22
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
91
23
|
*
|
|
92
24
|
* ## Example usage
|
|
93
25
|
*
|
|
94
26
|
* ```ts
|
|
95
|
-
* const crawler = new
|
|
96
|
-
* requestHandler({
|
|
97
|
-
*
|
|
27
|
+
* const crawler = new FileDownload({
|
|
28
|
+
* async requestHandler({ response, request }) {
|
|
29
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
98
30
|
* },
|
|
99
31
|
* });
|
|
100
32
|
*
|
|
@@ -106,40 +38,49 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
106
38
|
* ```
|
|
107
39
|
*/
|
|
108
40
|
export class FileDownload extends BasicCrawler {
|
|
41
|
+
#ignoreTlsErrors;
|
|
42
|
+
/**
|
|
43
|
+
* @internal
|
|
44
|
+
*/
|
|
45
|
+
static optionsShape = {
|
|
46
|
+
...BasicCrawler.optionsShape,
|
|
47
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
48
|
+
};
|
|
49
|
+
/** @internal */
|
|
50
|
+
static optionsSchema = z.strictObject(FileDownload.optionsShape);
|
|
109
51
|
// TODO hooks
|
|
110
52
|
constructor(options = {}) {
|
|
53
|
+
const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
|
|
111
54
|
super({
|
|
112
|
-
...
|
|
113
|
-
contextPipelineBuilder: () => this
|
|
55
|
+
...basicCrawlerOptions,
|
|
56
|
+
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
114
57
|
});
|
|
58
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
115
59
|
}
|
|
116
|
-
buildContextPipeline() {
|
|
117
|
-
return
|
|
118
|
-
action: async (context) => this.initiateDownload(context),
|
|
119
|
-
cleanup: async (context) => {
|
|
120
|
-
if (!context.response.bodyUsed) {
|
|
121
|
-
// Nobody consumed the body — cancel it so the
|
|
122
|
-
// underlying connection can be released.
|
|
123
|
-
await context.response.body?.cancel();
|
|
124
|
-
}
|
|
125
|
-
await context[kBodyDrained];
|
|
126
|
-
},
|
|
127
|
-
});
|
|
60
|
+
#buildContextPipeline() {
|
|
61
|
+
return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
|
|
128
62
|
}
|
|
129
|
-
async initiateDownload(context) {
|
|
63
|
+
async #initiateDownload(context, onCleanup) {
|
|
130
64
|
const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
|
|
131
65
|
session: context.session,
|
|
66
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
132
67
|
});
|
|
133
68
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
134
69
|
context.request.url = response.url;
|
|
135
70
|
const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
|
|
136
|
-
|
|
71
|
+
onCleanup(async () => {
|
|
72
|
+
if (!trackedResponse.bodyUsed) {
|
|
73
|
+
// Nobody consumed the body — cancel it so the
|
|
74
|
+
// underlying connection can be released.
|
|
75
|
+
await trackedResponse.body?.cancel();
|
|
76
|
+
}
|
|
77
|
+
await bodyDrained;
|
|
78
|
+
});
|
|
79
|
+
return {
|
|
137
80
|
request: context.request,
|
|
138
81
|
response: trackedResponse,
|
|
139
82
|
contentType: { type, encoding },
|
|
140
|
-
[kBodyDrained]: bodyDrained,
|
|
141
83
|
};
|
|
142
|
-
return contextExtension;
|
|
143
84
|
}
|
|
144
85
|
}
|
|
145
86
|
/**
|
|
@@ -162,30 +103,6 @@ function trackBodyConsumption(response) {
|
|
|
162
103
|
});
|
|
163
104
|
return { response: trackedResponse, bodyDrained };
|
|
164
105
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
* This instance can then serve as a `requestHandler` of your {@link FileDownload}.
|
|
168
|
-
* Defaults to the {@link FileDownloadCrawlingContext}.
|
|
169
|
-
*
|
|
170
|
-
* > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
|
|
171
|
-
*
|
|
172
|
-
* ```ts
|
|
173
|
-
* import { FileDownload, createFileRouter } from 'crawlee';
|
|
174
|
-
*
|
|
175
|
-
* const router = createFileRouter();
|
|
176
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
177
|
-
* ctx.log.info('...');
|
|
178
|
-
* });
|
|
179
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
180
|
-
* ctx.log.info('...');
|
|
181
|
-
* });
|
|
182
|
-
*
|
|
183
|
-
* const crawler = new FileDownload({
|
|
184
|
-
* requestHandler: router,
|
|
185
|
-
* });
|
|
186
|
-
* await crawler.run();
|
|
187
|
-
* ```
|
|
188
|
-
*/
|
|
189
|
-
export function createFileRouter(routes) {
|
|
190
|
-
return Router.create(routes);
|
|
106
|
+
export function createFileRouter(routesOrSchemas) {
|
|
107
|
+
return Router.create(routesOrSchemas);
|
|
191
108
|
}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest,
|
|
1
|
+
import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
|
-
import { type LoadedRequest } from '@crawlee/core';
|
|
4
3
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
5
|
-
import {
|
|
4
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
5
|
+
import type { AnyNode } from 'domhandler';
|
|
6
6
|
import type { JsonValue } from 'type-fest';
|
|
7
7
|
import { z } from 'zod';
|
|
8
8
|
/**
|
|
9
9
|
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
10
|
-
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default
|
|
10
|
+
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
|
|
11
|
+
* concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
|
|
11
12
|
*
|
|
12
13
|
* A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
|
|
13
14
|
* wholesale, tuning included, so spread these options in if you want to keep it:
|
|
@@ -35,7 +36,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
35
36
|
* {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
|
|
36
37
|
* clients do; the native fetch fallback cannot disable TLS verification and warns instead).
|
|
37
38
|
*
|
|
38
|
-
* @default
|
|
39
|
+
* @default false
|
|
39
40
|
*/
|
|
40
41
|
ignoreTlsErrors?: boolean;
|
|
41
42
|
/**
|
|
@@ -77,7 +78,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
77
78
|
* ]
|
|
78
79
|
* ```
|
|
79
80
|
*/
|
|
80
|
-
postNavigationHooks?:
|
|
81
|
+
postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
|
|
81
82
|
/**
|
|
82
83
|
* An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
|
|
83
84
|
* you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
@@ -115,13 +116,11 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
115
116
|
saveResponseCookies?: boolean;
|
|
116
117
|
}
|
|
117
118
|
export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
|
|
118
|
-
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
119
|
-
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
|
|
120
119
|
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
121
120
|
/**
|
|
122
121
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
123
122
|
*/
|
|
124
|
-
request: LoadedRequest<
|
|
123
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
125
124
|
/**
|
|
126
125
|
* The HTTP response object containing status code, headers, and other response metadata.
|
|
127
126
|
*/
|
|
@@ -148,7 +147,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
|
|
|
148
147
|
encoding: BufferEncoding;
|
|
149
148
|
};
|
|
150
149
|
/**
|
|
151
|
-
* Wait for an element matching the selector to appear.
|
|
150
|
+
* Wait for an element matching the selector to appear.
|
|
151
|
+
*
|
|
152
|
+
* `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
|
|
153
|
+
* nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
|
|
154
|
+
* the selector instead, with `timeoutMs` defaulting to 5s.
|
|
152
155
|
*
|
|
153
156
|
* **Example usage:**
|
|
154
157
|
* ```ts
|
|
@@ -160,9 +163,25 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
|
|
|
160
163
|
* ```
|
|
161
164
|
*/
|
|
162
165
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
|
|
168
|
+
* elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
|
|
169
|
+
* matches.
|
|
170
|
+
*/
|
|
171
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
|
|
172
|
+
/**
|
|
173
|
+
* Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
|
|
174
|
+
* match.
|
|
175
|
+
*/
|
|
176
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
|
|
163
177
|
/**
|
|
164
178
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
165
|
-
*
|
|
179
|
+
* This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
|
|
180
|
+
* the same return type as the `$` context property, so use it only if you are abstracting your workflow to
|
|
181
|
+
* support different context types in one handler.
|
|
182
|
+
*
|
|
183
|
+
* When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
|
|
184
|
+
* {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
|
|
166
185
|
*
|
|
167
186
|
* **Example usage:**
|
|
168
187
|
* ```ts
|
|
@@ -172,9 +191,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
|
|
|
172
191
|
* });
|
|
173
192
|
* ```
|
|
174
193
|
*/
|
|
175
|
-
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<
|
|
194
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
176
195
|
}
|
|
177
196
|
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
|
197
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
198
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
178
199
|
}
|
|
179
200
|
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
180
201
|
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
|
|
@@ -249,15 +270,10 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
249
270
|
*/
|
|
250
271
|
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
251
272
|
#private;
|
|
273
|
+
/**
|
|
274
|
+
* @internal
|
|
275
|
+
*/
|
|
252
276
|
protected static optionsShape: {
|
|
253
|
-
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
254
|
-
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
255
|
-
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
256
|
-
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
257
|
-
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
258
|
-
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
259
|
-
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
260
|
-
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
261
277
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
262
278
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
263
279
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -290,21 +306,20 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
290
306
|
}, z.core.$strict>]>>;
|
|
291
307
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
292
308
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
293
|
-
httpClient: z.ZodOptional<z.
|
|
309
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
294
310
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
295
|
-
configuration: z.ZodOptional<z.
|
|
311
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
296
312
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
297
313
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
298
|
-
eventManager: z.ZodOptional<z.
|
|
314
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
299
315
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
300
316
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
301
317
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
318
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
302
319
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
303
320
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
304
321
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
305
322
|
id: z.ZodOptional<z.ZodString>;
|
|
306
|
-
};
|
|
307
|
-
protected static optionsSchema: z.ZodObject<{
|
|
308
323
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
309
324
|
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
310
325
|
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
@@ -313,6 +328,9 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
313
328
|
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
314
329
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
315
330
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
331
|
+
};
|
|
332
|
+
/** @internal */
|
|
333
|
+
protected static optionsSchema: z.ZodObject<{
|
|
316
334
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
317
335
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
318
336
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -345,66 +363,39 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
345
363
|
}, z.core.$strict>]>>;
|
|
346
364
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
347
365
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
348
|
-
httpClient: z.ZodOptional<z.
|
|
366
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
349
367
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
350
|
-
configuration: z.ZodOptional<z.
|
|
368
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
351
369
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
352
370
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
353
|
-
eventManager: z.ZodOptional<z.
|
|
371
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
354
372
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
355
373
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
356
374
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
375
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
357
376
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
358
377
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
359
378
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
360
379
|
id: z.ZodOptional<z.ZodString>;
|
|
380
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
381
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
382
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
383
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
384
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
385
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
386
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
387
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
361
388
|
}, z.core.$strict>;
|
|
362
389
|
/**
|
|
363
390
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
364
391
|
*/
|
|
365
392
|
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
|
393
|
+
/** @internal */
|
|
366
394
|
protected getNavigationTimeoutMillis(): number;
|
|
367
|
-
/**
|
|
368
|
-
* Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
|
|
369
|
-
* concurrency shortcuts on top. Not called for a supplied
|
|
370
|
-
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
|
|
371
|
-
* keep the tuning.
|
|
372
|
-
*/
|
|
395
|
+
/** @internal */
|
|
373
396
|
protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
|
|
374
397
|
protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
|
|
375
|
-
private prepareHttpRequest;
|
|
376
398
|
private makeHttpRequest;
|
|
377
|
-
private processHttpResponse;
|
|
378
|
-
private handleBlockedRequestByContent;
|
|
379
|
-
protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
|
|
380
|
-
/**
|
|
381
|
-
* Function to make the HTTP request. It performs optimizations
|
|
382
|
-
* on the request such as only downloading the request body if the
|
|
383
|
-
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
384
|
-
*/
|
|
385
|
-
private requestFunction;
|
|
386
|
-
/**
|
|
387
|
-
* Encodes and parses response according to the provided content type
|
|
388
|
-
*/
|
|
389
|
-
private parseResponse;
|
|
390
|
-
/**
|
|
391
|
-
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
392
|
-
*/
|
|
393
|
-
private getRequestOptions;
|
|
394
|
-
private encodeResponse;
|
|
395
|
-
/**
|
|
396
|
-
* Checks and extends supported mime types
|
|
397
|
-
*/
|
|
398
|
-
private extendSupportedMimeTypes;
|
|
399
|
-
/**
|
|
400
|
-
* Handles timeout request
|
|
401
|
-
*/
|
|
402
|
-
private handleRequestTimeout;
|
|
403
|
-
private abortDownloadOfBody;
|
|
404
|
-
/**
|
|
405
|
-
* @internal wraps public utility for mocking purposes
|
|
406
|
-
*/
|
|
407
|
-
private requestAsBrowser;
|
|
408
399
|
}
|
|
409
400
|
/**
|
|
410
401
|
* Creates new {@link Router} instance that works based on request labels.
|