@crawlee/http 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,67 +1,9 @@
1
- import { Transform } from 'node:stream';
2
- import { BasicCrawler } from '@crawlee/basic';
1
+ import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
2
  import { ResponseWithUrl } from '@crawlee/http-client';
3
+ import { parseArgument } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
4
5
  import { Router } from '../index.js';
5
6
  import { parseContentTypeFromResponse } from './utils.js';
6
- const kBodyDrained = Symbol('bodyDrained');
7
- /**
8
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
9
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
10
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
11
- * it will throw an error.
12
- *
13
- * Can be used e.g. to abort a download if the network speed is too slow.
14
- * @returns Transform stream that monitors the speed of the incoming data.
15
- */
16
- export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
17
- let snapshots = [];
18
- const checkInterval = setInterval(() => {
19
- const now = Date.now();
20
- snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
21
- const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
22
- const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
23
- if (totalBytes / 1024 / elapsed < minSpeedKbps) {
24
- clearInterval(checkInterval);
25
- stream.emit('error', new Error(`Stream speed too slow, aborting...`));
26
- }
27
- }, checkProgressIntervalMs);
28
- const stream = new Transform({
29
- transform: (chunk, _, callback) => {
30
- snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
31
- callback(null, chunk);
32
- },
33
- final: (callback) => {
34
- clearInterval(checkInterval);
35
- callback();
36
- },
37
- });
38
- return stream;
39
- }
40
- /**
41
- * Creates a transform stream that logs the progress of the incoming data.
42
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
43
- *
44
- * Can be used e.g. to log the progress of a download.
45
- * @returns Transform stream logging the progress of the incoming data.
46
- */
47
- export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
48
- let transferredBytes = 0;
49
- let lastLogTime = Date.now();
50
- return new Transform({
51
- transform: (chunk, _, callback) => {
52
- transferredBytes += chunk.length;
53
- if (Date.now() - lastLogTime > loggingInterval) {
54
- lastLogTime = Date.now();
55
- logTransferredBytes(transferredBytes);
56
- }
57
- callback(null, chunk);
58
- },
59
- flush: (callback) => {
60
- logTransferredBytes(transferredBytes);
61
- callback();
62
- },
63
- });
64
- }
65
7
  /**
66
8
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
67
9
  *
@@ -69,32 +11,22 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
69
11
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
70
12
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
71
13
  *
72
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
14
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
73
15
  *
74
- * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
16
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
75
17
  *
76
- * > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
18
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
77
19
  *
78
20
  * The crawler finishes when there are no more {@link Request} objects to crawl.
79
21
  *
80
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
81
- *
82
- * ```
83
- * preNavigationHooks: [
84
- * (crawlingContext) => {
85
- * // ...
86
- * },
87
- * ]
88
- * ```
89
- *
90
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
22
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
91
23
  *
92
24
  * ## Example usage
93
25
  *
94
26
  * ```ts
95
- * const crawler = new FileDownloader({
96
- * requestHandler({ body, request }) {
97
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
27
+ * const crawler = new FileDownload({
28
+ * async requestHandler({ response, request }) {
29
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
98
30
  * },
99
31
  * });
100
32
  *
@@ -106,40 +38,49 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
106
38
  * ```
107
39
  */
108
40
  export class FileDownload extends BasicCrawler {
41
+ #ignoreTlsErrors;
42
+ /**
43
+ * @internal
44
+ */
45
+ static optionsShape = {
46
+ ...BasicCrawler.optionsShape,
47
+ ignoreTlsErrors: z.boolean().default(false),
48
+ };
49
+ /** @internal */
50
+ static optionsSchema = z.strictObject(FileDownload.optionsShape);
109
51
  // TODO hooks
110
52
  constructor(options = {}) {
53
+ const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
111
54
  super({
112
- ...options,
113
- contextPipelineBuilder: () => this.buildContextPipeline(),
55
+ ...basicCrawlerOptions,
56
+ contextPipelineBuilder: () => this.#buildContextPipeline(),
114
57
  });
58
+ this.#ignoreTlsErrors = ignoreTlsErrors;
115
59
  }
116
- buildContextPipeline() {
117
- return super.buildContextPipeline().compose({
118
- action: async (context) => this.initiateDownload(context),
119
- cleanup: async (context) => {
120
- if (!context.response.bodyUsed) {
121
- // Nobody consumed the body — cancel it so the
122
- // underlying connection can be released.
123
- await context.response.body?.cancel();
124
- }
125
- await context[kBodyDrained];
126
- },
127
- });
60
+ #buildContextPipeline() {
61
+ return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
128
62
  }
129
- async initiateDownload(context) {
63
+ async #initiateDownload(context, onCleanup) {
130
64
  const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
131
65
  session: context.session,
66
+ ignoreTlsErrors: this.#ignoreTlsErrors,
132
67
  });
133
68
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
134
69
  context.request.url = response.url;
135
70
  const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
136
- const contextExtension = {
71
+ onCleanup(async () => {
72
+ if (!trackedResponse.bodyUsed) {
73
+ // Nobody consumed the body — cancel it so the
74
+ // underlying connection can be released.
75
+ await trackedResponse.body?.cancel();
76
+ }
77
+ await bodyDrained;
78
+ });
79
+ return {
137
80
  request: context.request,
138
81
  response: trackedResponse,
139
82
  contentType: { type, encoding },
140
- [kBodyDrained]: bodyDrained,
141
83
  };
142
- return contextExtension;
143
84
  }
144
85
  }
145
86
  /**
@@ -162,30 +103,6 @@ function trackBodyConsumption(response) {
162
103
  });
163
104
  return { response: trackedResponse, bodyDrained };
164
105
  }
165
- /**
166
- * Creates new {@link Router} instance that works based on request labels.
167
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
168
- * Defaults to the {@link FileDownloadCrawlingContext}.
169
- *
170
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
171
- *
172
- * ```ts
173
- * import { FileDownload, createFileRouter } from 'crawlee';
174
- *
175
- * const router = createFileRouter();
176
- * router.addHandler('label-a', async (ctx) => {
177
- * ctx.log.info('...');
178
- * });
179
- * router.addDefaultHandler(async (ctx) => {
180
- * ctx.log.info('...');
181
- * });
182
- *
183
- * const crawler = new FileDownload({
184
- * requestHandler: router,
185
- * });
186
- * await crawler.run();
187
- * ```
188
- */
189
- export function createFileRouter(routes) {
190
- return Router.create(routes);
106
+ export function createFileRouter(routesOrSchemas) {
107
+ return Router.create(routesOrSchemas);
191
108
  }
@@ -1,13 +1,14 @@
1
- import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
1
+ import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
- import { type LoadedRequest } from '@crawlee/core';
4
3
  import type { Awaitable, Dictionary } from '@crawlee/types';
5
- import { type CheerioRoot } from '@crawlee/utils/internal';
4
+ import type { Cheerio, CheerioAPI } from 'cheerio';
5
+ import type { AnyNode } from 'domhandler';
6
6
  import type { JsonValue } from 'type-fest';
7
7
  import { z } from 'zod';
8
8
  /**
9
9
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
10
- * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
10
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
11
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
11
12
  *
12
13
  * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
13
14
  * wholesale, tuning included, so spread these options in if you want to keep it:
@@ -35,7 +36,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
35
36
  * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
36
37
  * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
37
38
  *
38
- * @default true
39
+ * @default false
39
40
  */
40
41
  ignoreTlsErrors?: boolean;
41
42
  /**
@@ -77,7 +78,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
77
78
  * ]
78
79
  * ```
79
80
  */
80
- postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
81
+ postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
81
82
  /**
82
83
  * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
83
84
  * you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
@@ -115,13 +116,11 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
115
116
  saveResponseCookies?: boolean;
116
117
  }
117
118
  export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
118
- export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
119
- JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
120
119
  interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
121
120
  /**
122
121
  * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
123
122
  */
124
- request: LoadedRequest<CrawleeRequest<UserData>>;
123
+ request: LoadedRequest<CrawlingRequest<UserData>>;
125
124
  /**
126
125
  * The HTTP response object containing status code, headers, and other response metadata.
127
126
  */
@@ -148,7 +147,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
148
147
  encoding: BufferEncoding;
149
148
  };
150
149
  /**
151
- * Wait for an element matching the selector to appear. Timeout is ignored.
150
+ * Wait for an element matching the selector to appear.
151
+ *
152
+ * `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
153
+ * nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
154
+ * the selector instead, with `timeoutMs` defaulting to 5s.
152
155
  *
153
156
  * **Example usage:**
154
157
  * ```ts
@@ -160,9 +163,25 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
160
163
  * ```
161
164
  */
162
165
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
166
+ /**
167
+ * Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
168
+ * elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
169
+ * matches.
170
+ */
171
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
172
+ /**
173
+ * Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
174
+ * match.
175
+ */
176
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
163
177
  /**
164
178
  * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
165
- * When provided with the `selector` argument, it will throw if it's not available.
179
+ * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
180
+ * the same return type as the `$` context property, so use it only if you are abstracting your workflow to
181
+ * support different context types in one handler.
182
+ *
183
+ * When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
184
+ * {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
166
185
  *
167
186
  * **Example usage:**
168
187
  * ```ts
@@ -172,9 +191,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
172
191
  * });
173
192
  * ```
174
193
  */
175
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
194
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
176
195
  }
177
196
  export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
197
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
198
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
178
199
  }
179
200
  export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
180
201
  JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
@@ -249,15 +270,10 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
249
270
  */
250
271
  export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
251
272
  #private;
273
+ /**
274
+ * @internal
275
+ */
252
276
  protected static optionsShape: {
253
- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
254
- ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
255
- additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
256
- suggestResponseEncoding: z.ZodOptional<z.ZodString>;
257
- forceResponseEncoding: z.ZodOptional<z.ZodString>;
258
- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
259
- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
260
- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
261
277
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
262
278
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
263
279
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -290,21 +306,20 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
290
306
  }, z.core.$strict>]>>;
291
307
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
292
308
  // @ts-ignore optional peer dependency or compatibility with es2022
293
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
309
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
294
310
  // @ts-ignore optional peer dependency or compatibility with es2022
295
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
311
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
296
312
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
297
313
  // @ts-ignore optional peer dependency or compatibility with es2022
298
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
314
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
299
315
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
300
316
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
301
317
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
318
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
302
319
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
303
320
  keepAlive: z.ZodOptional<z.ZodBoolean>;
304
321
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
305
322
  id: z.ZodOptional<z.ZodString>;
306
- };
307
- protected static optionsSchema: z.ZodObject<{
308
323
  navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
309
324
  ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
310
325
  additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -313,6 +328,9 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
313
328
  saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
314
329
  preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
315
330
  postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
331
+ };
332
+ /** @internal */
333
+ protected static optionsSchema: z.ZodObject<{
316
334
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
317
335
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
318
336
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -345,66 +363,39 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
345
363
  }, z.core.$strict>]>>;
346
364
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
347
365
  // @ts-ignore optional peer dependency or compatibility with es2022
348
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
366
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
349
367
  // @ts-ignore optional peer dependency or compatibility with es2022
350
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
368
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
351
369
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
352
370
  // @ts-ignore optional peer dependency or compatibility with es2022
353
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
371
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
354
372
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
355
373
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
356
374
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
375
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
357
376
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
358
377
  keepAlive: z.ZodOptional<z.ZodBoolean>;
359
378
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
360
379
  id: z.ZodOptional<z.ZodString>;
380
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
381
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
382
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
383
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
384
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
385
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
386
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
387
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
361
388
  }, z.core.$strict>;
362
389
  /**
363
390
  * All `HttpCrawlerOptions` parameters are passed via an options object.
364
391
  */
365
392
  constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
393
+ /** @internal */
366
394
  protected getNavigationTimeoutMillis(): number;
367
- /**
368
- * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
369
- * concurrency shortcuts on top. Not called for a supplied
370
- * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
371
- * keep the tuning.
372
- */
395
+ /** @internal */
373
396
  protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
374
397
  protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
375
- private prepareHttpRequest;
376
398
  private makeHttpRequest;
377
- private processHttpResponse;
378
- private handleBlockedRequestByContent;
379
- protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
380
- /**
381
- * Function to make the HTTP request. It performs optimizations
382
- * on the request such as only downloading the request body if the
383
- * received content type matches text/html, application/xml, application/xhtml+xml.
384
- */
385
- private requestFunction;
386
- /**
387
- * Encodes and parses response according to the provided content type
388
- */
389
- private parseResponse;
390
- /**
391
- * Combines the provided `requestOptions` with mandatory (non-overridable) values.
392
- */
393
- private getRequestOptions;
394
- private encodeResponse;
395
- /**
396
- * Checks and extends supported mime types
397
- */
398
- private extendSupportedMimeTypes;
399
- /**
400
- * Handles timeout request
401
- */
402
- private handleRequestTimeout;
403
- private abortDownloadOfBody;
404
- /**
405
- * @internal wraps public utility for mocking purposes
406
- */
407
- private requestAsBrowser;
408
399
  }
409
400
  /**
410
401
  * Creates new {@link Router} instance that works based on request labels.