@crawlee/http 4.0.0-beta.26 → 4.0.0-beta.261

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,27 +1,38 @@
1
1
  import { Readable } from 'node:stream';
2
2
  import util from 'node:util';
3
- import { BasicCrawler, BLOCKED_STATUS_CODES, Configuration, ContextPipeline, mergeCookies, RequestState, Router, SessionError, } from '@crawlee/basic';
3
+ import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
4
4
  import { ResponseWithUrl } from '@crawlee/http-client';
5
- import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
6
- import * as cheerio from 'cheerio';
7
- import contentTypeParser from 'content-type';
5
+ import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
6
+ import { isTypeValid, parse as parseContentType } from 'content-type';
8
7
  import iconv from 'iconv-lite';
9
- import ow from 'ow';
10
- import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
11
- import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
8
+ import { z } from 'zod';
9
+ import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
10
+ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
12
11
  /**
13
12
  * Default mime types, which HttpScraper supports.
14
13
  */
15
14
  const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
16
15
  const APPLICATION_JSON_MIME_TYPE = 'application/json';
17
- const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
16
+ /**
17
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
18
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
19
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
20
+ *
21
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
22
+ * wholesale, tuning included, so spread these options in if you want to keep it:
23
+ *
24
+ * ```typescript
25
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
26
+ * ```
27
+ */
28
+ export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
18
29
  desiredConcurrency: 10,
19
- snapshotterOptions: {
20
- eventLoopSnapshotIntervalSecs: 2,
21
- maxBlockedMillis: 100,
22
- },
23
- systemStatusOptions: {
24
- maxEventLoopOverloadedRatio: 0.7,
30
+ loadSignals: {
31
+ eventLoop: {
32
+ snapshotIntervalSecs: 2,
33
+ maxBlockedMillis: 100,
34
+ overloadedRatio: 0.7,
35
+ },
25
36
  },
26
37
  };
27
38
  /**
@@ -35,13 +46,15 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
35
46
  *
36
47
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
37
48
  *
38
- * The source URLs are represented using {@link Request} objects that are fed from
39
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
40
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
49
+ * The source URLs are represented using {@link Request} objects that are fed from the
50
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
51
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
52
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
53
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
54
+ * result as `requestManager`.
41
55
  *
42
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
43
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
44
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
56
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
57
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
45
58
  *
46
59
  * The crawler finishes when there are no more {@link Request} objects to crawl.
47
60
  *
@@ -55,18 +68,18 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
55
68
  * ]
56
69
  * ```
57
70
  *
58
- * By default, this crawler only processes web pages with the `text/html`
59
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
71
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
72
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
60
73
  * and skips pages with other content types. If you want the crawler to process other content types,
61
74
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
62
75
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
63
76
  * For details, see {@link HttpCrawlerOptions.requestHandler}.
64
77
  *
65
- * New requests are only dispatched when there is enough free CPU and memory available,
66
- * using the functionality provided by the {@link AutoscaledPool} class.
67
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
68
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
69
- * {@link AutoscaledPool} options are available directly in the constructor.
78
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
79
+ * {@link ConcurrencySystem}.
80
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
81
+ * constructor, or, for finer control, by injecting a pre-configured
82
+ * {@link ConcurrencySystem|`concurrencySystem`}.
70
83
  *
71
84
  * **Example usage:**
72
85
  *
@@ -92,157 +105,228 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
92
105
  * @category Crawlers
93
106
  */
94
107
  export class HttpCrawler extends BasicCrawler {
95
- config;
96
- preNavigationHooks;
97
- postNavigationHooks;
98
- persistCookiesPerSession;
99
- navigationTimeoutMillis;
100
- ignoreSslErrors;
101
- suggestResponseEncoding;
102
- forceResponseEncoding;
103
- additionalHttpErrorStatusCodes;
104
- ignoreHttpErrorStatusCodes;
105
- supportedMimeTypes;
108
+ // Internal storage uses the base (non-extended) context types. The public option types are
109
+ // extension-aware for consumer DX, but internally the pipeline composes hooks against the
110
+ // concrete crawling context, which does not statically carry `ContextExtension`. The members
111
+ // added by `extendContext` are present at runtime regardless.
112
+ #preNavigationHooks;
113
+ #postNavigationHooks;
114
+ #saveResponseCookies;
115
+ #navigationTimeoutMillis;
116
+ #ignoreTlsErrors;
117
+ #suggestResponseEncoding;
118
+ #forceResponseEncoding;
119
+ #supportedMimeTypes;
120
+ #downloadAbortControllers = new WeakMap();
121
+ /**
122
+ * @internal
123
+ */
106
124
  static optionsShape = {
107
125
  ...BasicCrawler.optionsShape,
108
- navigationTimeoutSecs: ow.optional.number,
109
- ignoreSslErrors: ow.optional.boolean,
110
- additionalMimeTypes: ow.optional.array.ofType(ow.string),
111
- suggestResponseEncoding: ow.optional.string,
112
- forceResponseEncoding: ow.optional.string,
113
- persistCookiesPerSession: ow.optional.boolean,
114
- additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
115
- ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
116
- preNavigationHooks: ow.optional.array,
117
- postNavigationHooks: ow.optional.array,
126
+ navigationTimeoutSecs: schemas.anyNumber.default(30),
127
+ ignoreTlsErrors: z.boolean().default(false),
128
+ additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
129
+ suggestResponseEncoding: z.string().optional(),
130
+ forceResponseEncoding: z.string().optional(),
131
+ saveResponseCookies: z.boolean().default(true),
132
+ preNavigationHooks: schemas.anyArray.default(() => []),
133
+ postNavigationHooks: schemas.anyArray.default(() => []),
118
134
  };
135
+ /** @internal */
136
+ static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
119
137
  /**
120
138
  * All `HttpCrawlerOptions` parameters are passed via an options object.
121
139
  */
122
- constructor(options = {}, config = Configuration.getGlobalConfig()) {
123
- ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
124
- const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
140
+ constructor(options = {}) {
141
+ const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks,
125
142
  // BasicCrawler
126
- autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, contextPipelineBuilder, ...basicCrawlerOptions } = options;
143
+ contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
127
144
  super({
128
145
  ...basicCrawlerOptions,
129
- autoscaledPoolOptions,
130
146
  contextPipelineBuilder: contextPipelineBuilder ??
131
147
  (() => this.buildContextPipeline()),
132
- }, config);
133
- this.config = config;
134
- // Cookies should be persisted per session only if session pool is used
135
- if (!this.useSessionPool && persistCookiesPerSession) {
136
- throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
137
- }
138
- this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
148
+ });
149
+ this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
139
150
  if (additionalMimeTypes.length)
140
- this._extendSupportedMimeTypes(additionalMimeTypes);
151
+ this.#extendSupportedMimeTypes(additionalMimeTypes);
141
152
  if (suggestResponseEncoding && forceResponseEncoding) {
142
153
  this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
143
154
  }
144
- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
145
- this.ignoreSslErrors = ignoreSslErrors;
146
- this.suggestResponseEncoding = suggestResponseEncoding;
147
- this.forceResponseEncoding = forceResponseEncoding;
148
- this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
149
- this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
150
- this.preNavigationHooks = preNavigationHooks;
151
- this.postNavigationHooks = [
152
- ({ request, response }) => this._abortDownloadOfBody(request, response),
155
+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
156
+ this.#ignoreTlsErrors = ignoreTlsErrors;
157
+ this.#suggestResponseEncoding = suggestResponseEncoding;
158
+ this.#forceResponseEncoding = forceResponseEncoding;
159
+ // Cast away the extension-aware option types to the base internal storage types (see the field
160
+ // declarations above). This is sound - the hooks only ever receive the base context plus the
161
+ // members `extendContext` added at runtime.
162
+ this.#preNavigationHooks = preNavigationHooks;
163
+ this.#postNavigationHooks = [
164
+ ({ request, response }) => this.#abortDownloadOfBody(request, response),
153
165
  ...postNavigationHooks,
154
166
  ];
155
- if (this.useSessionPool) {
156
- this.persistCookiesPerSession = persistCookiesPerSession ?? true;
157
- }
158
- else {
159
- this.persistCookiesPerSession = false;
160
- }
167
+ this.#saveResponseCookies = saveResponseCookies;
168
+ }
169
+ /** @internal */
170
+ getNavigationTimeoutMillis() {
171
+ return this.#navigationTimeoutMillis;
172
+ }
173
+ /** @internal */
174
+ createDefaultConcurrencySystem(options) {
175
+ return super.createDefaultConcurrencySystem({
176
+ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
177
+ ...options,
178
+ });
161
179
  }
162
180
  buildContextPipeline() {
163
- return ContextPipeline.create()
164
- .compose({
165
- action: this.makeHttpRequest.bind(this),
166
- })
167
- .compose({ action: this.processHttpResponse.bind(this) })
168
- .compose({ action: this.handleBlockedRequestByContent.bind(this) });
181
+ // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
182
+ // the response-derived members, so the guarded action is bypassed and the context left untouched.
183
+ const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
184
+ // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
185
+ // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
186
+ // window the navigation uses instead of each step being timed on its own.
187
+ const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
188
+ const windowGuard = (step) => skipGuard(async (ctx) => {
189
+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
190
+ if (remaining <= 0) {
191
+ throw new TimeoutError(navigationTimedOut);
192
+ }
193
+ return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
194
+ });
195
+ let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
196
+ for (const hook of this.#preNavigationHooks) {
197
+ pipeline = pipeline.compose(windowGuard(hook));
198
+ }
199
+ let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
200
+ for (const hook of this.#postNavigationHooks) {
201
+ pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
202
+ }
203
+ return pipelineWithNavigation
204
+ .compose(this.#processHttpResponse.bind(this))
205
+ .compose(this.#handleBlockedRequestByContent.bind(this));
169
206
  }
170
- async makeHttpRequest(crawlingContext) {
171
- const { request, session } = crawlingContext;
207
+ async #prepareHttpRequest(crawlingContext) {
208
+ const { request } = crawlingContext;
172
209
  if (request.skipNavigation) {
173
210
  return {
174
- request: new Proxy(request, {
175
- get(target, propertyName, receiver) {
176
- if (propertyName === 'loadedUrl') {
177
- throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
178
- }
179
- return Reflect.get(target, propertyName, receiver);
180
- },
181
- }),
211
+ request: skippedNavigationRequest(request),
182
212
  get response() {
183
- throw new Error('The `response` property is not available - `skipNavigation` was used');
213
+ throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
184
214
  },
185
215
  };
186
216
  }
187
- const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
188
217
  request.state = RequestState.BEFORE_NAV;
189
- // Execute pre navigation hooks before applying session pool cookies,
190
- // as they may also set cookies in the session
191
- await this._executeHooks(this.preNavigationHooks, crawlingContext);
218
+ return {};
219
+ }
220
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
221
+ async makeHttpRequest(crawlingContext) {
192
222
  tryCancel();
193
- const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
194
- const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
223
+ const { request, session } = crawlingContext;
195
224
  const proxyUrl = crawlingContext.proxyInfo?.url;
196
- const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, cookieString }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
225
+ const downloadAbortController = new AbortController();
226
+ this.#downloadAbortControllers.set(request, downloadAbortController);
227
+ // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
228
+ // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
229
+ // client abort.
230
+ const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
197
231
  tryCancel();
198
232
  request.loadedUrl = httpResponse?.url;
199
233
  request.state = RequestState.AFTER_NAV;
200
234
  return { request: request, response: httpResponse };
201
235
  }
202
- async processHttpResponse(crawlingContext) {
236
+ async #processHttpResponse(crawlingContext) {
203
237
  if (crawlingContext.request.skipNavigation) {
204
238
  return {
205
239
  get contentType() {
206
- throw new Error('The `contentType` property is not available - `skipNavigation` was used');
240
+ throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
207
241
  },
208
242
  get body() {
209
- throw new Error('The `body` property is not available - `skipNavigation` was used');
243
+ throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
210
244
  },
211
245
  get json() {
212
- throw new Error('The `json` property is not available - `skipNavigation` was used');
246
+ throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
213
247
  },
214
248
  get waitForSelector() {
215
- throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used');
249
+ throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
216
250
  },
217
251
  get parseWithCheerio() {
218
- throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used');
252
+ throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
253
+ },
254
+ get querySelectorAll() {
255
+ throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
256
+ },
257
+ get querySelectorOne() {
258
+ throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
219
259
  },
220
260
  };
221
261
  }
222
- await this._executeHooks(this.postNavigationHooks, crawlingContext);
223
262
  tryCancel();
224
- const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
263
+ // Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
264
+ // error is still a rate limit the domain should back off from.
265
+ if (crawlingContext.response.status === 429) {
266
+ const retryAfter = crawlingContext.response.headers.get('retry-after');
267
+ if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
268
+ // This is the one path that never reads the body, so cancel it to release the connection
269
+ // rather than leaving it to the garbage collector.
270
+ await crawlingContext.response.body?.cancel().catch(() => { });
271
+ throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
272
+ }
273
+ }
274
+ // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
275
+ // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
276
+ // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
277
+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
278
+ if (remaining <= 0) {
279
+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
280
+ }
281
+ const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
282
+ // Stop the download along with the read, so a body that never ends doesn't outlive the request.
283
+ this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
284
+ throw error;
285
+ });
225
286
  tryCancel();
226
287
  const response = parsed.response;
227
288
  const contentType = parsed.contentType;
228
- const waitForSelector = async (selector, _timeoutMs) => {
229
- const $ = cheerio.load(parsed.body.toString());
230
- if ($(selector).get().length === 0) {
289
+ const loadBody = async () => {
290
+ const { load } = await import('cheerio/slim');
291
+ return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
292
+ };
293
+ const querySelectorAll = async (selector, _timeoutMs) => {
294
+ const matches = (await loadBody())(selector);
295
+ if (matches.length === 0) {
231
296
  throw new Error(`Selector '${selector}' not found.`);
232
297
  }
298
+ return matches;
299
+ };
300
+ const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
301
+ const waitForSelector = async (selector, timeoutMs) => {
302
+ await querySelectorAll(selector, timeoutMs);
233
303
  };
234
304
  const parseWithCheerio = async (selector, timeoutMs) => {
235
- const $ = cheerio.load(parsed.body.toString());
305
+ const $ = await loadBody();
236
306
  if (selector) {
237
307
  await crawlingContext.waitForSelector(selector, timeoutMs);
238
308
  }
239
309
  return $;
240
310
  };
241
- if (this.useSessionPool) {
242
- this._throwOnBlockedRequest(crawlingContext.session, response.status);
243
- }
244
- if (this.persistCookiesPerSession) {
245
- crawlingContext.session.setCookiesFromResponse(response);
311
+ this.throwOnBlockedRequest(response.status);
312
+ if (this.#saveResponseCookies) {
313
+ try {
314
+ for (const cookie of getCookiesFromResponse(response)) {
315
+ if (!cookie)
316
+ continue;
317
+ try {
318
+ await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
319
+ ignoreError: false,
320
+ });
321
+ }
322
+ catch (e) {
323
+ this.log.debug(`Could not set cookie: ${e.message}`);
324
+ }
325
+ }
326
+ }
327
+ catch (e) {
328
+ this.log.exception(e, 'Could not get cookies from response');
329
+ }
246
330
  }
247
331
  return {
248
332
  get json() {
@@ -252,20 +336,22 @@ export class HttpCrawler extends BasicCrawler {
252
336
  return JSON.parse(jsonString);
253
337
  },
254
338
  waitForSelector,
339
+ querySelectorOne,
340
+ querySelectorAll,
255
341
  parseWithCheerio,
256
342
  contentType,
257
343
  body: parsed.body,
258
344
  };
259
345
  }
260
- async handleBlockedRequestByContent(crawlingContext) {
346
+ async #handleBlockedRequestByContent(crawlingContext) {
261
347
  if (this.retryOnBlocked) {
262
- const error = await this.isRequestBlocked(crawlingContext);
348
+ const error = await this.#isRequestBlocked(crawlingContext);
263
349
  if (error)
264
350
  throw new SessionError(error);
265
351
  }
266
352
  return {};
267
353
  }
268
- async isRequestBlocked(crawlingContext) {
354
+ async #isRequestBlocked(crawlingContext) {
269
355
  if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
270
356
  const $ = await crawlingContext.parseWithCheerio();
271
357
  const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
@@ -273,43 +359,28 @@ export class HttpCrawler extends BasicCrawler {
273
359
  return `Found selectors: ${foundSelectors.join(', ')}`;
274
360
  }
275
361
  }
276
- const blockedStatusCodes =
277
- // eslint-disable-next-line dot-notation
278
- (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
279
- ? // eslint-disable-next-line dot-notation
280
- this.sessionPool['blockedStatusCodes']
281
- : BLOCKED_STATUS_CODES;
282
- if (blockedStatusCodes.includes(crawlingContext.response.status)) {
362
+ if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
283
363
  return `Blocked by status code ${crawlingContext.response.status}`;
284
364
  }
285
365
  return false;
286
366
  }
287
- /**
288
- * Returns the `Cookie` header value based on the current context and
289
- * any changes that occurred in the navigation hooks.
290
- */
291
- _applyCookies({ session, request }, preHookCookies, postHookCookies) {
292
- const sessionCookie = session?.getCookieString(request.url) ?? '';
293
- const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
294
- return mergeCookies(request.url, sourceCookies);
295
- }
296
367
  /**
297
368
  * Function to make the HTTP request. It performs optimizations
298
369
  * on the request such as only downloading the request body if the
299
370
  * received content type matches text/html, application/xml, application/xhtml+xml.
300
371
  */
301
- async _requestFunction({ request, session, proxyUrl, cookieString, }) {
302
- const opts = this._getRequestOptions(request, session, proxyUrl);
372
+ async #requestFunction({ request, session, proxyUrl, signal }) {
373
+ const opts = this.#getRequestOptions(request, proxyUrl);
303
374
  try {
304
- return await this._requestAsBrowser(opts, session, cookieString);
375
+ return await this.#requestAsBrowser(opts, session, signal);
305
376
  }
306
377
  catch (e) {
307
378
  if (e instanceof Error && e.constructor.name === 'TimeoutError') {
308
- this._handleRequestTimeout(session);
309
- return new Response(); // this will never happen, as _handleRequestTimeout always throws
379
+ this.#handleRequestTimeout(session);
380
+ return new Response(); // this will never happen, as handleRequestTimeout always throws
310
381
  }
311
382
  if (this.isProxyError(e)) {
312
- throw new SessionError(this._getMessageFromError(e));
383
+ throw new SessionError(this.getMessageFromError(e));
313
384
  }
314
385
  else {
315
386
  throw e;
@@ -319,18 +390,15 @@ export class HttpCrawler extends BasicCrawler {
319
390
  /**
320
391
  * Encodes and parses response according to the provided content type
321
392
  */
322
- async _parseResponse(request, response) {
393
+ async #parseResponse(request, response) {
323
394
  const { status } = response;
324
395
  const { type, charset } = parseContentTypeFromResponse(response);
325
- const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
326
- const contentType = { type, encoding };
327
396
  if (status >= 400 && status <= 599) {
328
- this.stats.registerStatusCode(status);
397
+ this.statistics.registerStatusCode(status);
329
398
  }
330
- const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
331
- const includeError = this.additionalHttpErrorStatusCodes.has(status);
332
- if ((status >= 500 && !excludeError) || includeError) {
333
- const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
399
+ if (this.isErrorStatusCode(status)) {
400
+ const { response: decoded } = this.#encodeResponse(request, response, charset);
401
+ const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
334
402
  // Errors are often sent as JSON, so attempt to parse them,
335
403
  // despite Accept header being set to text/html.
336
404
  if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -340,60 +408,60 @@ export class HttpCrawler extends BasicCrawler {
340
408
  message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
341
409
  throw new Error(`${status} - ${message}`);
342
410
  }
343
- if (includeError) {
411
+ if (this.additionalHttpErrorStatusCodes.has(status)) {
344
412
  throw new Error(`${status} - Error status code was set by user.`);
345
413
  }
346
414
  // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
347
415
  throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
348
416
  }
349
- else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
350
- return { response, contentType, body: await reencodedResponse.text() };
417
+ if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
418
+ // The charset comes from the document itself, so the raw bytes are what we need -
419
+ // decoding them through `encodeResponse` first would consume the body for nothing.
420
+ const rawBytes = Buffer.from(await response.arrayBuffer());
421
+ const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
422
+ const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
423
+ const body = iconv.encodingExists(charsetToUse)
424
+ ? iconv.decode(rawBytes, charsetToUse)
425
+ : rawBytes.toString('utf8');
426
+ return { response, contentType: { type, encoding: 'utf-8' }, body };
351
427
  }
352
- else {
353
- const body = Buffer.from(await reencodedResponse.bytes());
354
- return {
355
- body,
356
- response,
357
- contentType,
358
- };
428
+ const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
429
+ const contentType = { type, encoding };
430
+ if (HTML_AND_XML_MIME_TYPES.includes(type)) {
431
+ return { response, contentType, body: await reencodedResponse.text() };
359
432
  }
433
+ return {
434
+ body: Buffer.from(await reencodedResponse.bytes()),
435
+ response,
436
+ contentType,
437
+ };
360
438
  }
361
439
  /**
362
440
  * Combines the provided `requestOptions` with mandatory (non-overridable) values.
363
441
  */
364
- _getRequestOptions(request, session, proxyUrl) {
442
+ #getRequestOptions(request, proxyUrl) {
365
443
  const requestOptions = {
366
444
  url: request.url,
367
445
  method: request.method,
368
446
  proxyUrl,
369
- timeout: this.navigationTimeoutMillis,
370
- cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
371
- sessionToken: session,
447
+ timeout: this.#navigationTimeoutMillis,
372
448
  headers: request.headers,
373
- https: {
374
- rejectUnauthorized: !this.ignoreSslErrors,
375
- },
376
449
  body: undefined,
377
450
  };
378
- // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
379
- Reflect.deleteProperty(requestOptions.headers, 'cookie');
380
- // Disable SSL verification for MITM proxies
381
- if (session?.proxyInfo?.ignoreTlsErrors) {
382
- requestOptions.https = {
383
- ...requestOptions.https,
384
- rejectUnauthorized: false,
385
- };
451
+ if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
452
+ requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
453
+ delete requestOptions.headers.cookie;
386
454
  }
387
455
  if (/PATCH|POST|PUT/.test(request.method))
388
456
  requestOptions.body = request.payload ?? '';
389
457
  return requestOptions;
390
458
  }
391
- _encodeResponse(request, response, encoding) {
392
- if (this.forceResponseEncoding) {
393
- encoding = this.forceResponseEncoding;
459
+ #encodeResponse(request, response, encoding) {
460
+ if (this.#forceResponseEncoding) {
461
+ encoding = this.#forceResponseEncoding;
394
462
  }
395
- else if (!encoding && this.suggestResponseEncoding) {
396
- encoding = this.suggestResponseEncoding;
463
+ else if (!encoding && this.#suggestResponseEncoding) {
464
+ encoding = this.#suggestResponseEncoding;
397
465
  }
398
466
  // Fall back to utf-8 if we still don't have encoding.
399
467
  const utf8 = 'utf8';
@@ -406,7 +474,9 @@ export class HttpCrawler extends BasicCrawler {
406
474
  // Try to re-encode a variety of unsupported encodings to utf-8
407
475
  if (iconv.encodingExists(encoding)) {
408
476
  const encodeStream = iconv.encodeStream(utf8);
409
- const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
477
+ const decodeStream = iconv
478
+ .decodeStream(encoding)
479
+ .on('error', (err) => encodeStream.emit('error', err));
410
480
  const reencodedBody = response.body
411
481
  ? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
412
482
  .pipe(decodeStream)
@@ -422,55 +492,55 @@ export class HttpCrawler extends BasicCrawler {
422
492
  /**
423
493
  * Checks and extends supported mime types
424
494
  */
425
- _extendSupportedMimeTypes(additionalMimeTypes) {
495
+ #extendSupportedMimeTypes(additionalMimeTypes) {
426
496
  for (const mimeType of additionalMimeTypes) {
427
497
  if (mimeType === '*/*') {
428
- this.supportedMimeTypes.add(mimeType);
498
+ this.#supportedMimeTypes.add(mimeType);
429
499
  continue;
430
500
  }
431
- try {
432
- const parsedType = contentTypeParser.parse(mimeType);
433
- this.supportedMimeTypes.add(parsedType.type);
434
- }
435
- catch (err) {
501
+ const parsedType = parseContentType(mimeType);
502
+ if (!isTypeValid(parsedType.type)) {
436
503
  throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
437
504
  }
505
+ this.#supportedMimeTypes.add(parsedType.type);
438
506
  }
439
507
  }
440
508
  /**
441
509
  * Handles timeout request
442
510
  */
443
- _handleRequestTimeout(session) {
444
- session?.markBad();
445
- throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
511
+ #handleRequestTimeout(session) {
512
+ session.markBad();
513
+ throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
446
514
  }
447
- _abortDownloadOfBody(request, response) {
515
+ #abortDownloadOfBody(request, response) {
448
516
  const { status } = response;
449
517
  const { type } = parseContentTypeFromResponse(response);
450
- // eslint-disable-next-line dot-notation -- accessing private property
451
- const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
452
- // if we retry the request, can the Content-Type change?
453
- const isTransientContentType = status >= 500 || blockedStatusCodes.includes(status);
454
- if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
518
+ const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
519
+ if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
455
520
  request.noRetry = true;
456
521
  throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
457
- `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
522
+ `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
458
523
  }
459
524
  }
460
525
  /**
461
526
  * @internal wraps public utility for mocking purposes
462
527
  */
463
- _requestAsBrowser = async (options, session, cookieString) => {
528
+ #requestAsBrowser = async (options, session, signal) => {
464
529
  const opts = processHttpRequestOptions({
465
530
  ...options,
466
- cookieJar: options.cookieJar,
467
531
  responseType: 'text',
468
532
  });
469
- if (cookieString) {
470
- opts.headers?.delete('Cookie');
471
- opts.headers?.delete('cookie');
472
- opts.headers?.set('Cookie', cookieString);
473
- }
533
+ // When saveResponseCookies is false, the response cookies must not mutate the
534
+ // session jar. Reads still go through the session (so session.setCookie() in pre-nav
535
+ // hooks keeps working) but a per-request clone is passed in so writes are discarded.
536
+ const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
537
+ // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
538
+ // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
539
+ // kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
540
+ // bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
541
+ // body still fails cleanly with a navigation timeout and its download stops too.
542
+ const cancelSignal = storage.getStore()?.cancelTask.signal;
543
+ const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
474
544
  const response = await this.httpClient.sendRequest(new Request(opts.url, {
475
545
  body: opts.body ? Readable.toWeb(opts.body) : undefined,
476
546
  headers: new Headers(opts.headers),
@@ -479,36 +549,15 @@ export class HttpCrawler extends BasicCrawler {
479
549
  duplex: 'half',
480
550
  }), {
481
551
  session,
482
- timeout: opts.timeout,
552
+ cookieJar,
553
+ proxyUrl: opts.proxyUrl,
554
+ signal: requestSignal,
555
+ timeoutMillis: cancelSignal ? undefined : opts.timeout,
556
+ ignoreTlsErrors: this.#ignoreTlsErrors,
483
557
  });
484
558
  return response;
485
559
  };
486
560
  }
487
- /**
488
- * Creates new {@link Router} instance that works based on request labels.
489
- * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
490
- * Defaults to the {@link HttpCrawlingContext}.
491
- *
492
- * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
493
- *
494
- * ```ts
495
- * import { HttpCrawler, createHttpRouter } from 'crawlee';
496
- *
497
- * const router = createHttpRouter();
498
- * router.addHandler('label-a', async (ctx) => {
499
- * ctx.log.info('...');
500
- * });
501
- * router.addDefaultHandler(async (ctx) => {
502
- * ctx.log.info('...');
503
- * });
504
- *
505
- * const crawler = new HttpCrawler({
506
- * requestHandler: router,
507
- * });
508
- * await crawler.run();
509
- * ```
510
- */
511
- export function createHttpRouter(routes) {
512
- return Router.create(routes);
561
+ export function createHttpRouter(routesOrSchemas) {
562
+ return Router.create(routesOrSchemas);
513
563
  }
514
- //# sourceMappingURL=http-crawler.js.map