@crawlee/http 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,13 @@
1
1
  import { Readable } from 'node:stream';
2
2
  import util from 'node:util';
3
- import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
4
- import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
3
+ import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
5
4
  import { ResponseWithUrl } from '@crawlee/http-client';
6
- import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
7
- import contentTypeParser from 'content-type';
5
+ import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
6
+ import { isTypeValid, parse as parseContentType } from 'content-type';
8
7
  import iconv from 'iconv-lite';
9
8
  import { z } from 'zod';
10
9
  import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
11
- import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
10
+ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
12
11
  /**
13
12
  * Default mime types, which HttpScraper supports.
14
13
  */
@@ -16,7 +15,8 @@ const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml
16
15
  const APPLICATION_JSON_MIME_TYPE = 'application/json';
17
16
  /**
18
17
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
19
- * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
18
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
19
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
20
20
  *
21
21
  * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
22
22
  * wholesale, tuning included, so spread these options in if you want to keep it:
@@ -117,10 +117,14 @@ export class HttpCrawler extends BasicCrawler {
117
117
  #suggestResponseEncoding;
118
118
  #forceResponseEncoding;
119
119
  #supportedMimeTypes;
120
+ #downloadAbortControllers = new WeakMap();
121
+ /**
122
+ * @internal
123
+ */
120
124
  static optionsShape = {
121
125
  ...BasicCrawler.optionsShape,
122
126
  navigationTimeoutSecs: schemas.anyNumber.default(30),
123
- ignoreTlsErrors: z.boolean().default(true),
127
+ ignoreTlsErrors: z.boolean().default(false),
124
128
  additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
125
129
  suggestResponseEncoding: z.string().optional(),
126
130
  forceResponseEncoding: z.string().optional(),
@@ -128,6 +132,7 @@ export class HttpCrawler extends BasicCrawler {
128
132
  preNavigationHooks: schemas.anyArray.default(() => []),
129
133
  postNavigationHooks: schemas.anyArray.default(() => []),
130
134
  };
135
+ /** @internal */
131
136
  static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
132
137
  /**
133
138
  * All `HttpCrawlerOptions` parameters are passed via an options object.
@@ -143,7 +148,7 @@ export class HttpCrawler extends BasicCrawler {
143
148
  });
144
149
  this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
145
150
  if (additionalMimeTypes.length)
146
- this.extendSupportedMimeTypes(additionalMimeTypes);
151
+ this.#extendSupportedMimeTypes(additionalMimeTypes);
147
152
  if (suggestResponseEncoding && forceResponseEncoding) {
148
153
  this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
149
154
  }
@@ -156,20 +161,16 @@ export class HttpCrawler extends BasicCrawler {
156
161
  // members `extendContext` added at runtime.
157
162
  this.#preNavigationHooks = preNavigationHooks;
158
163
  this.#postNavigationHooks = [
159
- ({ request, response }) => this.abortDownloadOfBody(request, response),
164
+ ({ request, response }) => this.#abortDownloadOfBody(request, response),
160
165
  ...postNavigationHooks,
161
166
  ];
162
167
  this.#saveResponseCookies = saveResponseCookies;
163
168
  }
169
+ /** @internal */
164
170
  getNavigationTimeoutMillis() {
165
171
  return this.#navigationTimeoutMillis;
166
172
  }
167
- /**
168
- * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
169
- * concurrency shortcuts on top. Not called for a supplied
170
- * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
171
- * keep the tuning.
172
- */
173
+ /** @internal */
173
174
  createDefaultConcurrencySystem(options) {
174
175
  return super.createDefaultConcurrencySystem({
175
176
  ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
@@ -179,9 +180,7 @@ export class HttpCrawler extends BasicCrawler {
179
180
  buildContextPipeline() {
180
181
  // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
181
182
  // the response-derived members, so the guarded action is bypassed and the context left untouched.
182
- const skipGuard = (action) => ({
183
- action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
184
- });
183
+ const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
185
184
  // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
186
185
  // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
187
186
  // window the navigation uses instead of each step being timed on its own.
@@ -193,9 +192,7 @@ export class HttpCrawler extends BasicCrawler {
193
192
  }
194
193
  return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
195
194
  });
196
- let pipeline = ContextPipeline.create().compose({
197
- action: this.prepareHttpRequest.bind(this),
198
- });
195
+ let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
199
196
  for (const hook of this.#preNavigationHooks) {
200
197
  pipeline = pipeline.compose(windowGuard(hook));
201
198
  }
@@ -204,21 +201,14 @@ export class HttpCrawler extends BasicCrawler {
204
201
  pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
205
202
  }
206
203
  return pipelineWithNavigation
207
- .compose({ action: this.processHttpResponse.bind(this) })
208
- .compose({ action: this.handleBlockedRequestByContent.bind(this) });
204
+ .compose(this.#processHttpResponse.bind(this))
205
+ .compose(this.#handleBlockedRequestByContent.bind(this));
209
206
  }
210
- async prepareHttpRequest(crawlingContext) {
207
+ async #prepareHttpRequest(crawlingContext) {
211
208
  const { request } = crawlingContext;
212
209
  if (request.skipNavigation) {
213
210
  return {
214
- request: new Proxy(request, {
215
- get(target, propertyName, receiver) {
216
- if (propertyName === 'loadedUrl') {
217
- throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
218
- }
219
- return Reflect.get(target, propertyName, receiver);
220
- },
221
- }),
211
+ request: skippedNavigationRequest(request),
222
212
  get response() {
223
213
  throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
224
214
  },
@@ -227,20 +217,23 @@ export class HttpCrawler extends BasicCrawler {
227
217
  request.state = RequestState.BEFORE_NAV;
228
218
  return {};
229
219
  }
220
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
230
221
  async makeHttpRequest(crawlingContext) {
231
222
  tryCancel();
232
223
  const { request, session } = crawlingContext;
233
224
  const proxyUrl = crawlingContext.proxyInfo?.url;
225
+ const downloadAbortController = new AbortController();
226
+ this.#downloadAbortControllers.set(request, downloadAbortController);
234
227
  // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
235
228
  // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
236
229
  // client abort.
237
- const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
230
+ const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
238
231
  tryCancel();
239
232
  request.loadedUrl = httpResponse?.url;
240
233
  request.state = RequestState.AFTER_NAV;
241
234
  return { request: request, response: httpResponse };
242
235
  }
243
- async processHttpResponse(crawlingContext) {
236
+ async #processHttpResponse(crawlingContext) {
244
237
  if (crawlingContext.request.skipNavigation) {
245
238
  return {
246
239
  get contentType() {
@@ -258,6 +251,12 @@ export class HttpCrawler extends BasicCrawler {
258
251
  get parseWithCheerio() {
259
252
  throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
260
253
  },
254
+ get querySelectorAll() {
255
+ throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
256
+ },
257
+ get querySelectorOne() {
258
+ throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
259
+ },
261
260
  };
262
261
  }
263
262
  tryCancel();
@@ -279,20 +278,31 @@ export class HttpCrawler extends BasicCrawler {
279
278
  if (remaining <= 0) {
280
279
  throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
281
280
  }
282
- const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
281
+ const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
282
+ // Stop the download along with the read, so a body that never ends doesn't outlive the request.
283
+ this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
284
+ throw error;
285
+ });
283
286
  tryCancel();
284
287
  const response = parsed.response;
285
288
  const contentType = parsed.contentType;
286
- const waitForSelector = async (selector, _timeoutMs) => {
287
- const cheerio = await import('cheerio');
288
- const $ = cheerio.load(parsed.body.toString());
289
- if ($(selector).get().length === 0) {
289
+ const loadBody = async () => {
290
+ const { load } = await import('cheerio/slim');
291
+ return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
292
+ };
293
+ const querySelectorAll = async (selector, _timeoutMs) => {
294
+ const matches = (await loadBody())(selector);
295
+ if (matches.length === 0) {
290
296
  throw new Error(`Selector '${selector}' not found.`);
291
297
  }
298
+ return matches;
299
+ };
300
+ const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
301
+ const waitForSelector = async (selector, timeoutMs) => {
302
+ await querySelectorAll(selector, timeoutMs);
292
303
  };
293
304
  const parseWithCheerio = async (selector, timeoutMs) => {
294
- const cheerio = await import('cheerio');
295
- const $ = cheerio.load(parsed.body.toString());
305
+ const $ = await loadBody();
296
306
  if (selector) {
297
307
  await crawlingContext.waitForSelector(selector, timeoutMs);
298
308
  }
@@ -326,20 +336,22 @@ export class HttpCrawler extends BasicCrawler {
326
336
  return JSON.parse(jsonString);
327
337
  },
328
338
  waitForSelector,
339
+ querySelectorOne,
340
+ querySelectorAll,
329
341
  parseWithCheerio,
330
342
  contentType,
331
343
  body: parsed.body,
332
344
  };
333
345
  }
334
- async handleBlockedRequestByContent(crawlingContext) {
346
+ async #handleBlockedRequestByContent(crawlingContext) {
335
347
  if (this.retryOnBlocked) {
336
- const error = await this.isRequestBlocked(crawlingContext);
348
+ const error = await this.#isRequestBlocked(crawlingContext);
337
349
  if (error)
338
350
  throw new SessionError(error);
339
351
  }
340
352
  return {};
341
353
  }
342
- async isRequestBlocked(crawlingContext) {
354
+ async #isRequestBlocked(crawlingContext) {
343
355
  if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
344
356
  const $ = await crawlingContext.parseWithCheerio();
345
357
  const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
@@ -357,14 +369,14 @@ export class HttpCrawler extends BasicCrawler {
357
369
  * on the request such as only downloading the request body if the
358
370
  * received content type matches text/html, application/xml, application/xhtml+xml.
359
371
  */
360
- async requestFunction({ request, session, proxyUrl }) {
361
- const opts = this.getRequestOptions(request, session, proxyUrl);
372
+ async #requestFunction({ request, session, proxyUrl, signal }) {
373
+ const opts = this.#getRequestOptions(request, proxyUrl);
362
374
  try {
363
- return await this.requestAsBrowser(opts, session);
375
+ return await this.#requestAsBrowser(opts, session, signal);
364
376
  }
365
377
  catch (e) {
366
378
  if (e instanceof Error && e.constructor.name === 'TimeoutError') {
367
- this.handleRequestTimeout(session);
379
+ this.#handleRequestTimeout(session);
368
380
  return new Response(); // this will never happen, as handleRequestTimeout always throws
369
381
  }
370
382
  if (this.isProxyError(e)) {
@@ -378,16 +390,15 @@ export class HttpCrawler extends BasicCrawler {
378
390
  /**
379
391
  * Encodes and parses response according to the provided content type
380
392
  */
381
- async parseResponse(request, response) {
393
+ async #parseResponse(request, response) {
382
394
  const { status } = response;
383
395
  const { type, charset } = parseContentTypeFromResponse(response);
384
- const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
385
- const contentType = { type, encoding };
386
396
  if (status >= 400 && status <= 599) {
387
397
  this.statistics.registerStatusCode(status);
388
398
  }
389
399
  if (this.isErrorStatusCode(status)) {
390
- const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
400
+ const { response: decoded } = this.#encodeResponse(request, response, charset);
401
+ const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
391
402
  // Errors are often sent as JSON, so attempt to parse them,
392
403
  // despite Accept header being set to text/html.
393
404
  if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -403,37 +414,37 @@ export class HttpCrawler extends BasicCrawler {
403
414
  // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
404
415
  throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
405
416
  }
406
- else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
407
- if (!charset && !this.#forceResponseEncoding) {
408
- const rawBytes = Buffer.from(await response.arrayBuffer());
409
- const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
410
- const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
411
- const body = iconv.encodingExists(charsetToUse)
412
- ? iconv.decode(rawBytes, charsetToUse)
413
- : rawBytes.toString('utf8');
414
- return { response, contentType: { type, encoding: 'utf-8' }, body };
415
- }
416
- return { response, contentType, body: await reencodedResponse.text() };
417
+ if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
418
+ // The charset comes from the document itself, so the raw bytes are what we need -
419
+ // decoding them through `encodeResponse` first would consume the body for nothing.
420
+ const rawBytes = Buffer.from(await response.arrayBuffer());
421
+ const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
422
+ const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
423
+ const body = iconv.encodingExists(charsetToUse)
424
+ ? iconv.decode(rawBytes, charsetToUse)
425
+ : rawBytes.toString('utf8');
426
+ return { response, contentType: { type, encoding: 'utf-8' }, body };
417
427
  }
418
- else {
419
- const body = Buffer.from(await reencodedResponse.bytes());
420
- return {
421
- body,
422
- response,
423
- contentType,
424
- };
428
+ const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
429
+ const contentType = { type, encoding };
430
+ if (HTML_AND_XML_MIME_TYPES.includes(type)) {
431
+ return { response, contentType, body: await reencodedResponse.text() };
425
432
  }
433
+ return {
434
+ body: Buffer.from(await reencodedResponse.bytes()),
435
+ response,
436
+ contentType,
437
+ };
426
438
  }
427
439
  /**
428
440
  * Combines the provided `requestOptions` with mandatory (non-overridable) values.
429
441
  */
430
- getRequestOptions(request, session, proxyUrl) {
442
+ #getRequestOptions(request, proxyUrl) {
431
443
  const requestOptions = {
432
444
  url: request.url,
433
445
  method: request.method,
434
446
  proxyUrl,
435
447
  timeout: this.#navigationTimeoutMillis,
436
- sessionToken: session,
437
448
  headers: request.headers,
438
449
  body: undefined,
439
450
  };
@@ -445,7 +456,7 @@ export class HttpCrawler extends BasicCrawler {
445
456
  requestOptions.body = request.payload ?? '';
446
457
  return requestOptions;
447
458
  }
448
- encodeResponse(request, response, encoding) {
459
+ #encodeResponse(request, response, encoding) {
449
460
  if (this.#forceResponseEncoding) {
450
461
  encoding = this.#forceResponseEncoding;
451
462
  }
@@ -481,29 +492,27 @@ export class HttpCrawler extends BasicCrawler {
481
492
  /**
482
493
  * Checks and extends supported mime types
483
494
  */
484
- extendSupportedMimeTypes(additionalMimeTypes) {
495
+ #extendSupportedMimeTypes(additionalMimeTypes) {
485
496
  for (const mimeType of additionalMimeTypes) {
486
497
  if (mimeType === '*/*') {
487
498
  this.#supportedMimeTypes.add(mimeType);
488
499
  continue;
489
500
  }
490
- try {
491
- const parsedType = contentTypeParser.parse(mimeType);
492
- this.#supportedMimeTypes.add(parsedType.type);
493
- }
494
- catch (err) {
501
+ const parsedType = parseContentType(mimeType);
502
+ if (!isTypeValid(parsedType.type)) {
495
503
  throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
496
504
  }
505
+ this.#supportedMimeTypes.add(parsedType.type);
497
506
  }
498
507
  }
499
508
  /**
500
509
  * Handles timeout request
501
510
  */
502
- handleRequestTimeout(session) {
511
+ #handleRequestTimeout(session) {
503
512
  session.markBad();
504
513
  throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
505
514
  }
506
- abortDownloadOfBody(request, response) {
515
+ #abortDownloadOfBody(request, response) {
507
516
  const { status } = response;
508
517
  const { type } = parseContentTypeFromResponse(response);
509
518
  const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
@@ -516,7 +525,7 @@ export class HttpCrawler extends BasicCrawler {
516
525
  /**
517
526
  * @internal wraps public utility for mocking purposes
518
527
  */
519
- requestAsBrowser = async (options, session) => {
528
+ #requestAsBrowser = async (options, session, signal) => {
520
529
  const opts = processHttpRequestOptions({
521
530
  ...options,
522
531
  responseType: 'text',
@@ -527,10 +536,11 @@ export class HttpCrawler extends BasicCrawler {
527
536
  const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
528
537
  // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
529
538
  // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
530
- // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
531
- // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
532
- // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
539
+ // kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
540
+ // bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
541
+ // body still fails cleanly with a navigation timeout and its download stops too.
533
542
  const cancelSignal = storage.getStore()?.cancelTask.signal;
543
+ const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
534
544
  const response = await this.httpClient.sendRequest(new Request(opts.url, {
535
545
  body: opts.body ? Readable.toWeb(opts.body) : undefined,
536
546
  headers: new Headers(opts.headers),
@@ -540,7 +550,8 @@ export class HttpCrawler extends BasicCrawler {
540
550
  }), {
541
551
  session,
542
552
  cookieJar,
543
- signal: cancelSignal,
553
+ proxyUrl: opts.proxyUrl,
554
+ signal: requestSignal,
544
555
  timeoutMillis: cancelSignal ? undefined : opts.timeout,
545
556
  ignoreTlsErrors: this.#ignoreTlsErrors,
546
557
  });
@@ -1,8 +1,3 @@
1
- import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
- /**
3
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
- */
5
- export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
6
1
  /**
7
2
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
8
3
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
@@ -1,44 +1,9 @@
1
1
  import { extname } from 'node:path';
2
- import { Readable } from 'node:stream';
3
- import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
4
- import contentTypeParser from 'content-type';
2
+ import { parseArgument } from '@crawlee/utils/internal';
3
+ import { isTypeValid, parse as parseContentType } from 'content-type';
5
4
  import mime from 'mime-types';
6
5
  import { z } from 'zod';
7
6
  const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
8
- /**
9
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
10
- */
11
- export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
12
- const url = new URL(request.url);
13
- const headers = new Headers(request.headers);
14
- applySearchParams(url, searchParams);
15
- if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
16
- throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
17
- }
18
- const body = (() => {
19
- if (form !== undefined) {
20
- return Readable.from(new URLSearchParams(form).toString());
21
- }
22
- if (json !== undefined) {
23
- return Readable.from(JSON.stringify(json));
24
- }
25
- if (request.body !== undefined) {
26
- return Readable.from(request.body);
27
- }
28
- return undefined;
29
- })();
30
- if (form !== undefined && !headers.has('content-type')) {
31
- headers.set('content-type', 'application/x-www-form-urlencoded');
32
- }
33
- if (json !== undefined && !headers.has('content-type')) {
34
- headers.set('content-type', 'application/json');
35
- }
36
- if (username !== undefined || password !== undefined) {
37
- const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
38
- headers.set('authorization', `Basic ${encodedAuth}`);
39
- }
40
- return { ...request, body, url, headers };
41
- }
42
7
  /**
43
8
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
44
9
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
@@ -59,18 +24,16 @@ export function parseContentTypeFromResponse(response) {
59
24
  const { url, headers } = response;
60
25
  let parsedContentType;
61
26
  if (headers.get('content-type')) {
62
- try {
63
- parsedContentType = contentTypeParser.parse(headers.get('content-type'));
64
- }
65
- catch {
66
- // Can not parse content type from Content-Type header. Try to parse it from file extension.
67
- }
27
+ const parsed = parseContentType(headers.get('content-type'));
28
+ // Can not parse content type from Content-Type header. Try to parse it from file extension.
29
+ if (isTypeValid(parsed.type))
30
+ parsedContentType = parsed;
68
31
  }
69
32
  // Parse content type from file extension as fallback
70
33
  if (!parsedContentType) {
71
34
  const parsedUrl = new URL(url);
72
35
  const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
73
- parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
36
+ parsedContentType = parseContentType(contentTypeFromExtname);
74
37
  }
75
38
  return {
76
39
  type: parsedContentType.type,
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-rc.0",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -47,21 +47,19 @@
47
47
  "access": "public"
48
48
  },
49
49
  "dependencies": {
50
- "@apify/timeout": "^0.4.4",
51
- "@apify/utilities": "^2.15.5",
52
- "@crawlee/basic": "4.0.0-rc.0",
53
- "@crawlee/core": "4.0.0-rc.0",
54
- "@crawlee/http-client": "4.0.0-rc.0",
55
- "@crawlee/types": "4.0.0-rc.0",
56
- "@crawlee/utils": "4.0.0-rc.0",
57
- "@types/content-type": "^1.1.8",
58
- "cheerio": "^1.0.0",
59
- "content-type": "^1.0.5",
60
- "iconv-lite": "^0.7.2",
61
- "mime-types": "^3.0.1",
62
- "tslib": "^2.8.1",
50
+ "@apify/timeout": "^1.0.1",
51
+ "@crawlee/basic": "4.0.0-rc.2",
52
+ "@crawlee/http-client": "4.0.0-rc.2",
53
+ "@crawlee/types": "4.0.0-rc.2",
54
+ "@crawlee/utils": "4.0.0-rc.2",
55
+ "cheerio": "^1.2.0",
56
+ "content-type": "^3.1.1",
57
+ "domhandler": "^6.0.1",
58
+ "iconv-lite": "^0.7.3",
59
+ "mime-types": "^3.0.2",
60
+ "tslib": "^2.9.1",
63
61
  "type-fest": "^4.41.0",
64
- "zod": "^4.4.3"
62
+ "zod": "^4.6.7"
65
63
  },
66
64
  "lerna": {
67
65
  "command": {
@@ -70,5 +68,5 @@
70
68
  }
71
69
  }
72
70
  },
73
- "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
71
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
74
72
  }