@crawlee/http 4.0.0-beta.239 → 4.0.0-beta.240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -117,6 +117,7 @@ export class HttpCrawler extends BasicCrawler {
117
117
  #suggestResponseEncoding;
118
118
  #forceResponseEncoding;
119
119
  #supportedMimeTypes;
120
+ #downloadAbortControllers = new WeakMap();
120
121
  /**
121
122
  * @internal
122
123
  */
@@ -231,10 +232,12 @@ export class HttpCrawler extends BasicCrawler {
231
232
  tryCancel();
232
233
  const { request, session } = crawlingContext;
233
234
  const proxyUrl = crawlingContext.proxyInfo?.url;
235
+ const downloadAbortController = new AbortController();
236
+ this.#downloadAbortControllers.set(request, downloadAbortController);
234
237
  // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
235
238
  // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
236
239
  // client abort.
237
- const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
240
+ const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
238
241
  tryCancel();
239
242
  request.loadedUrl = httpResponse?.url;
240
243
  request.state = RequestState.AFTER_NAV;
@@ -285,7 +288,11 @@ export class HttpCrawler extends BasicCrawler {
285
288
  if (remaining <= 0) {
286
289
  throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
287
290
  }
288
- const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
291
+ const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
292
+ // Stop the download along with the read, so a body that never ends doesn't outlive the request.
293
+ this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
294
+ throw error;
295
+ });
289
296
  tryCancel();
290
297
  const response = parsed.response;
291
298
  const contentType = parsed.contentType;
@@ -372,10 +379,10 @@ export class HttpCrawler extends BasicCrawler {
372
379
  * on the request such as only downloading the request body if the
373
380
  * received content type matches text/html, application/xml, application/xhtml+xml.
374
381
  */
375
- async #requestFunction({ request, session, proxyUrl }) {
382
+ async #requestFunction({ request, session, proxyUrl, signal }) {
376
383
  const opts = this.#getRequestOptions(request, proxyUrl);
377
384
  try {
378
- return await this.#requestAsBrowser(opts, session);
385
+ return await this.#requestAsBrowser(opts, session, signal);
379
386
  }
380
387
  catch (e) {
381
388
  if (e instanceof Error && e.constructor.name === 'TimeoutError') {
@@ -528,7 +535,7 @@ export class HttpCrawler extends BasicCrawler {
528
535
  /**
529
536
  * @internal wraps public utility for mocking purposes
530
537
  */
531
- #requestAsBrowser = async (options, session) => {
538
+ #requestAsBrowser = async (options, session, signal) => {
532
539
  const opts = processHttpRequestOptions({
533
540
  ...options,
534
541
  responseType: 'text',
@@ -539,10 +546,11 @@ export class HttpCrawler extends BasicCrawler {
539
546
  const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
540
547
  // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
541
548
  // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
542
- // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
543
- // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
544
- // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
549
+ // kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
550
+ // bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
551
+ // body still fails cleanly with a navigation timeout and its download stops too.
545
552
  const cancelSignal = storage.getStore()?.cancelTask.signal;
553
+ const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
546
554
  const response = await this.httpClient.sendRequest(new Request(opts.url, {
547
555
  body: opts.body ? Readable.toWeb(opts.body) : undefined,
548
556
  headers: new Headers(opts.headers),
@@ -553,7 +561,7 @@ export class HttpCrawler extends BasicCrawler {
553
561
  session,
554
562
  cookieJar,
555
563
  proxyUrl: opts.proxyUrl,
556
- signal: cancelSignal,
564
+ signal: requestSignal,
557
565
  timeoutMillis: cancelSignal ? undefined : opts.timeout,
558
566
  ignoreTlsErrors: this.#ignoreTlsErrors,
559
567
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.239",
3
+ "version": "4.0.0-beta.240",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.239",
52
- "@crawlee/http-client": "4.0.0-beta.239",
53
- "@crawlee/types": "4.0.0-beta.239",
54
- "@crawlee/utils": "4.0.0-beta.239",
51
+ "@crawlee/basic": "4.0.0-beta.240",
52
+ "@crawlee/http-client": "4.0.0-beta.240",
53
+ "@crawlee/types": "4.0.0-beta.240",
54
+ "@crawlee/utils": "4.0.0-beta.240",
55
55
  "cheerio": "^1.0.0",
56
56
  "content-type": "^3.1.1",
57
57
  "domhandler": "^6.0.0",
@@ -68,5 +68,5 @@
68
68
  }
69
69
  }
70
70
  },
71
- "gitHead": "5fadaae0607c497eeffd0029fb17955f69d2ed46"
71
+ "gitHead": "040d91e9f36dfd4c99f73b63abd97b92b82d89a4"
72
72
  }