@crawlee/http 4.0.0-beta.239 → 4.0.0-beta.240
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/http-crawler.js +17 -9
- package/package.json +6 -6
|
@@ -117,6 +117,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
117
117
|
#suggestResponseEncoding;
|
|
118
118
|
#forceResponseEncoding;
|
|
119
119
|
#supportedMimeTypes;
|
|
120
|
+
#downloadAbortControllers = new WeakMap();
|
|
120
121
|
/**
|
|
121
122
|
* @internal
|
|
122
123
|
*/
|
|
@@ -231,10 +232,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
231
232
|
tryCancel();
|
|
232
233
|
const { request, session } = crawlingContext;
|
|
233
234
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
235
|
+
const downloadAbortController = new AbortController();
|
|
236
|
+
this.#downloadAbortControllers.set(request, downloadAbortController);
|
|
234
237
|
// Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
|
|
235
238
|
// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
|
|
236
239
|
// client abort.
|
|
237
|
-
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
240
|
+
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
238
241
|
tryCancel();
|
|
239
242
|
request.loadedUrl = httpResponse?.url;
|
|
240
243
|
request.state = RequestState.AFTER_NAV;
|
|
@@ -285,7 +288,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
285
288
|
if (remaining <= 0) {
|
|
286
289
|
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
287
290
|
}
|
|
288
|
-
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`)
|
|
291
|
+
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
|
|
292
|
+
// Stop the download along with the read, so a body that never ends doesn't outlive the request.
|
|
293
|
+
this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
|
|
294
|
+
throw error;
|
|
295
|
+
});
|
|
289
296
|
tryCancel();
|
|
290
297
|
const response = parsed.response;
|
|
291
298
|
const contentType = parsed.contentType;
|
|
@@ -372,10 +379,10 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
372
379
|
* on the request such as only downloading the request body if the
|
|
373
380
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
374
381
|
*/
|
|
375
|
-
async #requestFunction({ request, session, proxyUrl }) {
|
|
382
|
+
async #requestFunction({ request, session, proxyUrl, signal }) {
|
|
376
383
|
const opts = this.#getRequestOptions(request, proxyUrl);
|
|
377
384
|
try {
|
|
378
|
-
return await this.#requestAsBrowser(opts, session);
|
|
385
|
+
return await this.#requestAsBrowser(opts, session, signal);
|
|
379
386
|
}
|
|
380
387
|
catch (e) {
|
|
381
388
|
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
@@ -528,7 +535,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
528
535
|
/**
|
|
529
536
|
* @internal wraps public utility for mocking purposes
|
|
530
537
|
*/
|
|
531
|
-
#requestAsBrowser = async (options, session) => {
|
|
538
|
+
#requestAsBrowser = async (options, session, signal) => {
|
|
532
539
|
const opts = processHttpRequestOptions({
|
|
533
540
|
...options,
|
|
534
541
|
responseType: 'text',
|
|
@@ -539,10 +546,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
539
546
|
const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
|
|
540
547
|
// Bind the request to the shared navigation window instead of a fixed per-request timeout, so
|
|
541
548
|
// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
|
|
542
|
-
// kill a lazily-read body mid-extension.
|
|
543
|
-
//
|
|
544
|
-
// still fails cleanly with a navigation timeout
|
|
549
|
+
// kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
|
|
550
|
+
// bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
|
|
551
|
+
// body still fails cleanly with a navigation timeout and its download stops too.
|
|
545
552
|
const cancelSignal = storage.getStore()?.cancelTask.signal;
|
|
553
|
+
const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
|
|
546
554
|
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
547
555
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
548
556
|
headers: new Headers(opts.headers),
|
|
@@ -553,7 +561,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
553
561
|
session,
|
|
554
562
|
cookieJar,
|
|
555
563
|
proxyUrl: opts.proxyUrl,
|
|
556
|
-
signal:
|
|
564
|
+
signal: requestSignal,
|
|
557
565
|
timeoutMillis: cancelSignal ? undefined : opts.timeout,
|
|
558
566
|
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
559
567
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.240",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.240",
|
|
52
|
+
"@crawlee/http-client": "4.0.0-beta.240",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.240",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.240",
|
|
55
55
|
"cheerio": "^1.0.0",
|
|
56
56
|
"content-type": "^3.1.1",
|
|
57
57
|
"domhandler": "^6.0.0",
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
},
|
|
71
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "040d91e9f36dfd4c99f73b63abd97b92b82d89a4"
|
|
72
72
|
}
|