@crawlee/http 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +131 -64
- package/internals/file-download.js +39 -122
- package/internals/http-crawler.d.ts +57 -66
- package/internals/http-crawler.js +97 -86
- package/internals/utils.d.ts +0 -5
- package/internals/utils.js +7 -44
- package/package.json +15 -17
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
|
|
4
|
-
import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
|
|
3
|
+
import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
|
|
5
4
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
6
|
-
import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
|
|
7
|
-
import
|
|
5
|
+
import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
|
|
6
|
+
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
8
7
|
import iconv from 'iconv-lite';
|
|
9
8
|
import { z } from 'zod';
|
|
10
9
|
import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
11
|
-
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse
|
|
10
|
+
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
|
|
12
11
|
/**
|
|
13
12
|
* Default mime types, which HttpScraper supports.
|
|
14
13
|
*/
|
|
@@ -16,7 +15,8 @@ const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml
|
|
|
16
15
|
const APPLICATION_JSON_MIME_TYPE = 'application/json';
|
|
17
16
|
/**
|
|
18
17
|
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
19
|
-
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default
|
|
18
|
+
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
|
|
19
|
+
* concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
|
|
20
20
|
*
|
|
21
21
|
* A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
|
|
22
22
|
* wholesale, tuning included, so spread these options in if you want to keep it:
|
|
@@ -117,10 +117,14 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
117
117
|
#suggestResponseEncoding;
|
|
118
118
|
#forceResponseEncoding;
|
|
119
119
|
#supportedMimeTypes;
|
|
120
|
+
#downloadAbortControllers = new WeakMap();
|
|
121
|
+
/**
|
|
122
|
+
* @internal
|
|
123
|
+
*/
|
|
120
124
|
static optionsShape = {
|
|
121
125
|
...BasicCrawler.optionsShape,
|
|
122
126
|
navigationTimeoutSecs: schemas.anyNumber.default(30),
|
|
123
|
-
ignoreTlsErrors: z.boolean().default(
|
|
127
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
124
128
|
additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
125
129
|
suggestResponseEncoding: z.string().optional(),
|
|
126
130
|
forceResponseEncoding: z.string().optional(),
|
|
@@ -128,6 +132,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
128
132
|
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
129
133
|
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
130
134
|
};
|
|
135
|
+
/** @internal */
|
|
131
136
|
static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
|
|
132
137
|
/**
|
|
133
138
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
@@ -143,7 +148,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
143
148
|
});
|
|
144
149
|
this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
|
|
145
150
|
if (additionalMimeTypes.length)
|
|
146
|
-
this
|
|
151
|
+
this.#extendSupportedMimeTypes(additionalMimeTypes);
|
|
147
152
|
if (suggestResponseEncoding && forceResponseEncoding) {
|
|
148
153
|
this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
|
|
149
154
|
}
|
|
@@ -156,20 +161,16 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
156
161
|
// members `extendContext` added at runtime.
|
|
157
162
|
this.#preNavigationHooks = preNavigationHooks;
|
|
158
163
|
this.#postNavigationHooks = [
|
|
159
|
-
({ request, response }) => this
|
|
164
|
+
({ request, response }) => this.#abortDownloadOfBody(request, response),
|
|
160
165
|
...postNavigationHooks,
|
|
161
166
|
];
|
|
162
167
|
this.#saveResponseCookies = saveResponseCookies;
|
|
163
168
|
}
|
|
169
|
+
/** @internal */
|
|
164
170
|
getNavigationTimeoutMillis() {
|
|
165
171
|
return this.#navigationTimeoutMillis;
|
|
166
172
|
}
|
|
167
|
-
/**
|
|
168
|
-
* Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
|
|
169
|
-
* concurrency shortcuts on top. Not called for a supplied
|
|
170
|
-
* {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
|
|
171
|
-
* keep the tuning.
|
|
172
|
-
*/
|
|
173
|
+
/** @internal */
|
|
173
174
|
createDefaultConcurrencySystem(options) {
|
|
174
175
|
return super.createDefaultConcurrencySystem({
|
|
175
176
|
...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
|
|
@@ -179,9 +180,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
179
180
|
buildContextPipeline() {
|
|
180
181
|
// When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
|
|
181
182
|
// the response-derived members, so the guarded action is bypassed and the context left untouched.
|
|
182
|
-
const skipGuard = (action) => ({
|
|
183
|
-
action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
|
|
184
|
-
});
|
|
183
|
+
const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
|
|
185
184
|
// A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
|
|
186
185
|
// hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
|
|
187
186
|
// window the navigation uses instead of each step being timed on its own.
|
|
@@ -193,9 +192,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
193
192
|
}
|
|
194
193
|
return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
|
|
195
194
|
});
|
|
196
|
-
let pipeline = ContextPipeline.create().compose(
|
|
197
|
-
action: this.prepareHttpRequest.bind(this),
|
|
198
|
-
});
|
|
195
|
+
let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
|
|
199
196
|
for (const hook of this.#preNavigationHooks) {
|
|
200
197
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
201
198
|
}
|
|
@@ -204,21 +201,14 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
204
201
|
pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
|
|
205
202
|
}
|
|
206
203
|
return pipelineWithNavigation
|
|
207
|
-
.compose(
|
|
208
|
-
.compose(
|
|
204
|
+
.compose(this.#processHttpResponse.bind(this))
|
|
205
|
+
.compose(this.#handleBlockedRequestByContent.bind(this));
|
|
209
206
|
}
|
|
210
|
-
async prepareHttpRequest(crawlingContext) {
|
|
207
|
+
async #prepareHttpRequest(crawlingContext) {
|
|
211
208
|
const { request } = crawlingContext;
|
|
212
209
|
if (request.skipNavigation) {
|
|
213
210
|
return {
|
|
214
|
-
request:
|
|
215
|
-
get(target, propertyName, receiver) {
|
|
216
|
-
if (propertyName === 'loadedUrl') {
|
|
217
|
-
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
218
|
-
}
|
|
219
|
-
return Reflect.get(target, propertyName, receiver);
|
|
220
|
-
},
|
|
221
|
-
}),
|
|
211
|
+
request: skippedNavigationRequest(request),
|
|
222
212
|
get response() {
|
|
223
213
|
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
224
214
|
},
|
|
@@ -227,20 +217,23 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
227
217
|
request.state = RequestState.BEFORE_NAV;
|
|
228
218
|
return {};
|
|
229
219
|
}
|
|
220
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
230
221
|
async makeHttpRequest(crawlingContext) {
|
|
231
222
|
tryCancel();
|
|
232
223
|
const { request, session } = crawlingContext;
|
|
233
224
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
225
|
+
const downloadAbortController = new AbortController();
|
|
226
|
+
this.#downloadAbortControllers.set(request, downloadAbortController);
|
|
234
227
|
// Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
|
|
235
228
|
// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
|
|
236
229
|
// client abort.
|
|
237
|
-
const httpResponse = await addTimeoutToPromise(async () => this
|
|
230
|
+
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
238
231
|
tryCancel();
|
|
239
232
|
request.loadedUrl = httpResponse?.url;
|
|
240
233
|
request.state = RequestState.AFTER_NAV;
|
|
241
234
|
return { request: request, response: httpResponse };
|
|
242
235
|
}
|
|
243
|
-
async processHttpResponse(crawlingContext) {
|
|
236
|
+
async #processHttpResponse(crawlingContext) {
|
|
244
237
|
if (crawlingContext.request.skipNavigation) {
|
|
245
238
|
return {
|
|
246
239
|
get contentType() {
|
|
@@ -258,6 +251,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
258
251
|
get parseWithCheerio() {
|
|
259
252
|
throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
|
|
260
253
|
},
|
|
254
|
+
get querySelectorAll() {
|
|
255
|
+
throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
|
|
256
|
+
},
|
|
257
|
+
get querySelectorOne() {
|
|
258
|
+
throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
|
|
259
|
+
},
|
|
261
260
|
};
|
|
262
261
|
}
|
|
263
262
|
tryCancel();
|
|
@@ -279,20 +278,31 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
279
278
|
if (remaining <= 0) {
|
|
280
279
|
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
281
280
|
}
|
|
282
|
-
const parsed = await addTimeoutToPromise(async () => this
|
|
281
|
+
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
|
|
282
|
+
// Stop the download along with the read, so a body that never ends doesn't outlive the request.
|
|
283
|
+
this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
|
|
284
|
+
throw error;
|
|
285
|
+
});
|
|
283
286
|
tryCancel();
|
|
284
287
|
const response = parsed.response;
|
|
285
288
|
const contentType = parsed.contentType;
|
|
286
|
-
const
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
|
|
289
|
+
const loadBody = async () => {
|
|
290
|
+
const { load } = await import('cheerio/slim');
|
|
291
|
+
return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
|
|
292
|
+
};
|
|
293
|
+
const querySelectorAll = async (selector, _timeoutMs) => {
|
|
294
|
+
const matches = (await loadBody())(selector);
|
|
295
|
+
if (matches.length === 0) {
|
|
290
296
|
throw new Error(`Selector '${selector}' not found.`);
|
|
291
297
|
}
|
|
298
|
+
return matches;
|
|
299
|
+
};
|
|
300
|
+
const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
|
|
301
|
+
const waitForSelector = async (selector, timeoutMs) => {
|
|
302
|
+
await querySelectorAll(selector, timeoutMs);
|
|
292
303
|
};
|
|
293
304
|
const parseWithCheerio = async (selector, timeoutMs) => {
|
|
294
|
-
const
|
|
295
|
-
const $ = cheerio.load(parsed.body.toString());
|
|
305
|
+
const $ = await loadBody();
|
|
296
306
|
if (selector) {
|
|
297
307
|
await crawlingContext.waitForSelector(selector, timeoutMs);
|
|
298
308
|
}
|
|
@@ -326,20 +336,22 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
326
336
|
return JSON.parse(jsonString);
|
|
327
337
|
},
|
|
328
338
|
waitForSelector,
|
|
339
|
+
querySelectorOne,
|
|
340
|
+
querySelectorAll,
|
|
329
341
|
parseWithCheerio,
|
|
330
342
|
contentType,
|
|
331
343
|
body: parsed.body,
|
|
332
344
|
};
|
|
333
345
|
}
|
|
334
|
-
async handleBlockedRequestByContent(crawlingContext) {
|
|
346
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
335
347
|
if (this.retryOnBlocked) {
|
|
336
|
-
const error = await this
|
|
348
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
337
349
|
if (error)
|
|
338
350
|
throw new SessionError(error);
|
|
339
351
|
}
|
|
340
352
|
return {};
|
|
341
353
|
}
|
|
342
|
-
async isRequestBlocked(crawlingContext) {
|
|
354
|
+
async #isRequestBlocked(crawlingContext) {
|
|
343
355
|
if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
|
|
344
356
|
const $ = await crawlingContext.parseWithCheerio();
|
|
345
357
|
const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
|
|
@@ -357,14 +369,14 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
357
369
|
* on the request such as only downloading the request body if the
|
|
358
370
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
359
371
|
*/
|
|
360
|
-
async requestFunction({ request, session, proxyUrl }) {
|
|
361
|
-
const opts = this
|
|
372
|
+
async #requestFunction({ request, session, proxyUrl, signal }) {
|
|
373
|
+
const opts = this.#getRequestOptions(request, proxyUrl);
|
|
362
374
|
try {
|
|
363
|
-
return await this
|
|
375
|
+
return await this.#requestAsBrowser(opts, session, signal);
|
|
364
376
|
}
|
|
365
377
|
catch (e) {
|
|
366
378
|
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
367
|
-
this
|
|
379
|
+
this.#handleRequestTimeout(session);
|
|
368
380
|
return new Response(); // this will never happen, as handleRequestTimeout always throws
|
|
369
381
|
}
|
|
370
382
|
if (this.isProxyError(e)) {
|
|
@@ -378,16 +390,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
378
390
|
/**
|
|
379
391
|
* Encodes and parses response according to the provided content type
|
|
380
392
|
*/
|
|
381
|
-
async parseResponse(request, response) {
|
|
393
|
+
async #parseResponse(request, response) {
|
|
382
394
|
const { status } = response;
|
|
383
395
|
const { type, charset } = parseContentTypeFromResponse(response);
|
|
384
|
-
const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
|
|
385
|
-
const contentType = { type, encoding };
|
|
386
396
|
if (status >= 400 && status <= 599) {
|
|
387
397
|
this.statistics.registerStatusCode(status);
|
|
388
398
|
}
|
|
389
399
|
if (this.isErrorStatusCode(status)) {
|
|
390
|
-
const
|
|
400
|
+
const { response: decoded } = this.#encodeResponse(request, response, charset);
|
|
401
|
+
const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
391
402
|
// Errors are often sent as JSON, so attempt to parse them,
|
|
392
403
|
// despite Accept header being set to text/html.
|
|
393
404
|
if (type === APPLICATION_JSON_MIME_TYPE) {
|
|
@@ -403,37 +414,37 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
403
414
|
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
|
|
404
415
|
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
|
|
405
416
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
}
|
|
416
|
-
return { response, contentType, body: await reencodedResponse.text() };
|
|
417
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
|
|
418
|
+
// The charset comes from the document itself, so the raw bytes are what we need -
|
|
419
|
+
// decoding them through `encodeResponse` first would consume the body for nothing.
|
|
420
|
+
const rawBytes = Buffer.from(await response.arrayBuffer());
|
|
421
|
+
const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
|
|
422
|
+
const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
|
|
423
|
+
const body = iconv.encodingExists(charsetToUse)
|
|
424
|
+
? iconv.decode(rawBytes, charsetToUse)
|
|
425
|
+
: rawBytes.toString('utf8');
|
|
426
|
+
return { response, contentType: { type, encoding: 'utf-8' }, body };
|
|
417
427
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
response,
|
|
423
|
-
contentType,
|
|
424
|
-
};
|
|
428
|
+
const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
|
|
429
|
+
const contentType = { type, encoding };
|
|
430
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type)) {
|
|
431
|
+
return { response, contentType, body: await reencodedResponse.text() };
|
|
425
432
|
}
|
|
433
|
+
return {
|
|
434
|
+
body: Buffer.from(await reencodedResponse.bytes()),
|
|
435
|
+
response,
|
|
436
|
+
contentType,
|
|
437
|
+
};
|
|
426
438
|
}
|
|
427
439
|
/**
|
|
428
440
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
429
441
|
*/
|
|
430
|
-
getRequestOptions(request,
|
|
442
|
+
#getRequestOptions(request, proxyUrl) {
|
|
431
443
|
const requestOptions = {
|
|
432
444
|
url: request.url,
|
|
433
445
|
method: request.method,
|
|
434
446
|
proxyUrl,
|
|
435
447
|
timeout: this.#navigationTimeoutMillis,
|
|
436
|
-
sessionToken: session,
|
|
437
448
|
headers: request.headers,
|
|
438
449
|
body: undefined,
|
|
439
450
|
};
|
|
@@ -445,7 +456,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
445
456
|
requestOptions.body = request.payload ?? '';
|
|
446
457
|
return requestOptions;
|
|
447
458
|
}
|
|
448
|
-
encodeResponse(request, response, encoding) {
|
|
459
|
+
#encodeResponse(request, response, encoding) {
|
|
449
460
|
if (this.#forceResponseEncoding) {
|
|
450
461
|
encoding = this.#forceResponseEncoding;
|
|
451
462
|
}
|
|
@@ -481,29 +492,27 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
481
492
|
/**
|
|
482
493
|
* Checks and extends supported mime types
|
|
483
494
|
*/
|
|
484
|
-
extendSupportedMimeTypes(additionalMimeTypes) {
|
|
495
|
+
#extendSupportedMimeTypes(additionalMimeTypes) {
|
|
485
496
|
for (const mimeType of additionalMimeTypes) {
|
|
486
497
|
if (mimeType === '*/*') {
|
|
487
498
|
this.#supportedMimeTypes.add(mimeType);
|
|
488
499
|
continue;
|
|
489
500
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
this.#supportedMimeTypes.add(parsedType.type);
|
|
493
|
-
}
|
|
494
|
-
catch (err) {
|
|
501
|
+
const parsedType = parseContentType(mimeType);
|
|
502
|
+
if (!isTypeValid(parsedType.type)) {
|
|
495
503
|
throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
|
|
496
504
|
}
|
|
505
|
+
this.#supportedMimeTypes.add(parsedType.type);
|
|
497
506
|
}
|
|
498
507
|
}
|
|
499
508
|
/**
|
|
500
509
|
* Handles timeout request
|
|
501
510
|
*/
|
|
502
|
-
handleRequestTimeout(session) {
|
|
511
|
+
#handleRequestTimeout(session) {
|
|
503
512
|
session.markBad();
|
|
504
513
|
throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
505
514
|
}
|
|
506
|
-
abortDownloadOfBody(request, response) {
|
|
515
|
+
#abortDownloadOfBody(request, response) {
|
|
507
516
|
const { status } = response;
|
|
508
517
|
const { type } = parseContentTypeFromResponse(response);
|
|
509
518
|
const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
|
|
@@ -516,7 +525,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
516
525
|
/**
|
|
517
526
|
* @internal wraps public utility for mocking purposes
|
|
518
527
|
*/
|
|
519
|
-
requestAsBrowser = async (options, session) => {
|
|
528
|
+
#requestAsBrowser = async (options, session, signal) => {
|
|
520
529
|
const opts = processHttpRequestOptions({
|
|
521
530
|
...options,
|
|
522
531
|
responseType: 'text',
|
|
@@ -527,10 +536,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
527
536
|
const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
|
|
528
537
|
// Bind the request to the shared navigation window instead of a fixed per-request timeout, so
|
|
529
538
|
// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
|
|
530
|
-
// kill a lazily-read body mid-extension.
|
|
531
|
-
//
|
|
532
|
-
// still fails cleanly with a navigation timeout
|
|
539
|
+
// kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
|
|
540
|
+
// bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
|
|
541
|
+
// body still fails cleanly with a navigation timeout and its download stops too.
|
|
533
542
|
const cancelSignal = storage.getStore()?.cancelTask.signal;
|
|
543
|
+
const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
|
|
534
544
|
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
535
545
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
536
546
|
headers: new Headers(opts.headers),
|
|
@@ -540,7 +550,8 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
540
550
|
}), {
|
|
541
551
|
session,
|
|
542
552
|
cookieJar,
|
|
543
|
-
|
|
553
|
+
proxyUrl: opts.proxyUrl,
|
|
554
|
+
signal: requestSignal,
|
|
544
555
|
timeoutMillis: cancelSignal ? undefined : opts.timeout,
|
|
545
556
|
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
546
557
|
});
|
package/internals/utils.d.ts
CHANGED
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
|
|
2
|
-
/**
|
|
3
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
4
|
-
*/
|
|
5
|
-
export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
|
|
6
1
|
/**
|
|
7
2
|
* Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
|
|
8
3
|
* declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
|
package/internals/utils.js
CHANGED
|
@@ -1,44 +1,9 @@
|
|
|
1
1
|
import { extname } from 'node:path';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import contentTypeParser from 'content-type';
|
|
2
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
3
|
+
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
5
4
|
import mime from 'mime-types';
|
|
6
5
|
import { z } from 'zod';
|
|
7
6
|
const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
|
|
8
|
-
/**
|
|
9
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
10
|
-
*/
|
|
11
|
-
export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
|
|
12
|
-
const url = new URL(request.url);
|
|
13
|
-
const headers = new Headers(request.headers);
|
|
14
|
-
applySearchParams(url, searchParams);
|
|
15
|
-
if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
|
|
16
|
-
throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
|
|
17
|
-
}
|
|
18
|
-
const body = (() => {
|
|
19
|
-
if (form !== undefined) {
|
|
20
|
-
return Readable.from(new URLSearchParams(form).toString());
|
|
21
|
-
}
|
|
22
|
-
if (json !== undefined) {
|
|
23
|
-
return Readable.from(JSON.stringify(json));
|
|
24
|
-
}
|
|
25
|
-
if (request.body !== undefined) {
|
|
26
|
-
return Readable.from(request.body);
|
|
27
|
-
}
|
|
28
|
-
return undefined;
|
|
29
|
-
})();
|
|
30
|
-
if (form !== undefined && !headers.has('content-type')) {
|
|
31
|
-
headers.set('content-type', 'application/x-www-form-urlencoded');
|
|
32
|
-
}
|
|
33
|
-
if (json !== undefined && !headers.has('content-type')) {
|
|
34
|
-
headers.set('content-type', 'application/json');
|
|
35
|
-
}
|
|
36
|
-
if (username !== undefined || password !== undefined) {
|
|
37
|
-
const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
|
|
38
|
-
headers.set('authorization', `Basic ${encodedAuth}`);
|
|
39
|
-
}
|
|
40
|
-
return { ...request, body, url, headers };
|
|
41
|
-
}
|
|
42
7
|
/**
|
|
43
8
|
* Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
|
|
44
9
|
* declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
|
|
@@ -59,18 +24,16 @@ export function parseContentTypeFromResponse(response) {
|
|
|
59
24
|
const { url, headers } = response;
|
|
60
25
|
let parsedContentType;
|
|
61
26
|
if (headers.get('content-type')) {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
// Can not parse content type from Content-Type header. Try to parse it from file extension.
|
|
67
|
-
}
|
|
27
|
+
const parsed = parseContentType(headers.get('content-type'));
|
|
28
|
+
// Can not parse content type from Content-Type header. Try to parse it from file extension.
|
|
29
|
+
if (isTypeValid(parsed.type))
|
|
30
|
+
parsedContentType = parsed;
|
|
68
31
|
}
|
|
69
32
|
// Parse content type from file extension as fallback
|
|
70
33
|
if (!parsedContentType) {
|
|
71
34
|
const parsedUrl = new URL(url);
|
|
72
35
|
const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
|
|
73
|
-
parsedContentType =
|
|
36
|
+
parsedContentType = parseContentType(contentTypeFromExtname);
|
|
74
37
|
}
|
|
75
38
|
return {
|
|
76
39
|
type: parsedContentType.type,
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -47,21 +47,19 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@apify/timeout": "^0.
|
|
51
|
-
"@
|
|
52
|
-
"@crawlee/
|
|
53
|
-
"@crawlee/
|
|
54
|
-
"@crawlee/
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
60
|
-
"
|
|
61
|
-
"mime-types": "^3.0.1",
|
|
62
|
-
"tslib": "^2.8.1",
|
|
50
|
+
"@apify/timeout": "^1.0.1",
|
|
51
|
+
"@crawlee/basic": "4.0.0-rc.2",
|
|
52
|
+
"@crawlee/http-client": "4.0.0-rc.2",
|
|
53
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
54
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
55
|
+
"cheerio": "^1.2.0",
|
|
56
|
+
"content-type": "^3.1.1",
|
|
57
|
+
"domhandler": "^6.0.1",
|
|
58
|
+
"iconv-lite": "^0.7.3",
|
|
59
|
+
"mime-types": "^3.0.2",
|
|
60
|
+
"tslib": "^2.9.1",
|
|
63
61
|
"type-fest": "^4.41.0",
|
|
64
|
-
"zod": "^4.
|
|
62
|
+
"zod": "^4.6.7"
|
|
65
63
|
},
|
|
66
64
|
"lerna": {
|
|
67
65
|
"command": {
|
|
@@ -70,5 +68,5 @@
|
|
|
70
68
|
}
|
|
71
69
|
}
|
|
72
70
|
},
|
|
73
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
74
72
|
}
|