@crawlee/http 4.0.0-beta.28 → 4.0.0-beta.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +134 -54
- package/internals/file-download.js +39 -120
- package/internals/http-crawler.d.ts +220 -232
- package/internals/http-crawler.js +280 -229
- package/internals/utils.d.ts +4 -4
- package/internals/utils.js +18 -47
- package/package.json +17 -18
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/file-download.d.ts.map +0 -1
- package/internals/file-download.js.map +0 -1
- package/internals/http-crawler.d.ts.map +0 -1
- package/internals/http-crawler.js.map +0 -1
- package/internals/utils.d.ts.map +0 -1
- package/internals/utils.js.map +0 -1
|
@@ -1,27 +1,38 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import { BasicCrawler,
|
|
3
|
+
import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
|
|
4
4
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
5
|
-
import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
|
|
6
|
-
import
|
|
7
|
-
import contentTypeParser from 'content-type';
|
|
5
|
+
import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
|
|
6
|
+
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
8
7
|
import iconv from 'iconv-lite';
|
|
9
|
-
import
|
|
10
|
-
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
|
-
import {
|
|
8
|
+
import { z } from 'zod';
|
|
9
|
+
import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
10
|
+
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
|
|
12
11
|
/**
|
|
13
12
|
* Default mime types, which HttpScraper supports.
|
|
14
13
|
*/
|
|
15
14
|
const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
|
|
16
15
|
const APPLICATION_JSON_MIME_TYPE = 'application/json';
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
18
|
+
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
|
|
19
|
+
* concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
|
|
20
|
+
*
|
|
21
|
+
* A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
|
|
22
|
+
* wholesale, tuning included, so spread these options in if you want to keep it:
|
|
23
|
+
*
|
|
24
|
+
* ```typescript
|
|
25
|
+
* new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
|
|
18
29
|
desiredConcurrency: 10,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
30
|
+
loadSignals: {
|
|
31
|
+
eventLoop: {
|
|
32
|
+
snapshotIntervalSecs: 2,
|
|
33
|
+
maxBlockedMillis: 100,
|
|
34
|
+
overloadedRatio: 0.7,
|
|
35
|
+
},
|
|
25
36
|
},
|
|
26
37
|
};
|
|
27
38
|
/**
|
|
@@ -35,13 +46,15 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
35
46
|
*
|
|
36
47
|
* This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
|
|
37
48
|
*
|
|
38
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
39
|
-
* {@link
|
|
40
|
-
*
|
|
49
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
50
|
+
* {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
|
|
51
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
52
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
53
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
54
|
+
* result as `requestManager`.
|
|
41
55
|
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
56
|
+
* > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
|
|
57
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
45
58
|
*
|
|
46
59
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
47
60
|
*
|
|
@@ -55,18 +68,18 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
55
68
|
* ]
|
|
56
69
|
* ```
|
|
57
70
|
*
|
|
58
|
-
* By default, this crawler only processes web pages with the `text/html`
|
|
59
|
-
* and `application/
|
|
71
|
+
* By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
72
|
+
* and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
|
|
60
73
|
* and skips pages with other content types. If you want the crawler to process other content types,
|
|
61
74
|
* use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
|
|
62
75
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
63
76
|
* For details, see {@link HttpCrawlerOptions.requestHandler}.
|
|
64
77
|
*
|
|
65
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* {@link
|
|
78
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
79
|
+
* {@link ConcurrencySystem}.
|
|
80
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
81
|
+
* constructor, or, for finer control, by injecting a pre-configured
|
|
82
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
70
83
|
*
|
|
71
84
|
* **Example usage:**
|
|
72
85
|
*
|
|
@@ -92,155 +105,228 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
92
105
|
* @category Crawlers
|
|
93
106
|
*/
|
|
94
107
|
export class HttpCrawler extends BasicCrawler {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
108
|
+
// Internal storage uses the base (non-extended) context types. The public option types are
|
|
109
|
+
// extension-aware for consumer DX, but internally the pipeline composes hooks against the
|
|
110
|
+
// concrete crawling context, which does not statically carry `ContextExtension`. The members
|
|
111
|
+
// added by `extendContext` are present at runtime regardless.
|
|
112
|
+
#preNavigationHooks;
|
|
113
|
+
#postNavigationHooks;
|
|
114
|
+
#saveResponseCookies;
|
|
115
|
+
#navigationTimeoutMillis;
|
|
116
|
+
#ignoreTlsErrors;
|
|
117
|
+
#suggestResponseEncoding;
|
|
118
|
+
#forceResponseEncoding;
|
|
119
|
+
#supportedMimeTypes;
|
|
120
|
+
#downloadAbortControllers = new WeakMap();
|
|
121
|
+
/**
|
|
122
|
+
* @internal
|
|
123
|
+
*/
|
|
105
124
|
static optionsShape = {
|
|
106
125
|
...BasicCrawler.optionsShape,
|
|
107
|
-
navigationTimeoutSecs:
|
|
108
|
-
|
|
109
|
-
additionalMimeTypes:
|
|
110
|
-
suggestResponseEncoding:
|
|
111
|
-
forceResponseEncoding:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
preNavigationHooks: ow.optional.array,
|
|
116
|
-
postNavigationHooks: ow.optional.array,
|
|
126
|
+
navigationTimeoutSecs: schemas.anyNumber.default(30),
|
|
127
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
128
|
+
additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
129
|
+
suggestResponseEncoding: z.string().optional(),
|
|
130
|
+
forceResponseEncoding: z.string().optional(),
|
|
131
|
+
saveResponseCookies: z.boolean().default(true),
|
|
132
|
+
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
133
|
+
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
117
134
|
};
|
|
135
|
+
/** @internal */
|
|
136
|
+
static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
|
|
118
137
|
/**
|
|
119
138
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
120
139
|
*/
|
|
121
140
|
constructor(options = {}) {
|
|
122
|
-
|
|
123
|
-
const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
|
|
141
|
+
const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks,
|
|
124
142
|
// BasicCrawler
|
|
125
|
-
|
|
143
|
+
contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
|
|
126
144
|
super({
|
|
127
145
|
...basicCrawlerOptions,
|
|
128
|
-
autoscaledPoolOptions,
|
|
129
146
|
contextPipelineBuilder: contextPipelineBuilder ??
|
|
130
147
|
(() => this.buildContextPipeline()),
|
|
131
148
|
});
|
|
132
|
-
|
|
133
|
-
if (!this.useSessionPool && persistCookiesPerSession) {
|
|
134
|
-
throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
|
|
135
|
-
}
|
|
136
|
-
this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
|
|
149
|
+
this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
|
|
137
150
|
if (additionalMimeTypes.length)
|
|
138
|
-
this
|
|
151
|
+
this.#extendSupportedMimeTypes(additionalMimeTypes);
|
|
139
152
|
if (suggestResponseEncoding && forceResponseEncoding) {
|
|
140
153
|
this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
|
|
141
154
|
}
|
|
142
|
-
this
|
|
143
|
-
this
|
|
144
|
-
this
|
|
145
|
-
this
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
this
|
|
150
|
-
|
|
155
|
+
this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
|
|
156
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
157
|
+
this.#suggestResponseEncoding = suggestResponseEncoding;
|
|
158
|
+
this.#forceResponseEncoding = forceResponseEncoding;
|
|
159
|
+
// Cast away the extension-aware option types to the base internal storage types (see the field
|
|
160
|
+
// declarations above). This is sound - the hooks only ever receive the base context plus the
|
|
161
|
+
// members `extendContext` added at runtime.
|
|
162
|
+
this.#preNavigationHooks = preNavigationHooks;
|
|
163
|
+
this.#postNavigationHooks = [
|
|
164
|
+
({ request, response }) => this.#abortDownloadOfBody(request, response),
|
|
151
165
|
...postNavigationHooks,
|
|
152
166
|
];
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
this.#saveResponseCookies = saveResponseCookies;
|
|
168
|
+
}
|
|
169
|
+
/** @internal */
|
|
170
|
+
getNavigationTimeoutMillis() {
|
|
171
|
+
return this.#navigationTimeoutMillis;
|
|
172
|
+
}
|
|
173
|
+
/** @internal */
|
|
174
|
+
createDefaultConcurrencySystem(options) {
|
|
175
|
+
return super.createDefaultConcurrencySystem({
|
|
176
|
+
...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
|
|
177
|
+
...options,
|
|
178
|
+
});
|
|
159
179
|
}
|
|
160
180
|
buildContextPipeline() {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
181
|
+
// When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
|
|
182
|
+
// the response-derived members, so the guarded action is bypassed and the context left untouched.
|
|
183
|
+
const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
|
|
184
|
+
// A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
|
|
185
|
+
// hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
|
|
186
|
+
// window the navigation uses instead of each step being timed on its own.
|
|
187
|
+
const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
|
|
188
|
+
const windowGuard = (step) => skipGuard(async (ctx) => {
|
|
189
|
+
const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
|
|
190
|
+
if (remaining <= 0) {
|
|
191
|
+
throw new TimeoutError(navigationTimedOut);
|
|
192
|
+
}
|
|
193
|
+
return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
|
|
194
|
+
});
|
|
195
|
+
let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
|
|
196
|
+
for (const hook of this.#preNavigationHooks) {
|
|
197
|
+
pipeline = pipeline.compose(windowGuard(hook));
|
|
198
|
+
}
|
|
199
|
+
let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
|
|
200
|
+
for (const hook of this.#postNavigationHooks) {
|
|
201
|
+
pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
|
|
202
|
+
}
|
|
203
|
+
return pipelineWithNavigation
|
|
204
|
+
.compose(this.#processHttpResponse.bind(this))
|
|
205
|
+
.compose(this.#handleBlockedRequestByContent.bind(this));
|
|
167
206
|
}
|
|
168
|
-
async
|
|
169
|
-
const { request
|
|
207
|
+
async #prepareHttpRequest(crawlingContext) {
|
|
208
|
+
const { request } = crawlingContext;
|
|
170
209
|
if (request.skipNavigation) {
|
|
171
210
|
return {
|
|
172
|
-
request:
|
|
173
|
-
get(target, propertyName, receiver) {
|
|
174
|
-
if (propertyName === 'loadedUrl') {
|
|
175
|
-
throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
176
|
-
}
|
|
177
|
-
return Reflect.get(target, propertyName, receiver);
|
|
178
|
-
},
|
|
179
|
-
}),
|
|
211
|
+
request: skippedNavigationRequest(request),
|
|
180
212
|
get response() {
|
|
181
|
-
throw new
|
|
213
|
+
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
182
214
|
},
|
|
183
215
|
};
|
|
184
216
|
}
|
|
185
|
-
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
|
|
186
217
|
request.state = RequestState.BEFORE_NAV;
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
218
|
+
return {};
|
|
219
|
+
}
|
|
220
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
221
|
+
async makeHttpRequest(crawlingContext) {
|
|
190
222
|
tryCancel();
|
|
191
|
-
const
|
|
192
|
-
const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
|
|
223
|
+
const { request, session } = crawlingContext;
|
|
193
224
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
194
|
-
const
|
|
225
|
+
const downloadAbortController = new AbortController();
|
|
226
|
+
this.#downloadAbortControllers.set(request, downloadAbortController);
|
|
227
|
+
// Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
|
|
228
|
+
// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
|
|
229
|
+
// client abort.
|
|
230
|
+
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
195
231
|
tryCancel();
|
|
196
232
|
request.loadedUrl = httpResponse?.url;
|
|
197
233
|
request.state = RequestState.AFTER_NAV;
|
|
198
234
|
return { request: request, response: httpResponse };
|
|
199
235
|
}
|
|
200
|
-
async processHttpResponse(crawlingContext) {
|
|
236
|
+
async #processHttpResponse(crawlingContext) {
|
|
201
237
|
if (crawlingContext.request.skipNavigation) {
|
|
202
238
|
return {
|
|
203
239
|
get contentType() {
|
|
204
|
-
throw new
|
|
240
|
+
throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
|
|
205
241
|
},
|
|
206
242
|
get body() {
|
|
207
|
-
throw new
|
|
243
|
+
throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
|
|
208
244
|
},
|
|
209
245
|
get json() {
|
|
210
|
-
throw new
|
|
246
|
+
throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
|
|
211
247
|
},
|
|
212
248
|
get waitForSelector() {
|
|
213
|
-
throw new
|
|
249
|
+
throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
|
|
214
250
|
},
|
|
215
251
|
get parseWithCheerio() {
|
|
216
|
-
throw new
|
|
252
|
+
throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
|
|
253
|
+
},
|
|
254
|
+
get querySelectorAll() {
|
|
255
|
+
throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
|
|
256
|
+
},
|
|
257
|
+
get querySelectorOne() {
|
|
258
|
+
throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
|
|
217
259
|
},
|
|
218
260
|
};
|
|
219
261
|
}
|
|
220
|
-
await this._executeHooks(this.postNavigationHooks, crawlingContext);
|
|
221
262
|
tryCancel();
|
|
222
|
-
|
|
263
|
+
// Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
|
|
264
|
+
// error is still a rate limit the domain should back off from.
|
|
265
|
+
if (crawlingContext.response.status === 429) {
|
|
266
|
+
const retryAfter = crawlingContext.response.headers.get('retry-after');
|
|
267
|
+
if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
|
|
268
|
+
// This is the one path that never reads the body, so cancel it to release the connection
|
|
269
|
+
// rather than leaving it to the garbage collector.
|
|
270
|
+
await crawlingContext.response.body?.cancel().catch(() => { });
|
|
271
|
+
throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
// Reading the body is still part of the navigation, so it draws from the same shared window: on a server
|
|
275
|
+
// that streams the body slowly the request completes (headers arrive) but the body read would otherwise
|
|
276
|
+
// run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
|
|
277
|
+
const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
|
|
278
|
+
if (remaining <= 0) {
|
|
279
|
+
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
280
|
+
}
|
|
281
|
+
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
|
|
282
|
+
// Stop the download along with the read, so a body that never ends doesn't outlive the request.
|
|
283
|
+
this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
|
|
284
|
+
throw error;
|
|
285
|
+
});
|
|
223
286
|
tryCancel();
|
|
224
287
|
const response = parsed.response;
|
|
225
288
|
const contentType = parsed.contentType;
|
|
226
|
-
const
|
|
227
|
-
const
|
|
228
|
-
|
|
289
|
+
const loadBody = async () => {
|
|
290
|
+
const { load } = await import('cheerio/slim');
|
|
291
|
+
return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
|
|
292
|
+
};
|
|
293
|
+
const querySelectorAll = async (selector, _timeoutMs) => {
|
|
294
|
+
const matches = (await loadBody())(selector);
|
|
295
|
+
if (matches.length === 0) {
|
|
229
296
|
throw new Error(`Selector '${selector}' not found.`);
|
|
230
297
|
}
|
|
298
|
+
return matches;
|
|
299
|
+
};
|
|
300
|
+
const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
|
|
301
|
+
const waitForSelector = async (selector, timeoutMs) => {
|
|
302
|
+
await querySelectorAll(selector, timeoutMs);
|
|
231
303
|
};
|
|
232
304
|
const parseWithCheerio = async (selector, timeoutMs) => {
|
|
233
|
-
const $ =
|
|
305
|
+
const $ = await loadBody();
|
|
234
306
|
if (selector) {
|
|
235
307
|
await crawlingContext.waitForSelector(selector, timeoutMs);
|
|
236
308
|
}
|
|
237
309
|
return $;
|
|
238
310
|
};
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
311
|
+
this.throwOnBlockedRequest(response.status);
|
|
312
|
+
if (this.#saveResponseCookies) {
|
|
313
|
+
try {
|
|
314
|
+
for (const cookie of getCookiesFromResponse(response)) {
|
|
315
|
+
if (!cookie)
|
|
316
|
+
continue;
|
|
317
|
+
try {
|
|
318
|
+
await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
|
|
319
|
+
ignoreError: false,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
catch (e) {
|
|
323
|
+
this.log.debug(`Could not set cookie: ${e.message}`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
catch (e) {
|
|
328
|
+
this.log.exception(e, 'Could not get cookies from response');
|
|
329
|
+
}
|
|
244
330
|
}
|
|
245
331
|
return {
|
|
246
332
|
get json() {
|
|
@@ -250,20 +336,22 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
250
336
|
return JSON.parse(jsonString);
|
|
251
337
|
},
|
|
252
338
|
waitForSelector,
|
|
339
|
+
querySelectorOne,
|
|
340
|
+
querySelectorAll,
|
|
253
341
|
parseWithCheerio,
|
|
254
342
|
contentType,
|
|
255
343
|
body: parsed.body,
|
|
256
344
|
};
|
|
257
345
|
}
|
|
258
|
-
async handleBlockedRequestByContent(crawlingContext) {
|
|
346
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
259
347
|
if (this.retryOnBlocked) {
|
|
260
|
-
const error = await this
|
|
348
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
261
349
|
if (error)
|
|
262
350
|
throw new SessionError(error);
|
|
263
351
|
}
|
|
264
352
|
return {};
|
|
265
353
|
}
|
|
266
|
-
async isRequestBlocked(crawlingContext) {
|
|
354
|
+
async #isRequestBlocked(crawlingContext) {
|
|
267
355
|
if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
|
|
268
356
|
const $ = await crawlingContext.parseWithCheerio();
|
|
269
357
|
const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
|
|
@@ -271,43 +359,28 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
271
359
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
272
360
|
}
|
|
273
361
|
}
|
|
274
|
-
|
|
275
|
-
// eslint-disable-next-line dot-notation
|
|
276
|
-
(this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
|
|
277
|
-
? // eslint-disable-next-line dot-notation
|
|
278
|
-
this.sessionPool['blockedStatusCodes']
|
|
279
|
-
: BLOCKED_STATUS_CODES;
|
|
280
|
-
if (blockedStatusCodes.includes(crawlingContext.response.status)) {
|
|
362
|
+
if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
|
|
281
363
|
return `Blocked by status code ${crawlingContext.response.status}`;
|
|
282
364
|
}
|
|
283
365
|
return false;
|
|
284
366
|
}
|
|
285
|
-
/**
|
|
286
|
-
* Returns the `Cookie` header value based on the current context and
|
|
287
|
-
* any changes that occurred in the navigation hooks.
|
|
288
|
-
*/
|
|
289
|
-
_applyCookies({ session, request }, preHookCookies, postHookCookies) {
|
|
290
|
-
const sessionCookie = session?.getCookieString(request.url) ?? '';
|
|
291
|
-
const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
|
|
292
|
-
return mergeCookies(request.url, sourceCookies);
|
|
293
|
-
}
|
|
294
367
|
/**
|
|
295
368
|
* Function to make the HTTP request. It performs optimizations
|
|
296
369
|
* on the request such as only downloading the request body if the
|
|
297
370
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
298
371
|
*/
|
|
299
|
-
async
|
|
300
|
-
const opts = this
|
|
372
|
+
async #requestFunction({ request, session, proxyUrl, signal }) {
|
|
373
|
+
const opts = this.#getRequestOptions(request, proxyUrl);
|
|
301
374
|
try {
|
|
302
|
-
return await this
|
|
375
|
+
return await this.#requestAsBrowser(opts, session, signal);
|
|
303
376
|
}
|
|
304
377
|
catch (e) {
|
|
305
378
|
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
306
|
-
this
|
|
307
|
-
return new Response(); // this will never happen, as
|
|
379
|
+
this.#handleRequestTimeout(session);
|
|
380
|
+
return new Response(); // this will never happen, as handleRequestTimeout always throws
|
|
308
381
|
}
|
|
309
382
|
if (this.isProxyError(e)) {
|
|
310
|
-
throw new SessionError(this.
|
|
383
|
+
throw new SessionError(this.getMessageFromError(e));
|
|
311
384
|
}
|
|
312
385
|
else {
|
|
313
386
|
throw e;
|
|
@@ -317,18 +390,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
317
390
|
/**
|
|
318
391
|
* Encodes and parses response according to the provided content type
|
|
319
392
|
*/
|
|
320
|
-
async
|
|
393
|
+
async #parseResponse(request, response) {
|
|
321
394
|
const { status } = response;
|
|
322
395
|
const { type, charset } = parseContentTypeFromResponse(response);
|
|
323
|
-
const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
|
|
324
|
-
const contentType = { type, encoding };
|
|
325
396
|
if (status >= 400 && status <= 599) {
|
|
326
|
-
this.
|
|
397
|
+
this.statistics.registerStatusCode(status);
|
|
327
398
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
399
|
+
if (this.isErrorStatusCode(status)) {
|
|
400
|
+
const { response: decoded } = this.#encodeResponse(request, response, charset);
|
|
401
|
+
const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
332
402
|
// Errors are often sent as JSON, so attempt to parse them,
|
|
333
403
|
// despite Accept header being set to text/html.
|
|
334
404
|
if (type === APPLICATION_JSON_MIME_TYPE) {
|
|
@@ -338,60 +408,60 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
338
408
|
message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
|
|
339
409
|
throw new Error(`${status} - ${message}`);
|
|
340
410
|
}
|
|
341
|
-
if (
|
|
411
|
+
if (this.additionalHttpErrorStatusCodes.has(status)) {
|
|
342
412
|
throw new Error(`${status} - Error status code was set by user.`);
|
|
343
413
|
}
|
|
344
414
|
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
|
|
345
415
|
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
|
|
346
416
|
}
|
|
347
|
-
|
|
348
|
-
|
|
417
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
|
|
418
|
+
// The charset comes from the document itself, so the raw bytes are what we need -
|
|
419
|
+
// decoding them through `encodeResponse` first would consume the body for nothing.
|
|
420
|
+
const rawBytes = Buffer.from(await response.arrayBuffer());
|
|
421
|
+
const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
|
|
422
|
+
const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
|
|
423
|
+
const body = iconv.encodingExists(charsetToUse)
|
|
424
|
+
? iconv.decode(rawBytes, charsetToUse)
|
|
425
|
+
: rawBytes.toString('utf8');
|
|
426
|
+
return { response, contentType: { type, encoding: 'utf-8' }, body };
|
|
349
427
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
response,
|
|
355
|
-
contentType,
|
|
356
|
-
};
|
|
428
|
+
const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
|
|
429
|
+
const contentType = { type, encoding };
|
|
430
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type)) {
|
|
431
|
+
return { response, contentType, body: await reencodedResponse.text() };
|
|
357
432
|
}
|
|
433
|
+
return {
|
|
434
|
+
body: Buffer.from(await reencodedResponse.bytes()),
|
|
435
|
+
response,
|
|
436
|
+
contentType,
|
|
437
|
+
};
|
|
358
438
|
}
|
|
359
439
|
/**
|
|
360
440
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
361
441
|
*/
|
|
362
|
-
|
|
442
|
+
#getRequestOptions(request, proxyUrl) {
|
|
363
443
|
const requestOptions = {
|
|
364
444
|
url: request.url,
|
|
365
445
|
method: request.method,
|
|
366
446
|
proxyUrl,
|
|
367
|
-
timeout: this
|
|
368
|
-
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
|
|
369
|
-
sessionToken: session,
|
|
447
|
+
timeout: this.#navigationTimeoutMillis,
|
|
370
448
|
headers: request.headers,
|
|
371
|
-
https: {
|
|
372
|
-
rejectUnauthorized: !this.ignoreSslErrors,
|
|
373
|
-
},
|
|
374
449
|
body: undefined,
|
|
375
450
|
};
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
if (session?.proxyInfo?.ignoreTlsErrors) {
|
|
380
|
-
requestOptions.https = {
|
|
381
|
-
...requestOptions.https,
|
|
382
|
-
rejectUnauthorized: false,
|
|
383
|
-
};
|
|
451
|
+
if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
|
|
452
|
+
requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
|
|
453
|
+
delete requestOptions.headers.cookie;
|
|
384
454
|
}
|
|
385
455
|
if (/PATCH|POST|PUT/.test(request.method))
|
|
386
456
|
requestOptions.body = request.payload ?? '';
|
|
387
457
|
return requestOptions;
|
|
388
458
|
}
|
|
389
|
-
|
|
390
|
-
if (this
|
|
391
|
-
encoding = this
|
|
459
|
+
#encodeResponse(request, response, encoding) {
|
|
460
|
+
if (this.#forceResponseEncoding) {
|
|
461
|
+
encoding = this.#forceResponseEncoding;
|
|
392
462
|
}
|
|
393
|
-
else if (!encoding && this
|
|
394
|
-
encoding = this
|
|
463
|
+
else if (!encoding && this.#suggestResponseEncoding) {
|
|
464
|
+
encoding = this.#suggestResponseEncoding;
|
|
395
465
|
}
|
|
396
466
|
// Fall back to utf-8 if we still don't have encoding.
|
|
397
467
|
const utf8 = 'utf8';
|
|
@@ -404,7 +474,9 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
404
474
|
// Try to re-encode a variety of unsupported encodings to utf-8
|
|
405
475
|
if (iconv.encodingExists(encoding)) {
|
|
406
476
|
const encodeStream = iconv.encodeStream(utf8);
|
|
407
|
-
const decodeStream = iconv
|
|
477
|
+
const decodeStream = iconv
|
|
478
|
+
.decodeStream(encoding)
|
|
479
|
+
.on('error', (err) => encodeStream.emit('error', err));
|
|
408
480
|
const reencodedBody = response.body
|
|
409
481
|
? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
|
|
410
482
|
.pipe(decodeStream)
|
|
@@ -420,55 +492,55 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
420
492
|
/**
|
|
421
493
|
* Checks and extends supported mime types
|
|
422
494
|
*/
|
|
423
|
-
|
|
495
|
+
#extendSupportedMimeTypes(additionalMimeTypes) {
|
|
424
496
|
for (const mimeType of additionalMimeTypes) {
|
|
425
497
|
if (mimeType === '*/*') {
|
|
426
|
-
this
|
|
498
|
+
this.#supportedMimeTypes.add(mimeType);
|
|
427
499
|
continue;
|
|
428
500
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
this.supportedMimeTypes.add(parsedType.type);
|
|
432
|
-
}
|
|
433
|
-
catch (err) {
|
|
501
|
+
const parsedType = parseContentType(mimeType);
|
|
502
|
+
if (!isTypeValid(parsedType.type)) {
|
|
434
503
|
throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
|
|
435
504
|
}
|
|
505
|
+
this.#supportedMimeTypes.add(parsedType.type);
|
|
436
506
|
}
|
|
437
507
|
}
|
|
438
508
|
/**
|
|
439
509
|
* Handles timeout request
|
|
440
510
|
*/
|
|
441
|
-
|
|
442
|
-
session
|
|
443
|
-
throw new Error(`
|
|
511
|
+
#handleRequestTimeout(session) {
|
|
512
|
+
session.markBad();
|
|
513
|
+
throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
444
514
|
}
|
|
445
|
-
|
|
515
|
+
#abortDownloadOfBody(request, response) {
|
|
446
516
|
const { status } = response;
|
|
447
517
|
const { type } = parseContentTypeFromResponse(response);
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
// if we retry the request, can the Content-Type change?
|
|
451
|
-
const isTransientContentType = status >= 500 || blockedStatusCodes.includes(status);
|
|
452
|
-
if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
|
|
518
|
+
const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
|
|
519
|
+
if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
|
|
453
520
|
request.noRetry = true;
|
|
454
521
|
throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
|
|
455
|
-
`but only ${Array.from(this
|
|
522
|
+
`but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
|
|
456
523
|
}
|
|
457
524
|
}
|
|
458
525
|
/**
|
|
459
526
|
* @internal wraps public utility for mocking purposes
|
|
460
527
|
*/
|
|
461
|
-
|
|
528
|
+
#requestAsBrowser = async (options, session, signal) => {
|
|
462
529
|
const opts = processHttpRequestOptions({
|
|
463
530
|
...options,
|
|
464
|
-
cookieJar: options.cookieJar,
|
|
465
531
|
responseType: 'text',
|
|
466
532
|
});
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
533
|
+
// When saveResponseCookies is false, the response cookies must not mutate the
|
|
534
|
+
// session jar. Reads still go through the session (so session.setCookie() in pre-nav
|
|
535
|
+
// hooks keeps working) but a per-request clone is passed in so writes are discarded.
|
|
536
|
+
const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
|
|
537
|
+
// Bind the request to the shared navigation window instead of a fixed per-request timeout, so
|
|
538
|
+
// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
|
|
539
|
+
// kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
|
|
540
|
+
// bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
|
|
541
|
+
// body still fails cleanly with a navigation timeout and its download stops too.
|
|
542
|
+
const cancelSignal = storage.getStore()?.cancelTask.signal;
|
|
543
|
+
const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
|
|
472
544
|
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
473
545
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
474
546
|
headers: new Headers(opts.headers),
|
|
@@ -477,36 +549,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
477
549
|
duplex: 'half',
|
|
478
550
|
}), {
|
|
479
551
|
session,
|
|
480
|
-
|
|
552
|
+
cookieJar,
|
|
553
|
+
proxyUrl: opts.proxyUrl,
|
|
554
|
+
signal: requestSignal,
|
|
555
|
+
timeoutMillis: cancelSignal ? undefined : opts.timeout,
|
|
556
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
481
557
|
});
|
|
482
558
|
return response;
|
|
483
559
|
};
|
|
484
560
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
* This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
|
|
488
|
-
* Defaults to the {@link HttpCrawlingContext}.
|
|
489
|
-
*
|
|
490
|
-
* > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
|
|
491
|
-
*
|
|
492
|
-
* ```ts
|
|
493
|
-
* import { HttpCrawler, createHttpRouter } from 'crawlee';
|
|
494
|
-
*
|
|
495
|
-
* const router = createHttpRouter();
|
|
496
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
497
|
-
* ctx.log.info('...');
|
|
498
|
-
* });
|
|
499
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
500
|
-
* ctx.log.info('...');
|
|
501
|
-
* });
|
|
502
|
-
*
|
|
503
|
-
* const crawler = new HttpCrawler({
|
|
504
|
-
* requestHandler: router,
|
|
505
|
-
* });
|
|
506
|
-
* await crawler.run();
|
|
507
|
-
* ```
|
|
508
|
-
*/
|
|
509
|
-
export function createHttpRouter(routes) {
|
|
510
|
-
return Router.create(routes);
|
|
561
|
+
export function createHttpRouter(routesOrSchemas) {
|
|
562
|
+
return Router.create(routesOrSchemas);
|
|
511
563
|
}
|
|
512
|
-
//# sourceMappingURL=http-crawler.js.map
|