@crawlee/http 4.0.0-beta.25 → 4.0.0-beta.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +14 -51
- package/internals/file-download.js +41 -109
- package/internals/http-crawler.d.ts +221 -228
- package/internals/http-crawler.js +287 -228
- package/internals/utils.d.ts +6 -1
- package/internals/utils.js +21 -15
- package/package.json +17 -18
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/file-download.d.ts.map +0 -1
- package/internals/file-download.js.map +0 -1
- package/internals/http-crawler.d.ts.map +0 -1
- package/internals/http-crawler.js.map +0 -1
- package/internals/utils.d.ts.map +0 -1
- package/internals/utils.js.map +0 -1
|
@@ -1,27 +1,38 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import { BasicCrawler,
|
|
3
|
+
import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
|
|
4
4
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
5
|
-
import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
|
|
6
|
-
import
|
|
7
|
-
import contentTypeParser from 'content-type';
|
|
5
|
+
import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
|
|
6
|
+
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
8
7
|
import iconv from 'iconv-lite';
|
|
9
|
-
import
|
|
10
|
-
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
|
-
import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
|
|
8
|
+
import { z } from 'zod';
|
|
9
|
+
import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
10
|
+
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
|
|
12
11
|
/**
|
|
13
12
|
* Default mime types, which HttpScraper supports.
|
|
14
13
|
*/
|
|
15
14
|
const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
|
|
16
15
|
const APPLICATION_JSON_MIME_TYPE = 'application/json';
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
18
|
+
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
|
|
19
|
+
* concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
|
|
20
|
+
*
|
|
21
|
+
* A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
|
|
22
|
+
* wholesale, tuning included, so spread these options in if you want to keep it:
|
|
23
|
+
*
|
|
24
|
+
* ```typescript
|
|
25
|
+
* new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
|
|
18
29
|
desiredConcurrency: 10,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
30
|
+
loadSignals: {
|
|
31
|
+
eventLoop: {
|
|
32
|
+
snapshotIntervalSecs: 2,
|
|
33
|
+
maxBlockedMillis: 100,
|
|
34
|
+
overloadedRatio: 0.7,
|
|
35
|
+
},
|
|
25
36
|
},
|
|
26
37
|
};
|
|
27
38
|
/**
|
|
@@ -35,13 +46,15 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
35
46
|
*
|
|
36
47
|
* This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
|
|
37
48
|
*
|
|
38
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
39
|
-
* {@link
|
|
40
|
-
*
|
|
49
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
50
|
+
* {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
|
|
51
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
52
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
53
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
54
|
+
* result as `requestManager`.
|
|
41
55
|
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
56
|
+
* > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
|
|
57
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
45
58
|
*
|
|
46
59
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
47
60
|
*
|
|
@@ -55,18 +68,18 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
55
68
|
* ]
|
|
56
69
|
* ```
|
|
57
70
|
*
|
|
58
|
-
* By default, this crawler only processes web pages with the `text/html`
|
|
59
|
-
* and `application/
|
|
71
|
+
* By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
72
|
+
* and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
|
|
60
73
|
* and skips pages with other content types. If you want the crawler to process other content types,
|
|
61
74
|
* use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
|
|
62
75
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
63
76
|
* For details, see {@link HttpCrawlerOptions.requestHandler}.
|
|
64
77
|
*
|
|
65
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* {@link
|
|
78
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
79
|
+
* {@link ConcurrencySystem}.
|
|
80
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
81
|
+
* constructor, or, for finer control, by injecting a pre-configured
|
|
82
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
70
83
|
*
|
|
71
84
|
* **Example usage:**
|
|
72
85
|
*
|
|
@@ -92,157 +105,238 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
92
105
|
* @category Crawlers
|
|
93
106
|
*/
|
|
94
107
|
export class HttpCrawler extends BasicCrawler {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
108
|
+
// Internal storage uses the base (non-extended) context types. The public option types are
|
|
109
|
+
// extension-aware for consumer DX, but internally the pipeline composes hooks against the
|
|
110
|
+
// concrete crawling context, which does not statically carry `ContextExtension`. The members
|
|
111
|
+
// added by `extendContext` are present at runtime regardless.
|
|
112
|
+
#preNavigationHooks;
|
|
113
|
+
#postNavigationHooks;
|
|
114
|
+
#saveResponseCookies;
|
|
115
|
+
#navigationTimeoutMillis;
|
|
116
|
+
#ignoreTlsErrors;
|
|
117
|
+
#suggestResponseEncoding;
|
|
118
|
+
#forceResponseEncoding;
|
|
119
|
+
#supportedMimeTypes;
|
|
120
|
+
#downloadAbortControllers = new WeakMap();
|
|
121
|
+
/**
|
|
122
|
+
* @internal
|
|
123
|
+
*/
|
|
106
124
|
static optionsShape = {
|
|
107
125
|
...BasicCrawler.optionsShape,
|
|
108
|
-
navigationTimeoutSecs:
|
|
109
|
-
|
|
110
|
-
additionalMimeTypes:
|
|
111
|
-
suggestResponseEncoding:
|
|
112
|
-
forceResponseEncoding:
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
preNavigationHooks: ow.optional.array,
|
|
117
|
-
postNavigationHooks: ow.optional.array,
|
|
126
|
+
navigationTimeoutSecs: schemas.anyNumber.default(30),
|
|
127
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
128
|
+
additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
129
|
+
suggestResponseEncoding: z.string().optional(),
|
|
130
|
+
forceResponseEncoding: z.string().optional(),
|
|
131
|
+
saveResponseCookies: z.boolean().default(true),
|
|
132
|
+
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
133
|
+
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
118
134
|
};
|
|
135
|
+
/** @internal */
|
|
136
|
+
static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
|
|
119
137
|
/**
|
|
120
138
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
121
139
|
*/
|
|
122
|
-
constructor(options = {}
|
|
123
|
-
|
|
124
|
-
const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
|
|
140
|
+
constructor(options = {}) {
|
|
141
|
+
const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks,
|
|
125
142
|
// BasicCrawler
|
|
126
|
-
|
|
143
|
+
contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
|
|
127
144
|
super({
|
|
128
145
|
...basicCrawlerOptions,
|
|
129
|
-
autoscaledPoolOptions,
|
|
130
146
|
contextPipelineBuilder: contextPipelineBuilder ??
|
|
131
147
|
(() => this.buildContextPipeline()),
|
|
132
|
-
}
|
|
133
|
-
this
|
|
134
|
-
// Cookies should be persisted per session only if session pool is used
|
|
135
|
-
if (!this.useSessionPool && persistCookiesPerSession) {
|
|
136
|
-
throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
|
|
137
|
-
}
|
|
138
|
-
this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
|
|
148
|
+
});
|
|
149
|
+
this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
|
|
139
150
|
if (additionalMimeTypes.length)
|
|
140
|
-
this
|
|
151
|
+
this.#extendSupportedMimeTypes(additionalMimeTypes);
|
|
141
152
|
if (suggestResponseEncoding && forceResponseEncoding) {
|
|
142
153
|
this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
|
|
143
154
|
}
|
|
144
|
-
this
|
|
145
|
-
this
|
|
146
|
-
this
|
|
147
|
-
this
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
this
|
|
152
|
-
|
|
155
|
+
this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
|
|
156
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
157
|
+
this.#suggestResponseEncoding = suggestResponseEncoding;
|
|
158
|
+
this.#forceResponseEncoding = forceResponseEncoding;
|
|
159
|
+
// Cast away the extension-aware option types to the base internal storage types (see the field
|
|
160
|
+
// declarations above). This is sound - the hooks only ever receive the base context plus the
|
|
161
|
+
// members `extendContext` added at runtime.
|
|
162
|
+
this.#preNavigationHooks = preNavigationHooks;
|
|
163
|
+
this.#postNavigationHooks = [
|
|
164
|
+
({ request, response }) => this.#abortDownloadOfBody(request, response),
|
|
153
165
|
...postNavigationHooks,
|
|
154
166
|
];
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
167
|
+
this.#saveResponseCookies = saveResponseCookies;
|
|
168
|
+
}
|
|
169
|
+
/** @internal */
|
|
170
|
+
getNavigationTimeoutMillis() {
|
|
171
|
+
return this.#navigationTimeoutMillis;
|
|
172
|
+
}
|
|
173
|
+
/** @internal */
|
|
174
|
+
createDefaultConcurrencySystem(options) {
|
|
175
|
+
return super.createDefaultConcurrencySystem({
|
|
176
|
+
...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
|
|
177
|
+
...options,
|
|
178
|
+
});
|
|
161
179
|
}
|
|
162
180
|
buildContextPipeline() {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
181
|
+
// When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
|
|
182
|
+
// the response-derived members, so the guarded action is bypassed and the context left untouched.
|
|
183
|
+
const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
|
|
184
|
+
// A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
|
|
185
|
+
// hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
|
|
186
|
+
// window the navigation uses instead of each step being timed on its own.
|
|
187
|
+
const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
|
|
188
|
+
const windowGuard = (step) => skipGuard(async (ctx) => {
|
|
189
|
+
const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
|
|
190
|
+
if (remaining <= 0) {
|
|
191
|
+
throw new TimeoutError(navigationTimedOut);
|
|
192
|
+
}
|
|
193
|
+
return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
|
|
194
|
+
});
|
|
195
|
+
let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
|
|
196
|
+
for (const hook of this.#preNavigationHooks) {
|
|
197
|
+
pipeline = pipeline.compose(windowGuard(hook));
|
|
198
|
+
}
|
|
199
|
+
let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
|
|
200
|
+
for (const hook of this.#postNavigationHooks) {
|
|
201
|
+
pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
|
|
202
|
+
}
|
|
203
|
+
return pipelineWithNavigation
|
|
204
|
+
.compose(this.#processHttpResponse.bind(this))
|
|
205
|
+
.compose(this.#handleBlockedRequestByContent.bind(this));
|
|
169
206
|
}
|
|
170
|
-
async
|
|
171
|
-
const { request
|
|
207
|
+
async #prepareHttpRequest(crawlingContext) {
|
|
208
|
+
const { request } = crawlingContext;
|
|
172
209
|
if (request.skipNavigation) {
|
|
173
210
|
return {
|
|
174
211
|
request: new Proxy(request, {
|
|
175
|
-
get(target, propertyName
|
|
212
|
+
get(target, propertyName) {
|
|
176
213
|
if (propertyName === 'loadedUrl') {
|
|
177
|
-
throw new
|
|
214
|
+
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
178
215
|
}
|
|
179
|
-
|
|
216
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
217
|
+
const value = Reflect.get(target, propertyName, target);
|
|
218
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
180
219
|
},
|
|
220
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
181
221
|
}),
|
|
182
222
|
get response() {
|
|
183
|
-
throw new
|
|
223
|
+
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
184
224
|
},
|
|
185
225
|
};
|
|
186
226
|
}
|
|
187
|
-
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
|
|
188
227
|
request.state = RequestState.BEFORE_NAV;
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
228
|
+
return {};
|
|
229
|
+
}
|
|
230
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
231
|
+
async makeHttpRequest(crawlingContext) {
|
|
192
232
|
tryCancel();
|
|
193
|
-
const
|
|
194
|
-
const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
|
|
233
|
+
const { request, session } = crawlingContext;
|
|
195
234
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
196
|
-
const
|
|
235
|
+
const downloadAbortController = new AbortController();
|
|
236
|
+
this.#downloadAbortControllers.set(request, downloadAbortController);
|
|
237
|
+
// Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
|
|
238
|
+
// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
|
|
239
|
+
// client abort.
|
|
240
|
+
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
197
241
|
tryCancel();
|
|
198
242
|
request.loadedUrl = httpResponse?.url;
|
|
199
243
|
request.state = RequestState.AFTER_NAV;
|
|
200
244
|
return { request: request, response: httpResponse };
|
|
201
245
|
}
|
|
202
|
-
async processHttpResponse(crawlingContext) {
|
|
246
|
+
async #processHttpResponse(crawlingContext) {
|
|
203
247
|
if (crawlingContext.request.skipNavigation) {
|
|
204
248
|
return {
|
|
205
249
|
get contentType() {
|
|
206
|
-
throw new
|
|
250
|
+
throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
|
|
207
251
|
},
|
|
208
252
|
get body() {
|
|
209
|
-
throw new
|
|
253
|
+
throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
|
|
210
254
|
},
|
|
211
255
|
get json() {
|
|
212
|
-
throw new
|
|
256
|
+
throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
|
|
213
257
|
},
|
|
214
258
|
get waitForSelector() {
|
|
215
|
-
throw new
|
|
259
|
+
throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
|
|
216
260
|
},
|
|
217
261
|
get parseWithCheerio() {
|
|
218
|
-
throw new
|
|
262
|
+
throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
|
|
263
|
+
},
|
|
264
|
+
get querySelectorAll() {
|
|
265
|
+
throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
|
|
266
|
+
},
|
|
267
|
+
get querySelectorOne() {
|
|
268
|
+
throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
|
|
219
269
|
},
|
|
220
270
|
};
|
|
221
271
|
}
|
|
222
|
-
await this._executeHooks(this.postNavigationHooks, crawlingContext);
|
|
223
272
|
tryCancel();
|
|
224
|
-
|
|
273
|
+
// Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
|
|
274
|
+
// error is still a rate limit the domain should back off from.
|
|
275
|
+
if (crawlingContext.response.status === 429) {
|
|
276
|
+
const retryAfter = crawlingContext.response.headers.get('retry-after');
|
|
277
|
+
if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
|
|
278
|
+
// This is the one path that never reads the body, so cancel it to release the connection
|
|
279
|
+
// rather than leaving it to the garbage collector.
|
|
280
|
+
await crawlingContext.response.body?.cancel().catch(() => { });
|
|
281
|
+
throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Reading the body is still part of the navigation, so it draws from the same shared window: on a server
|
|
285
|
+
// that streams the body slowly the request completes (headers arrive) but the body read would otherwise
|
|
286
|
+
// run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
|
|
287
|
+
const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
|
|
288
|
+
if (remaining <= 0) {
|
|
289
|
+
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
290
|
+
}
|
|
291
|
+
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
|
|
292
|
+
// Stop the download along with the read, so a body that never ends doesn't outlive the request.
|
|
293
|
+
this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
|
|
294
|
+
throw error;
|
|
295
|
+
});
|
|
225
296
|
tryCancel();
|
|
226
297
|
const response = parsed.response;
|
|
227
298
|
const contentType = parsed.contentType;
|
|
228
|
-
const
|
|
229
|
-
const
|
|
230
|
-
|
|
299
|
+
const loadBody = async () => {
|
|
300
|
+
const { load } = await import('cheerio/slim');
|
|
301
|
+
return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
|
|
302
|
+
};
|
|
303
|
+
const querySelectorAll = async (selector, _timeoutMs) => {
|
|
304
|
+
const matches = (await loadBody())(selector);
|
|
305
|
+
if (matches.length === 0) {
|
|
231
306
|
throw new Error(`Selector '${selector}' not found.`);
|
|
232
307
|
}
|
|
308
|
+
return matches;
|
|
309
|
+
};
|
|
310
|
+
const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
|
|
311
|
+
const waitForSelector = async (selector, timeoutMs) => {
|
|
312
|
+
await querySelectorAll(selector, timeoutMs);
|
|
233
313
|
};
|
|
234
314
|
const parseWithCheerio = async (selector, timeoutMs) => {
|
|
235
|
-
const $ =
|
|
315
|
+
const $ = await loadBody();
|
|
236
316
|
if (selector) {
|
|
237
317
|
await crawlingContext.waitForSelector(selector, timeoutMs);
|
|
238
318
|
}
|
|
239
319
|
return $;
|
|
240
320
|
};
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
321
|
+
this.throwOnBlockedRequest(response.status);
|
|
322
|
+
if (this.#saveResponseCookies) {
|
|
323
|
+
try {
|
|
324
|
+
for (const cookie of getCookiesFromResponse(response)) {
|
|
325
|
+
if (!cookie)
|
|
326
|
+
continue;
|
|
327
|
+
try {
|
|
328
|
+
await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
|
|
329
|
+
ignoreError: false,
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
catch (e) {
|
|
333
|
+
this.log.debug(`Could not set cookie: ${e.message}`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
catch (e) {
|
|
338
|
+
this.log.exception(e, 'Could not get cookies from response');
|
|
339
|
+
}
|
|
246
340
|
}
|
|
247
341
|
return {
|
|
248
342
|
get json() {
|
|
@@ -252,20 +346,22 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
252
346
|
return JSON.parse(jsonString);
|
|
253
347
|
},
|
|
254
348
|
waitForSelector,
|
|
349
|
+
querySelectorOne,
|
|
350
|
+
querySelectorAll,
|
|
255
351
|
parseWithCheerio,
|
|
256
352
|
contentType,
|
|
257
353
|
body: parsed.body,
|
|
258
354
|
};
|
|
259
355
|
}
|
|
260
|
-
async handleBlockedRequestByContent(crawlingContext) {
|
|
356
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
261
357
|
if (this.retryOnBlocked) {
|
|
262
|
-
const error = await this
|
|
358
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
263
359
|
if (error)
|
|
264
360
|
throw new SessionError(error);
|
|
265
361
|
}
|
|
266
362
|
return {};
|
|
267
363
|
}
|
|
268
|
-
async isRequestBlocked(crawlingContext) {
|
|
364
|
+
async #isRequestBlocked(crawlingContext) {
|
|
269
365
|
if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
|
|
270
366
|
const $ = await crawlingContext.parseWithCheerio();
|
|
271
367
|
const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
|
|
@@ -273,43 +369,28 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
273
369
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
274
370
|
}
|
|
275
371
|
}
|
|
276
|
-
|
|
277
|
-
// eslint-disable-next-line dot-notation
|
|
278
|
-
(this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
|
|
279
|
-
? // eslint-disable-next-line dot-notation
|
|
280
|
-
this.sessionPool['blockedStatusCodes']
|
|
281
|
-
: BLOCKED_STATUS_CODES;
|
|
282
|
-
if (blockedStatusCodes.includes(crawlingContext.response.status)) {
|
|
372
|
+
if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
|
|
283
373
|
return `Blocked by status code ${crawlingContext.response.status}`;
|
|
284
374
|
}
|
|
285
375
|
return false;
|
|
286
376
|
}
|
|
287
|
-
/**
|
|
288
|
-
* Returns the `Cookie` header value based on the current context and
|
|
289
|
-
* any changes that occurred in the navigation hooks.
|
|
290
|
-
*/
|
|
291
|
-
_applyCookies({ session, request }, preHookCookies, postHookCookies) {
|
|
292
|
-
const sessionCookie = session?.getCookieString(request.url) ?? '';
|
|
293
|
-
const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
|
|
294
|
-
return mergeCookies(request.url, sourceCookies);
|
|
295
|
-
}
|
|
296
377
|
/**
|
|
297
378
|
* Function to make the HTTP request. It performs optimizations
|
|
298
379
|
* on the request such as only downloading the request body if the
|
|
299
380
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
300
381
|
*/
|
|
301
|
-
async
|
|
302
|
-
const opts = this
|
|
382
|
+
async #requestFunction({ request, session, proxyUrl, signal }) {
|
|
383
|
+
const opts = this.#getRequestOptions(request, proxyUrl);
|
|
303
384
|
try {
|
|
304
|
-
return await this
|
|
385
|
+
return await this.#requestAsBrowser(opts, session, signal);
|
|
305
386
|
}
|
|
306
387
|
catch (e) {
|
|
307
388
|
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
308
|
-
this
|
|
309
|
-
return new Response(); // this will never happen, as
|
|
389
|
+
this.#handleRequestTimeout(session);
|
|
390
|
+
return new Response(); // this will never happen, as handleRequestTimeout always throws
|
|
310
391
|
}
|
|
311
392
|
if (this.isProxyError(e)) {
|
|
312
|
-
throw new SessionError(this.
|
|
393
|
+
throw new SessionError(this.getMessageFromError(e));
|
|
313
394
|
}
|
|
314
395
|
else {
|
|
315
396
|
throw e;
|
|
@@ -319,18 +400,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
319
400
|
/**
|
|
320
401
|
* Encodes and parses response according to the provided content type
|
|
321
402
|
*/
|
|
322
|
-
async
|
|
403
|
+
async #parseResponse(request, response) {
|
|
323
404
|
const { status } = response;
|
|
324
405
|
const { type, charset } = parseContentTypeFromResponse(response);
|
|
325
|
-
const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
|
|
326
|
-
const contentType = { type, encoding };
|
|
327
406
|
if (status >= 400 && status <= 599) {
|
|
328
|
-
this.
|
|
407
|
+
this.statistics.registerStatusCode(status);
|
|
329
408
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
409
|
+
if (this.isErrorStatusCode(status)) {
|
|
410
|
+
const { response: decoded } = this.#encodeResponse(request, response, charset);
|
|
411
|
+
const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
334
412
|
// Errors are often sent as JSON, so attempt to parse them,
|
|
335
413
|
// despite Accept header being set to text/html.
|
|
336
414
|
if (type === APPLICATION_JSON_MIME_TYPE) {
|
|
@@ -340,60 +418,60 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
340
418
|
message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
|
|
341
419
|
throw new Error(`${status} - ${message}`);
|
|
342
420
|
}
|
|
343
|
-
if (
|
|
421
|
+
if (this.additionalHttpErrorStatusCodes.has(status)) {
|
|
344
422
|
throw new Error(`${status} - Error status code was set by user.`);
|
|
345
423
|
}
|
|
346
424
|
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
|
|
347
425
|
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
|
|
348
426
|
}
|
|
349
|
-
|
|
350
|
-
|
|
427
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
|
|
428
|
+
// The charset comes from the document itself, so the raw bytes are what we need -
|
|
429
|
+
// decoding them through `encodeResponse` first would consume the body for nothing.
|
|
430
|
+
const rawBytes = Buffer.from(await response.arrayBuffer());
|
|
431
|
+
const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
|
|
432
|
+
const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
|
|
433
|
+
const body = iconv.encodingExists(charsetToUse)
|
|
434
|
+
? iconv.decode(rawBytes, charsetToUse)
|
|
435
|
+
: rawBytes.toString('utf8');
|
|
436
|
+
return { response, contentType: { type, encoding: 'utf-8' }, body };
|
|
351
437
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
response,
|
|
357
|
-
contentType,
|
|
358
|
-
};
|
|
438
|
+
const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
|
|
439
|
+
const contentType = { type, encoding };
|
|
440
|
+
if (HTML_AND_XML_MIME_TYPES.includes(type)) {
|
|
441
|
+
return { response, contentType, body: await reencodedResponse.text() };
|
|
359
442
|
}
|
|
443
|
+
return {
|
|
444
|
+
body: Buffer.from(await reencodedResponse.bytes()),
|
|
445
|
+
response,
|
|
446
|
+
contentType,
|
|
447
|
+
};
|
|
360
448
|
}
|
|
361
449
|
/**
|
|
362
450
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
363
451
|
*/
|
|
364
|
-
|
|
452
|
+
#getRequestOptions(request, proxyUrl) {
|
|
365
453
|
const requestOptions = {
|
|
366
454
|
url: request.url,
|
|
367
455
|
method: request.method,
|
|
368
456
|
proxyUrl,
|
|
369
|
-
timeout: this
|
|
370
|
-
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
|
|
371
|
-
sessionToken: session,
|
|
457
|
+
timeout: this.#navigationTimeoutMillis,
|
|
372
458
|
headers: request.headers,
|
|
373
|
-
https: {
|
|
374
|
-
rejectUnauthorized: !this.ignoreSslErrors,
|
|
375
|
-
},
|
|
376
459
|
body: undefined,
|
|
377
460
|
};
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
if (session?.proxyInfo?.ignoreTlsErrors) {
|
|
382
|
-
requestOptions.https = {
|
|
383
|
-
...requestOptions.https,
|
|
384
|
-
rejectUnauthorized: false,
|
|
385
|
-
};
|
|
461
|
+
if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
|
|
462
|
+
requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
|
|
463
|
+
delete requestOptions.headers.cookie;
|
|
386
464
|
}
|
|
387
465
|
if (/PATCH|POST|PUT/.test(request.method))
|
|
388
466
|
requestOptions.body = request.payload ?? '';
|
|
389
467
|
return requestOptions;
|
|
390
468
|
}
|
|
391
|
-
|
|
392
|
-
if (this
|
|
393
|
-
encoding = this
|
|
469
|
+
#encodeResponse(request, response, encoding) {
|
|
470
|
+
if (this.#forceResponseEncoding) {
|
|
471
|
+
encoding = this.#forceResponseEncoding;
|
|
394
472
|
}
|
|
395
|
-
else if (!encoding && this
|
|
396
|
-
encoding = this
|
|
473
|
+
else if (!encoding && this.#suggestResponseEncoding) {
|
|
474
|
+
encoding = this.#suggestResponseEncoding;
|
|
397
475
|
}
|
|
398
476
|
// Fall back to utf-8 if we still don't have encoding.
|
|
399
477
|
const utf8 = 'utf8';
|
|
@@ -406,7 +484,9 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
406
484
|
// Try to re-encode a variety of unsupported encodings to utf-8
|
|
407
485
|
if (iconv.encodingExists(encoding)) {
|
|
408
486
|
const encodeStream = iconv.encodeStream(utf8);
|
|
409
|
-
const decodeStream = iconv
|
|
487
|
+
const decodeStream = iconv
|
|
488
|
+
.decodeStream(encoding)
|
|
489
|
+
.on('error', (err) => encodeStream.emit('error', err));
|
|
410
490
|
const reencodedBody = response.body
|
|
411
491
|
? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
|
|
412
492
|
.pipe(decodeStream)
|
|
@@ -422,55 +502,55 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
422
502
|
/**
|
|
423
503
|
* Checks and extends supported mime types
|
|
424
504
|
*/
|
|
425
|
-
|
|
505
|
+
#extendSupportedMimeTypes(additionalMimeTypes) {
|
|
426
506
|
for (const mimeType of additionalMimeTypes) {
|
|
427
507
|
if (mimeType === '*/*') {
|
|
428
|
-
this
|
|
508
|
+
this.#supportedMimeTypes.add(mimeType);
|
|
429
509
|
continue;
|
|
430
510
|
}
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
this.supportedMimeTypes.add(parsedType.type);
|
|
434
|
-
}
|
|
435
|
-
catch (err) {
|
|
511
|
+
const parsedType = parseContentType(mimeType);
|
|
512
|
+
if (!isTypeValid(parsedType.type)) {
|
|
436
513
|
throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
|
|
437
514
|
}
|
|
515
|
+
this.#supportedMimeTypes.add(parsedType.type);
|
|
438
516
|
}
|
|
439
517
|
}
|
|
440
518
|
/**
|
|
441
519
|
* Handles timeout request
|
|
442
520
|
*/
|
|
443
|
-
|
|
444
|
-
session
|
|
445
|
-
throw new Error(`
|
|
521
|
+
#handleRequestTimeout(session) {
|
|
522
|
+
session.markBad();
|
|
523
|
+
throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
446
524
|
}
|
|
447
|
-
|
|
525
|
+
#abortDownloadOfBody(request, response) {
|
|
448
526
|
const { status } = response;
|
|
449
527
|
const { type } = parseContentTypeFromResponse(response);
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
// if we retry the request, can the Content-Type change?
|
|
453
|
-
const isTransientContentType = status >= 500 || blockedStatusCodes.includes(status);
|
|
454
|
-
if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
|
|
528
|
+
const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
|
|
529
|
+
if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
|
|
455
530
|
request.noRetry = true;
|
|
456
531
|
throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
|
|
457
|
-
`but only ${Array.from(this
|
|
532
|
+
`but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
|
|
458
533
|
}
|
|
459
534
|
}
|
|
460
535
|
/**
|
|
461
536
|
* @internal wraps public utility for mocking purposes
|
|
462
537
|
*/
|
|
463
|
-
|
|
538
|
+
#requestAsBrowser = async (options, session, signal) => {
|
|
464
539
|
const opts = processHttpRequestOptions({
|
|
465
540
|
...options,
|
|
466
|
-
cookieJar: options.cookieJar,
|
|
467
541
|
responseType: 'text',
|
|
468
542
|
});
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
543
|
+
// When saveResponseCookies is false, the response cookies must not mutate the
|
|
544
|
+
// session jar. Reads still go through the session (so session.setCookie() in pre-nav
|
|
545
|
+
// hooks keeps working) but a per-request clone is passed in so writes are discarded.
|
|
546
|
+
const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
|
|
547
|
+
// Bind the request to the shared navigation window instead of a fixed per-request timeout, so
|
|
548
|
+
// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
|
|
549
|
+
// kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
|
|
550
|
+
// bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
|
|
551
|
+
// body still fails cleanly with a navigation timeout and its download stops too.
|
|
552
|
+
const cancelSignal = storage.getStore()?.cancelTask.signal;
|
|
553
|
+
const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
|
|
474
554
|
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
475
555
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
476
556
|
headers: new Headers(opts.headers),
|
|
@@ -479,36 +559,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
479
559
|
duplex: 'half',
|
|
480
560
|
}), {
|
|
481
561
|
session,
|
|
482
|
-
|
|
562
|
+
cookieJar,
|
|
563
|
+
proxyUrl: opts.proxyUrl,
|
|
564
|
+
signal: requestSignal,
|
|
565
|
+
timeoutMillis: cancelSignal ? undefined : opts.timeout,
|
|
566
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
483
567
|
});
|
|
484
568
|
return response;
|
|
485
569
|
};
|
|
486
570
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
* This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
|
|
490
|
-
* Defaults to the {@link HttpCrawlingContext}.
|
|
491
|
-
*
|
|
492
|
-
* > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
|
|
493
|
-
*
|
|
494
|
-
* ```ts
|
|
495
|
-
* import { HttpCrawler, createHttpRouter } from 'crawlee';
|
|
496
|
-
*
|
|
497
|
-
* const router = createHttpRouter();
|
|
498
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
499
|
-
* ctx.log.info('...');
|
|
500
|
-
* });
|
|
501
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
502
|
-
* ctx.log.info('...');
|
|
503
|
-
* });
|
|
504
|
-
*
|
|
505
|
-
* const crawler = new HttpCrawler({
|
|
506
|
-
* requestHandler: router,
|
|
507
|
-
* });
|
|
508
|
-
* await crawler.run();
|
|
509
|
-
* ```
|
|
510
|
-
*/
|
|
511
|
-
export function createHttpRouter(routes) {
|
|
512
|
-
return Router.create(routes);
|
|
571
|
+
export function createHttpRouter(routesOrSchemas) {
|
|
572
|
+
return Router.create(routesOrSchemas);
|
|
513
573
|
}
|
|
514
|
-
//# sourceMappingURL=http-crawler.js.map
|