@crawlee/browser 4.0.0-beta.261 → 4.0.0-beta.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/browser-crawler.js +47 -15
- package/package.json +6 -6
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
|
|
2
|
-
import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
|
|
2
|
+
import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
|
|
3
3
|
import { sleep } from '@crawlee/utils';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
|
|
@@ -150,23 +150,30 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
150
150
|
return ContextPipeline.create().compose(this.#preparePage.bind(this));
|
|
151
151
|
}
|
|
152
152
|
async #containsSelectors(page, selectors) {
|
|
153
|
-
const
|
|
153
|
+
const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
|
|
154
|
+
// a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
|
|
155
|
+
const elements = await query().catch(async () => {
|
|
156
|
+
await sleep(1000);
|
|
157
|
+
return query();
|
|
158
|
+
});
|
|
159
|
+
const foundSelectors = elements
|
|
154
160
|
.map((x, i) => [x, selectors[i]])
|
|
155
161
|
.filter(([x]) => x !== null)
|
|
156
162
|
.map(([, selector]) => selector);
|
|
157
163
|
return foundSelectors.length > 0 ? foundSelectors : null;
|
|
158
164
|
}
|
|
165
|
+
async #detectSelfResolvingChallenge({ page, response, }) {
|
|
166
|
+
const statusCode = response?.status();
|
|
167
|
+
if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
|
|
168
|
+
return 'Cloudflare';
|
|
169
|
+
}
|
|
170
|
+
if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
|
|
171
|
+
return 'AWS WAF';
|
|
172
|
+
}
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
159
175
|
async #isRequestBlocked(crawlingContext) {
|
|
160
176
|
const { page, response } = crawlingContext;
|
|
161
|
-
// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
|
|
162
|
-
if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
|
|
163
|
-
await sleep(5000);
|
|
164
|
-
// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
|
|
165
|
-
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
166
|
-
if (!foundSelectors)
|
|
167
|
-
return false;
|
|
168
|
-
return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
|
|
169
|
-
}
|
|
170
177
|
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
171
178
|
const statusCode = response?.status() ?? 0;
|
|
172
179
|
if (foundSelectors)
|
|
@@ -342,11 +349,36 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
342
349
|
}
|
|
343
350
|
}
|
|
344
351
|
async #handleBlockedRequestByContent(crawlingContext) {
|
|
345
|
-
if (this.retryOnBlocked)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
352
|
+
if (!this.retryOnBlocked)
|
|
353
|
+
return {};
|
|
354
|
+
const page = crawlingContext.page;
|
|
355
|
+
// A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
|
|
356
|
+
let navigationResponse;
|
|
357
|
+
const onResponse = (response) => {
|
|
358
|
+
if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
|
|
359
|
+
navigationResponse = response;
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
page.on('response', onResponse);
|
|
363
|
+
try {
|
|
364
|
+
// Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
|
|
365
|
+
// Wait 5 seconds for the JS challenge to load / resolve before judging the page.
|
|
366
|
+
const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
|
|
367
|
+
if (challenge) {
|
|
368
|
+
await sleep(5000);
|
|
369
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
370
|
+
if (foundSelectors) {
|
|
371
|
+
throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
|
|
372
|
+
}
|
|
373
|
+
return navigationResponse ? { response: navigationResponse } : {};
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
page.off('response', onResponse);
|
|
349
378
|
}
|
|
379
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
380
|
+
if (error)
|
|
381
|
+
throw new SessionError(error);
|
|
350
382
|
return {};
|
|
351
383
|
}
|
|
352
384
|
async #restoreRequestState(crawlingContext) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.263",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.263",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.263",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.263",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.263",
|
|
55
55
|
"tslib": "^2.9.1",
|
|
56
56
|
"zod": "^4.6.7"
|
|
57
57
|
},
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "7303c694793b8f5cc17f5b16328f435fae121668"
|
|
78
78
|
}
|