@crawlee/browser 4.0.0-beta.261 → 4.0.0-beta.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
- import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
2
+ import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
3
  import { sleep } from '@crawlee/utils';
4
4
  import { z } from 'zod';
5
5
  import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
@@ -150,23 +150,30 @@ export class BrowserCrawler extends BasicCrawler {
150
150
  return ContextPipeline.create().compose(this.#preparePage.bind(this));
151
151
  }
152
152
  async #containsSelectors(page, selectors) {
153
- const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
153
+ const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
154
+ // a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
155
+ const elements = await query().catch(async () => {
156
+ await sleep(1000);
157
+ return query();
158
+ });
159
+ const foundSelectors = elements
154
160
  .map((x, i) => [x, selectors[i]])
155
161
  .filter(([x]) => x !== null)
156
162
  .map(([, selector]) => selector);
157
163
  return foundSelectors.length > 0 ? foundSelectors : null;
158
164
  }
165
+ async #detectSelfResolvingChallenge({ page, response, }) {
166
+ const statusCode = response?.status();
167
+ if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
168
+ return 'Cloudflare';
169
+ }
170
+ if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
171
+ return 'AWS WAF';
172
+ }
173
+ return null;
174
+ }
159
175
  async #isRequestBlocked(crawlingContext) {
160
176
  const { page, response } = crawlingContext;
161
- // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
162
- if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
163
- await sleep(5000);
164
- // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
165
- const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
166
- if (!foundSelectors)
167
- return false;
168
- return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
169
- }
170
177
  const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
171
178
  const statusCode = response?.status() ?? 0;
172
179
  if (foundSelectors)
@@ -342,11 +349,36 @@ export class BrowserCrawler extends BasicCrawler {
342
349
  }
343
350
  }
344
351
  async #handleBlockedRequestByContent(crawlingContext) {
345
- if (this.retryOnBlocked) {
346
- const error = await this.#isRequestBlocked(crawlingContext);
347
- if (error)
348
- throw new SessionError(error);
352
+ if (!this.retryOnBlocked)
353
+ return {};
354
+ const page = crawlingContext.page;
355
+ // A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
356
+ let navigationResponse;
357
+ const onResponse = (response) => {
358
+ if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
359
+ navigationResponse = response;
360
+ }
361
+ };
362
+ page.on('response', onResponse);
363
+ try {
364
+ // Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
365
+ // Wait 5 seconds for the JS challenge to load / resolve before judging the page.
366
+ const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
367
+ if (challenge) {
368
+ await sleep(5000);
369
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
370
+ if (foundSelectors) {
371
+ throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
372
+ }
373
+ return navigationResponse ? { response: navigationResponse } : {};
374
+ }
375
+ }
376
+ finally {
377
+ page.off('response', onResponse);
349
378
  }
379
+ const error = await this.#isRequestBlocked(crawlingContext);
380
+ if (error)
381
+ throw new SessionError(error);
350
382
  return {};
351
383
  }
352
384
  async #restoreRequestState(crawlingContext) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-beta.261",
3
+ "version": "4.0.0-beta.263",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.261",
52
- "@crawlee/browser-pool": "4.0.0-beta.261",
53
- "@crawlee/types": "4.0.0-beta.261",
54
- "@crawlee/utils": "4.0.0-beta.261",
51
+ "@crawlee/basic": "4.0.0-beta.263",
52
+ "@crawlee/browser-pool": "4.0.0-beta.263",
53
+ "@crawlee/types": "4.0.0-beta.263",
54
+ "@crawlee/utils": "4.0.0-beta.263",
55
55
  "tslib": "^2.9.1",
56
56
  "zod": "^4.6.7"
57
57
  },
@@ -74,5 +74,5 @@
74
74
  }
75
75
  }
76
76
  },
77
- "gitHead": "24e897c21c5f7ba19846bbca3eba4def5331a081"
77
+ "gitHead": "7303c694793b8f5cc17f5b16328f435fae121668"
78
78
  }