@crawlee/utils 4.0.0-beta.262 → 4.0.0-beta.264

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,11 @@
1
1
  export declare const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export declare const AWS_WAF_RETRY_CSS_SELECTORS: string[];
2
9
  /**
3
10
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
11
  */
@@ -1,9 +1,20 @@
1
1
  export const CLOUDFLARE_RETRY_CSS_SELECTORS = ['#turnstile-wrapper iframe[src^="https://challenges.cloudflare.com"]'];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export const AWS_WAF_RETRY_CSS_SELECTORS = [
9
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/challenge.js"]) > body > #challenge-container',
10
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/captcha.js"]) > body > #captcha-container',
11
+ ];
2
12
  /**
3
13
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
14
  */
5
15
  export const RETRY_CSS_SELECTORS = [
6
16
  ...CLOUDFLARE_RETRY_CSS_SELECTORS,
17
+ ...AWS_WAF_RETRY_CSS_SELECTORS,
7
18
  'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]',
8
19
  'iframe[src*="_Incapsula_Resource"]',
9
20
  ];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.262",
3
+ "version": "4.0.0-beta.264",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.2",
46
- "@crawlee/http-client": "4.0.0-beta.262",
47
- "@crawlee/types": "4.0.0-beta.262",
46
+ "@crawlee/http-client": "4.0.0-beta.264",
47
+ "@crawlee/types": "4.0.0-beta.264",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "56d3158b87fd8f83d195634092f6fbfe3b0e4616"
66
+ "gitHead": "cafd0ea4b7984a5bb7c436d8f0c30c1d05469bac"
67
67
  }