@crawlee/utils 4.0.0-beta.261 → 4.0.0-beta.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,11 @@
1
1
  export declare const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export declare const AWS_WAF_RETRY_CSS_SELECTORS: string[];
2
9
  /**
3
10
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
11
  */
@@ -1,9 +1,20 @@
1
1
  export const CLOUDFLARE_RETRY_CSS_SELECTORS = ['#turnstile-wrapper iframe[src^="https://challenges.cloudflare.com"]'];
2
+ /**
3
+ * The AWS WAF interstitials: the JS challenge, served with a `202` status, and the CAPTCHA it escalates to, served with a `405`.
4
+ * Neither status is a blocked one. Matching the container as a direct child of `<body>` keeps regular pages
5
+ * that load the AWS WAF integration scripts, or embed the CAPTCHA into their own markup, from matching.
6
+ * The `:has()` argument must not start with `>`, Playwright's selector engine never matches it on the root element.
7
+ */
8
+ export const AWS_WAF_RETRY_CSS_SELECTORS = [
9
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/challenge.js"]) > body > #challenge-container',
10
+ 'html:has(head > script[src*=".awswaf.com/"][src$="/captcha.js"]) > body > #captcha-container',
11
+ ];
2
12
  /**
3
13
  * CSS selectors for elements that should trigger a retry, as the crawler is likely getting blocked.
4
14
  */
5
15
  export const RETRY_CSS_SELECTORS = [
6
16
  ...CLOUDFLARE_RETRY_CSS_SELECTORS,
17
+ ...AWS_WAF_RETRY_CSS_SELECTORS,
7
18
  'div#infoDiv0 a[href*="//www.google.com/policies/terms/"]',
8
19
  'iframe[src*="_Incapsula_Resource"]',
9
20
  ];
@@ -239,44 +239,66 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
239
239
  // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
240
240
  // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
241
241
  let droppedUrlEntries = 0;
242
- for await (const item of items) {
243
- if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
244
- if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
245
- logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
246
- continue;
242
+ const iterator = items[Symbol.asyncIterator]();
243
+ try {
244
+ while (true) {
245
+ let next;
246
+ try {
247
+ next = await iterator.next();
247
248
  }
248
- // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
249
- // parent. Raw string sources have no parent URL, so the check is skipped.
250
- if (source.type === 'url') {
251
- const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
252
- if (!allowed) {
253
- logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
254
- continue;
249
+ catch (e) {
250
+ // Parsing failed: give up on this sitemap, keep the others.
251
+ // Raw sources already logged the error in their pipeline callback.
252
+ if (source.type !== 'raw') {
253
+ logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
255
254
  }
255
+ break;
256
256
  }
257
- sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
258
- if (emitNestedSitemaps) {
259
- yield { loc: item.url, originSitemapUrl: null };
260
- }
261
- }
262
- if (item.type === 'url') {
263
- // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
264
- if (source.type === 'url') {
265
- const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
266
- if (!allowed) {
267
- droppedUrlEntries++;
268
- logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
257
+ if (next.done)
258
+ break;
259
+ const item = next.value;
260
+ if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
261
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
262
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
269
263
  continue;
270
264
  }
265
+ // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
266
+ // parent. Raw string sources have no parent URL, so the check is skipped.
267
+ if (source.type === 'url') {
268
+ const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
269
+ if (!allowed) {
270
+ logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
271
+ continue;
272
+ }
273
+ }
274
+ sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
275
+ if (emitNestedSitemaps) {
276
+ yield { loc: item.url, originSitemapUrl: null };
277
+ }
278
+ }
279
+ if (item.type === 'url') {
280
+ // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
281
+ if (source.type === 'url') {
282
+ const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
283
+ if (!allowed) {
284
+ droppedUrlEntries++;
285
+ logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
286
+ continue;
287
+ }
288
+ }
289
+ yield {
290
+ ...item,
291
+ originSitemapUrl: source.type === 'url'
292
+ ? source.url
293
+ : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
294
+ };
271
295
  }
272
- yield {
273
- ...item,
274
- originSitemapUrl: source.type === 'url'
275
- ? source.url
276
- : `raw://${createHash('sha256').update(source.content).digest('base64')}`,
277
- };
278
296
  }
279
297
  }
298
+ finally {
299
+ // Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
300
+ await iterator.return?.();
301
+ }
280
302
  if (droppedUrlEntries > 0 && source.type === 'url') {
281
303
  logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
282
304
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.261",
3
+ "version": "4.0.0-beta.263",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.2",
46
- "@crawlee/http-client": "4.0.0-beta.261",
47
- "@crawlee/types": "4.0.0-beta.261",
46
+ "@crawlee/http-client": "4.0.0-beta.263",
47
+ "@crawlee/types": "4.0.0-beta.263",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "24e897c21c5f7ba19846bbca3eba4def5331a081"
66
+ "gitHead": "7303c694793b8f5cc17f5b16328f435fae121668"
67
67
  }