@crawlee/playwright 3.17.1-beta.0 → 3.17.1-beta.10

package/internals/enqueue-links/click-elements.js

@@ -132,7 +132,7 @@ async function clickElementsAndInterceptNavigationRequests(options) {
     await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
     // browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);
     page.off('framenavigated', onFrameNavigated);
-    await context.unroute('*', onInterceptedRequest);
+    await context.unroute('**', onInterceptedRequest);
     const serializedRequests = Array.from(uniqueRequests);
     return serializedRequests.map((r) => JSON.parse(r));
 }
@@ -176,7 +176,16 @@ function createTargetCreatedHandler(requests) {
  * @ignore
  */
 function isTopFrameNavigationRequest(page, req) {
-    return req.isNavigationRequest() && req.frame() === page.mainFrame();
+    try {
+        return req.isNavigationRequest() && req.frame() === page.mainFrame();
+    }
+    catch {
+        // `req.frame()` throws when the owning frame is unavailable - e.g. the request was
+        // issued by a service worker, or before/after its frame existed (see #3216). Such a
+        // request is not a top-frame navigation, so swallow the throw and let it pass through
+        // instead of crashing the route handler (which would leave the route unhandled).
+        return false;
+    }
 }
 /**
  * @ignore

package/internals/utils/playwright-utils.js

@@ -511,7 +511,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     });
     options.isChallengeCallback ?? (options.isChallengeCallback = async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
+            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
+            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
         });
     });
     const retryBlocked = async () => {
@@ -532,7 +534,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
-        const div = document.querySelector('.main-content div');
+        // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+        // fall back to the first content div for older challenge layouts.
+        const div = document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+            document.querySelector('.main-content div');
         return div?.getBoundingClientRect();
     })
         .catch(() => undefined);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "3.17.1-beta.0",
+    "version": "3.17.1-beta.10",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -56,11 +56,11 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.3.1",
-        "@crawlee/browser": "3.17.1-beta.0",
-        "@crawlee/browser-pool": "3.17.1-beta.0",
-        "@crawlee/core": "3.17.1-beta.0",
-        "@crawlee/types": "3.17.1-beta.0",
-        "@crawlee/utils": "3.17.1-beta.0",
+        "@crawlee/browser": "3.17.1-beta.10",
+        "@crawlee/browser-pool": "3.17.1-beta.10",
+        "@crawlee/core": "3.17.1-beta.10",
+        "@crawlee/types": "3.17.1-beta.10",
+        "@crawlee/utils": "3.17.1-beta.10",
         "cheerio": "1.0.0-rc.12",
         "jquery": "^3.6.0",
         "lodash.isequal": "^4.5.0",
@@ -89,5 +89,5 @@
             }
         }
     },
-    "gitHead": "d76458ce1dc2c433988ea09a14e222b56e29bf06"
+    "gitHead": "2c63bfc2cf7471408ea7e1c0aee385adc2e78971"
 }