mcp-scraper 0.2.7 → 0.2.9

package/dist/index.cjs

@@ -272,6 +272,8 @@ var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X)
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
+var DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS = 2500;
+var SERP_CHALLENGE_SOLVE_POLL_MS = 500;
 function positiveIntFromEnv(name, fallback) {
   const raw = process.env[name];
   if (!raw) return fallback;
@@ -583,20 +585,20 @@ var BrowserDriver = class {
       const diag = await this.captureDiagnostics(url);
       throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
     }
-    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count();
-    if (captchaCount > 0) {
+    if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      throw new CaptchaError(this.captchaMessage());
+      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
+      if (!solved) throw new CaptchaError(this.captchaMessage());
     }
     const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
     if (fastFound) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
       return { hasPaa: true };
     }
-    const captchaAfter = await this.page.locator(PAASelectors.captchaMarker).count();
-    if (captchaAfter > 0) {
+    if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      throw new CaptchaError(this.captchaMessage());
+      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
+      if (!solved) throw new CaptchaError(this.captchaMessage());
     }
     for (let i = 1; i <= 6; i++) {
       await this.page.evaluate((f) => {
@@ -612,6 +614,35 @@ var BrowserDriver = class {
     await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: false });
     return { hasPaa: false };
   }
+  async isSerpChallengeVisible() {
+    if (!this.page) return false;
+    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count().catch(() => 0);
+    if (captchaCount > 0) return true;
+    const finalUrl = this.page.url();
+    if (/google\.[^/]+\/sorry\//i.test(finalUrl)) return true;
+    const bodySnippet = await this.page.evaluate(() => {
+      const text = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
+      return text.slice(0, 1e3);
+    }).catch(() => "");
+    return /recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(bodySnippet);
+  }
+  async waitForSerpChallengeToClear(navDebug, requestedUrl) {
+    if (!this.kernelClient || !this.page) return false;
+    const waitMs = positiveIntFromEnv("SERP_CHALLENGE_SOLVE_WAIT_MS", DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS);
+    const deadline = Date.now() + waitMs;
+    while (Date.now() < deadline) {
+      await this.page.waitForTimeout(Math.min(SERP_CHALLENGE_SOLVE_POLL_MS, Math.max(250, deadline - Date.now())));
+      if (!await this.isSerpChallengeVisible()) {
+        await this.page.waitForTimeout(1e3);
+        if (!await this.isSerpChallengeVisible()) {
+          await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: null, captchaDetected: false });
+          return true;
+        }
+      }
+    }
+    await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: false, captchaDetected: true });
+    return false;
+  }
   async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
     if (!navDebug || !this.page) return;
     try {
@@ -659,7 +690,12 @@ var BrowserDriver = class {
   async navigateTo(url) {
     try {
       await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
+        const solved = await this.waitForSerpChallengeToClear(null, url);
+        if (!solved) throw new CaptchaError(this.captchaMessage());
+      }
     } catch (err) {
+      if (err instanceof CaptchaError) throw err;
       const diag = await this.captureDiagnostics(url);
       throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
     }