mcp-scraper 0.2.6 → 0.2.8

package/dist/bin/mcp-stdio-server.js

@@ -2,9 +2,9 @@
 import {
   HttpMcpToolExecutor,
   buildPaaExtractorMcpServer
-} from "../chunk-7WB2W6FO.js";
+} from "../chunk-6NEXSNSA.js";
 import "../chunk-M2S27J6Z.js";
-import "../chunk-T6SIPPOQ.js";
+import "../chunk-I26QN7WQ.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/bin/paa-harvest.cjs

@@ -262,6 +262,8 @@ var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X)
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
+var DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS = 2500;
+var SERP_CHALLENGE_SOLVE_POLL_MS = 500;
 function positiveIntFromEnv(name, fallback) {
   const raw = process.env[name];
   if (!raw) return fallback;
@@ -573,20 +575,20 @@ var BrowserDriver = class {
       const diag = await this.captureDiagnostics(url);
       throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
     }
-    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count();
-    if (captchaCount > 0) {
+    if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      throw new CaptchaError(this.captchaMessage());
+      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
+      if (!solved) throw new CaptchaError(this.captchaMessage());
     }
     const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
     if (fastFound) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
       return { hasPaa: true };
     }
-    const captchaAfter = await this.page.locator(PAASelectors.captchaMarker).count();
-    if (captchaAfter > 0) {
+    if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      throw new CaptchaError(this.captchaMessage());
+      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
+      if (!solved) throw new CaptchaError(this.captchaMessage());
     }
     for (let i = 1; i <= 6; i++) {
       await this.page.evaluate((f) => {
@@ -602,6 +604,35 @@ var BrowserDriver = class {
     await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: false });
     return { hasPaa: false };
   }
+  async isSerpChallengeVisible() {
+    if (!this.page) return false;
+    const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count().catch(() => 0);
+    if (captchaCount > 0) return true;
+    const finalUrl = this.page.url();
+    if (/google\.[^/]+\/sorry\//i.test(finalUrl)) return true;
+    const bodySnippet = await this.page.evaluate(() => {
+      const text = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
+      return text.slice(0, 1e3);
+    }).catch(() => "");
+    return /recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(bodySnippet);
+  }
+  async waitForSerpChallengeToClear(navDebug, requestedUrl) {
+    if (!this.kernelClient || !this.page) return false;
+    const waitMs = positiveIntFromEnv("SERP_CHALLENGE_SOLVE_WAIT_MS", DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS);
+    const deadline = Date.now() + waitMs;
+    while (Date.now() < deadline) {
+      await this.page.waitForTimeout(Math.min(SERP_CHALLENGE_SOLVE_POLL_MS, Math.max(250, deadline - Date.now())));
+      if (!await this.isSerpChallengeVisible()) {
+        await this.page.waitForTimeout(1e3);
+        if (!await this.isSerpChallengeVisible()) {
+          await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: null, captchaDetected: false });
+          return true;
+        }
+      }
+    }
+    await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: false, captchaDetected: true });
+    return false;
+  }
   async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
     if (!navDebug || !this.page) return;
     try {
@@ -649,7 +680,12 @@ var BrowserDriver = class {
   async navigateTo(url) {
     try {
       await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
+        const solved = await this.waitForSerpChallengeToClear(null, url);
+        if (!solved) throw new CaptchaError(this.captchaMessage());
+      }
     } catch (err) {
+      if (err instanceof CaptchaError) throw err;
       const diag = await this.captureDiagnostics(url);
       throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
     }