mcp-scraper 0.30.0 → 0.32.0

package/dist/bin/mcp-stdio-server.js

@@ -7,16 +7,16 @@ import {
   registerMemoryMcpTools,
   registerPaaExtractorMcpTools,
   registerSerpIntelligenceCaptureTools
-} from "../chunk-T2CIPAOK.js";
+} from "../chunk-YOOEAXWP.js";
 import "../chunk-R7EETU7Z.js";
 import "../chunk-MTSBI7ZH.js";
 import {
   renderInstallTerminal
 } from "../chunk-DUS4ENOY.js";
-import "../chunk-ZQSHKWV4.js";
+import "../chunk-PTHY7NYD.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-WQUF7AH7.js";
+} from "../chunk-CPQF3PYW.js";
 import "../chunk-4767BA2O.js";
 import "../chunk-M2S27J6Z.js";
 import "../chunk-ICT7DDHL.js";

package/dist/bin/paa-harvest.cjs

@@ -47,6 +47,9 @@ var HarvestOptionsSchema = import_zod.z.object({
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop"),
   proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
   proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
+  keepDefaultProxy: import_zod.z.boolean().optional(),
+  requireUsEgress: import_zod.z.boolean().optional(),
+  maxAttempts: import_zod.z.number().int().min(1).max(12).optional(),
   debug: import_zod.z.boolean().default(false),
   depth: import_zod.z.number().int().min(1).max(30).default(3),
   maxQuestions: import_zod.z.number().int().min(1).max(1e3).default(100),
@@ -493,6 +496,15 @@ function positiveIntFromEnv(name, fallback) {
 function proxyIdSuffix(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
+function serpArrivalUrl(googleUrl) {
+  const base = process.env.SERP_REDIRECT_BASE?.trim().replace(/\/+$/, "");
+  if (!base) return googleUrl;
+  return `${base}/g/${Buffer.from(googleUrl, "utf8").toString("base64url")}`;
+}
+function isUsCountry(country) {
+  if (!country) return false;
+  return /united states|u\.?s\.?a?\.?|^us$/i.test(country.trim());
+}
 function errorText(err) {
   return err instanceof Error ? err.message : String(err);
 }
@@ -626,7 +638,7 @@ var BrowserDriver = class {
       this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
       let defaultProxyDisabled = null;
       let defaultProxyDisableError = null;
-      if (!config.kernelProxyId) {
+      if (!config.kernelProxyId && !config.keepDefaultProxy) {
         try {
           await withTimeout(
             this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -676,9 +688,15 @@ var BrowserDriver = class {
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
-      if (this.debugEnabled) {
+      if (this.debugEnabled || config.requireUsEgress) {
         this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
       }
+      if (config.requireUsEgress) {
+        const country = this.debugSnapshot.networkLocation?.country ?? null;
+        if (!isUsCountry(country)) {
+          throw new CaptchaError(`Non-US egress (${country ?? "unknown"}) \u2014 retrying with a fresh session`);
+        }
+      }
       return;
     }
     const launchOpts = {
@@ -818,6 +836,7 @@ var BrowserDriver = class {
     if (options?.num) params.set("num", String(options.num));
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
+    const navUrl = serpArrivalUrl(url);
     const navDebug = options?.debug ? {
       requestedUrl: url,
       finalUrl: null,
@@ -830,7 +849,7 @@ var BrowserDriver = class {
     } : null;
     if (navDebug) this.debugSnapshot.serpNavigation = navDebug;
     try {
-      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
     } catch (err) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: null, captchaDetected: null });
       const diag = await this.captureDiagnostics(url);
@@ -2356,6 +2375,8 @@ var PAAExtractor = class {
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
+      keepDefaultProxy: options.keepDefaultProxy,
+      requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
@@ -3306,7 +3327,8 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
+  const requestedMaxAttempts = raw.maxAttempts;
+  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;