mcp-scraper 0.30.0 → 0.32.0

package/dist/index.cjs

@@ -48,6 +48,9 @@ var HarvestOptionsSchema = import_zod.z.object({
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop"),
   proxyMode: import_zod.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
   proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
+  keepDefaultProxy: import_zod.z.boolean().optional(),
+  requireUsEgress: import_zod.z.boolean().optional(),
+  maxAttempts: import_zod.z.number().int().min(1).max(12).optional(),
   debug: import_zod.z.boolean().default(false),
   depth: import_zod.z.number().int().min(1).max(30).default(3),
   maxQuestions: import_zod.z.number().int().min(1).max(1e3).default(100),
@@ -544,6 +547,15 @@ function positiveIntFromEnv(name, fallback) {
 function proxyIdSuffix(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
+function serpArrivalUrl(googleUrl) {
+  const base = process.env.SERP_REDIRECT_BASE?.trim().replace(/\/+$/, "");
+  if (!base) return googleUrl;
+  return `${base}/g/${Buffer.from(googleUrl, "utf8").toString("base64url")}`;
+}
+function isUsCountry(country) {
+  if (!country) return false;
+  return /united states|u\.?s\.?a?\.?|^us$/i.test(country.trim());
+}
 function errorText(err) {
   return err instanceof Error ? err.message : String(err);
 }
@@ -677,7 +689,7 @@ var BrowserDriver = class {
       this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
       let defaultProxyDisabled = null;
       let defaultProxyDisableError = null;
-      if (!config.kernelProxyId) {
+      if (!config.kernelProxyId && !config.keepDefaultProxy) {
         try {
           await withTimeout(
             this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -727,9 +739,15 @@ var BrowserDriver = class {
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
-      if (this.debugEnabled) {
+      if (this.debugEnabled || config.requireUsEgress) {
         this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
       }
+      if (config.requireUsEgress) {
+        const country = this.debugSnapshot.networkLocation?.country ?? null;
+        if (!isUsCountry(country)) {
+          throw new CaptchaError(`Non-US egress (${country ?? "unknown"}) \u2014 retrying with a fresh session`);
+        }
+      }
       return;
     }
     const launchOpts = {
@@ -869,6 +887,7 @@ var BrowserDriver = class {
     if (options?.num) params.set("num", String(options.num));
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
+    const navUrl = serpArrivalUrl(url);
     const navDebug = options?.debug ? {
       requestedUrl: url,
       finalUrl: null,
@@ -881,7 +900,7 @@ var BrowserDriver = class {
     } : null;
     if (navDebug) this.debugSnapshot.serpNavigation = navDebug;
     try {
-      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
     } catch (err) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: null, captchaDetected: null });
       const diag = await this.captureDiagnostics(url);
@@ -2407,6 +2426,8 @@ var PAAExtractor = class {
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
+      keepDefaultProxy: options.keepDefaultProxy,
+      requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
@@ -3357,7 +3378,8 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
+  const requestedMaxAttempts = raw.maxAttempts;
+  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;