mcp-scraper 0.30.0 → 0.32.0

package/dist/bin/paa-harvest.js

@@ -1,12 +1,12 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-YH5DRFWQ.js";
+} from "../chunk-RDCCU7CQ.js";
 import {
   browserServiceApiKey
 } from "../chunk-MTSBI7ZH.js";
 import "../chunk-K443GQY5.js";
-import "../chunk-ZQSHKWV4.js";
+import "../chunk-PTHY7NYD.js";
 import "../chunk-M2S27J6Z.js";
 import "../chunk-ICT7DDHL.js";
 

package/dist/{chunk-KAMKTSJQ.js → chunk-55M5CE5L.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE
-} from "./chunk-ZQSHKWV4.js";
+} from "./chunk-PTHY7NYD.js";
 
 // src/lib/slugify.ts
 function slugify(s) {
@@ -1925,4 +1925,4 @@ export {
   runWorkflow,
   runWorkflowStep
 };
-//# sourceMappingURL=chunk-KAMKTSJQ.js.map
+//# sourceMappingURL=chunk-55M5CE5L.js.map

package/dist/chunk-CPQF3PYW.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.32.0";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-CPQF3PYW.js.map

package/dist/chunk-CPQF3PYW.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.32.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-ZQSHKWV4.js → chunk-PTHY7NYD.js}

@@ -10,6 +10,9 @@ var HarvestOptionsSchema = z.object({
   device: z.enum(["desktop", "mobile"]).default("desktop"),
   proxyMode: z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
   proxyZip: z.string().regex(/^\d{5}$/).optional(),
+  keepDefaultProxy: z.boolean().optional(),
+  requireUsEgress: z.boolean().optional(),
+  maxAttempts: z.number().int().min(1).max(12).optional(),
   debug: z.boolean().default(false),
   depth: z.number().int().min(1).max(30).default(3),
   maxQuestions: z.number().int().min(1).max(1e3).default(100),
@@ -111,4 +114,4 @@ export {
   RawMapsReviewStatsSchema,
   RawMapsAboutAttributeSchema
 };
-//# sourceMappingURL=chunk-ZQSHKWV4.js.map
+//# sourceMappingURL=chunk-PTHY7NYD.js.map

package/dist/chunk-PTHY7NYD.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas.ts"],"sourcesContent":["import { z } from 'zod'\n\nexport const DEFAULT_PROXY_MODE = 'none' as const\nexport const DEFAULT_MAPS_PROXY_MODE = 'none' as const\n\nexport const HarvestOptionsSchema = z.object({\n  query:        z.string().min(1),\n  location:     z.string().optional(),\n  gl:           z.string().length(2).default('us'),\n  hl:           z.string().length(2).default('en'),\n  device:       z.enum(['desktop', 'mobile']).default('desktop'),\n  proxyMode:    z.enum(['location', 'configured', 'none']).default(DEFAULT_PROXY_MODE),\n  proxyZip:     z.string().regex(/^\\d{5}$/).optional(),\n  keepDefaultProxy: z.boolean().optional(),\n  requireUsEgress: z.boolean().optional(),\n  maxAttempts:  z.number().int().min(1).max(12).optional(),\n  debug:        z.boolean().default(false),\n  depth:        z.number().int().min(1).max(30).default(3),\n  maxQuestions: z.number().int().min(1).max(1000).default(100),\n  headless:     z.boolean().default(false),\n  profileDir:   z.string().optional(),\n  proxy:        z.string().url().optional(),\n  kernelApiKey: z.string().optional(),\n  kernelProxyId: z.string().optional(),\n  kernelProxyResolution: z.unknown().optional(),\n  outputDir:    z.string().default('./paa-output'),\n  format:       z.enum(['json', 'csv', 'both']).default('both'),\n  serpOnly:     z.boolean().default(false),\n  pages:        z.number().int().min(1).max(2).default(1),\n  softDeadlineMs: z.number().optional(),\n})\n\nexport const MapsPlaceOptionsSchema = z.object({\n  businessName:  z.string().min(1),\n  location:      z.string().min(1),\n  gl:            z.string().length(2).default('us'),\n  hl:            z.string().length(2).default('en'),\n  includeReviews: z.boolean().default(false),\n  maxReviews:    z.number().int().min(1).max(500).default(50),\n  includeServices: z.boolean().default(false),\n  kernelApiKey:  z.string().optional(),\n  kernelProxyId: z.string().optional(),\n  headless:      z.boolean().default(true),\n})\n\nexport const MapsSearchOptionsSchema = z.object({\n  query:         z.string().min(1),\n  location:      z.string().optional(),\n  gl:            z.string().length(2).default('us'),\n  hl:            z.string().length(2).default('en'),\n  maxResults:    z.number().int().min(1).max(50).default(10),\n  includeServices: z.boolean().default(false),\n  proxyMode:     z.enum(['location', 'configured', 'none']).default(DEFAULT_MAPS_PROXY_MODE),\n  proxyZip:      z.string().regex(/^\\d{5}$/).optional(),\n  debug:         z.boolean().default(false),\n  kernelApiKey:  z.string().optional(),\n  kernelProxyId: z.string().optional(),\n  kernelProxyResolution: z.unknown().optional(),\n  headless:      z.boolean().default(true),\n})\n\nexport const RawPAAItemSchema = z.object({\n  question:    z.string().min(1),\n  answer:      z.string().optional(),\n  sourceTitle: z.string().optional(),\n  sourceSite:  z.string().optional(),\n  sourceCite:  z.string().optional(),\n})\n\nexport const RawMapsOverviewSchema = z.object({\n  name:         z.string().nullable(),\n  rating:       z.string().nullable(),\n  reviewCount:  z.string().nullable(),\n  category:     z.string().nullable(),\n  address:      z.string().nullable(),\n  hoursSummary: z.string().nullable(),\n  phone:        z.string().nullable(),\n  phoneDisplay: z.string().nullable(),\n  website:      z.string().nullable(),\n  plusCode:     z.string().nullable(),\n  bookingUrl:   z.string().nullable(),\n})\n\nexport const RawMapsHoursRowSchema = z.object({\n  day:   z.string(),\n  hours: z.string(),\n})\n\nexport const RawMapsReviewStatsSchema = z.object({\n  reviewHistogram: z.array(z.object({\n    stars: z.number(),\n    count: z.string(),\n  })),\n  reviewTopics: z.array(z.object({\n    label: z.string(),\n    count: z.string(),\n  })),\n})\n\nexport const RawMapsReviewCardSchema = z.object({\n  reviewId:      z.string(),\n  author:        z.string().nullable(),\n  stars:         z.string().nullable(),\n  date:          z.string().nullable(),\n  text:          z.string().nullable(),\n  ownerResponse: z.string().nullable(),\n})\n\nexport const RawMapsAboutAttributeSchema = z.object({\n  section:   z.string(),\n  attribute: z.string(),\n})\n"],"mappings":";AAAA,SAAS,SAAS;AAEX,IAAM,qBAAqB;AAC3B,IAAM,0BAA0B;AAEhC,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,OAAc,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC9B,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,IAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAC/C,IAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAC/C,QAAc,EAAE,KAAK,CAAC,WAAW,QAAQ,CAAC,EAAE,QAAQ,SAAS;AAAA,EAC7D,WAAc,EAAE,KAAK,CAAC,YAAY,cAAc,MAAM,CAAC,EAAE,QAAQ,kBAAkB;AAAA,EACnF,UAAc,EAAE,OAAO,EAAE,MAAM,SAAS,EAAE,SAAS;AAAA,EACnD,kBAAkB,EAAE,QAAQ,EAAE,SAAS;AAAA,EACvC,iBAAiB,EAAE,QAAQ,EAAE,SAAS;AAAA,EACtC,aAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS;AAAA,EACvD,OAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC;AAAA,EACvD,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAI,EAAE,QAAQ,GAAG;AAAA,EAC3D,UAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,YAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,EACxC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,uBAAuB,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5C,WAAc,EAAE,OAAO,EAAE,QAAQ,cAAc;AAAA,EAC/C,QAAc,EAAE,KAAK,CAAC,QAAQ,OAAO,MAAM,CAAC,EAAE,QAAQ,MAAM;AAAA,EAC5D,UAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC;AAAA,EACtD,gBAAgB,EAAE,OAAO,EAAE,SAAS;AACtC,CAAC;AAEM,IAAM,yBAAyB,EAAE,OAAO;AAAA,EAC7C,cAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,UAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACzC,YAAe,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,QAAQ,EAAE;AAAA,EAC1D,iBAAiB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EAC1C,cAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,UAAe,EAAE,QAAQ,EAAE,QAAQ,IAAI;AACzC,CAAC;AAEM,IAAM,0BAA0B,EAAE,OAAO;AAAA,EAC9C,OAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,UAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,YAAe,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE;AAAA,EACzD,iBAAiB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EAC1C,WAAe,EAAE,KAAK,CAAC,YAAY,cAAc,MAAM,CAAC,EAAE,QAAQ,uBAAuB;AAAA,EACzF,UAAe,EAAE,OAAO,EAAE,MAAM,SAAS,EAAE,SAAS;AAAA,EACpD,OAAe,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACxC,cAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,uBAAuB,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5C,UAAe,EAAE,QAAQ,EAAE,QAAQ,IAAI;AACzC,CAAC;AAEM,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,UAAa,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC7B,QAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,aAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,YAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,YAAa,EAAE,OAAO,EAAE,SAAS;AACnC,CAAC;AAEM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,MAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,QAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,aAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,SAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,OAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,SAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,YAAc,EAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAEM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,KAAO,EAAE,OAAO;AAAA,EAChB,OAAO,EAAE,OAAO;AAClB,CAAC;AAEM,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,iBAAiB,EAAE,MAAM,EAAE,OAAO;AAAA,IAChC,OAAO,EAAE,OAAO;AAAA,IAChB,OAAO,EAAE,OAAO;AAAA,EAClB,CAAC,CAAC;AAAA,EACF,cAAc,EAAE,MAAM,EAAE,OAAO;AAAA,IAC7B,OAAO,EAAE,OAAO;AAAA,IAChB,OAAO,EAAE,OAAO;AAAA,EAClB,CAAC,CAAC;AACJ,CAAC;AAEM,IAAM,0BAA0B,EAAE,OAAO;AAAA,EAC9C,UAAe,EAAE,OAAO;AAAA,EACxB,QAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,OAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,MAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,MAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AACrC,CAAC;AAEM,IAAM,8BAA8B,EAAE,OAAO;AAAA,EAClD,SAAW,EAAE,OAAO;AAAA,EACpB,WAAW,EAAE,OAAO;AACtB,CAAC;","names":[]}

package/dist/{chunk-YH5DRFWQ.js → chunk-RDCCU7CQ.js}

@@ -13,7 +13,7 @@ import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
   RawPAAItemSchema
-} from "./chunk-ZQSHKWV4.js";
+} from "./chunk-PTHY7NYD.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -146,6 +146,15 @@ function positiveIntFromEnv(name, fallback) {
 function proxyIdSuffix(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
+function serpArrivalUrl(googleUrl) {
+  const base = process.env.SERP_REDIRECT_BASE?.trim().replace(/\/+$/, "");
+  if (!base) return googleUrl;
+  return `${base}/g/${Buffer.from(googleUrl, "utf8").toString("base64url")}`;
+}
+function isUsCountry(country) {
+  if (!country) return false;
+  return /united states|u\.?s\.?a?\.?|^us$/i.test(country.trim());
+}
 function errorText(err) {
   return err instanceof Error ? err.message : String(err);
 }
@@ -279,7 +288,7 @@ var BrowserDriver = class {
       this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
       let defaultProxyDisabled = null;
       let defaultProxyDisableError = null;
-      if (!config.kernelProxyId) {
+      if (!config.kernelProxyId && !config.keepDefaultProxy) {
         try {
           await withTimeout(
             this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -329,9 +338,15 @@ var BrowserDriver = class {
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
-      if (this.debugEnabled) {
+      if (this.debugEnabled || config.requireUsEgress) {
         this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
       }
+      if (config.requireUsEgress) {
+        const country = this.debugSnapshot.networkLocation?.country ?? null;
+        if (!isUsCountry(country)) {
+          throw new CaptchaError(`Non-US egress (${country ?? "unknown"}) \u2014 retrying with a fresh session`);
+        }
+      }
       return;
     }
     const launchOpts = {
@@ -471,6 +486,7 @@ var BrowserDriver = class {
     if (options?.num) params.set("num", String(options.num));
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
+    const navUrl = serpArrivalUrl(url);
     const navDebug = options?.debug ? {
       requestedUrl: url,
       finalUrl: null,
@@ -483,7 +499,7 @@ var BrowserDriver = class {
     } : null;
     if (navDebug) this.debugSnapshot.serpNavigation = navDebug;
     try {
-      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
     } catch (err) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: null, captchaDetected: null });
       const diag = await this.captureDiagnostics(url);
@@ -2009,6 +2025,8 @@ var PAAExtractor = class {
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
+      keepDefaultProxy: options.keepDefaultProxy,
+      requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
@@ -2937,7 +2955,8 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
+  const requestedMaxAttempts = raw.maxAttempts;
+  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;
@@ -3161,4 +3180,4 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-YH5DRFWQ.js.map
+//# sourceMappingURL=chunk-RDCCU7CQ.js.map