mcp-scraper 0.31.0 → 0.32.1

package/README.md

@@ -88,7 +88,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.30.0`, SHA-256 `848ebe2ac12ea4fdd08ae95d368ca537eaf8a1bbd32456baff7a8c2c817e457b`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.32.1`, SHA-256 `a8b3ff927f58382f5e550cbe079a4276e448c6750858ec9712f2f945cd79ffa3`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -13834,6 +13834,15 @@ function positiveIntFromEnv(name, fallback) {
 function proxyIdSuffix(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
+function serpArrivalUrl(googleUrl) {
+  const base = process.env.SERP_REDIRECT_BASE?.trim().replace(/\/+$/, "");
+  if (!base) return googleUrl;
+  return `${base}/g/${Buffer.from(googleUrl, "utf8").toString("base64url")}`;
+}
+function isUsCountry(country) {
+  if (!country) return false;
+  return /united states|u\.?s\.?a?\.?|^us$/i.test(country.trim());
+}
 function errorText(err) {
   return err instanceof Error ? err.message : String(err);
 }
@@ -13988,7 +13997,7 @@ var init_BrowserDriver = __esm({
           this.kernelProxyType = config.kernelProxyResolution?.proxyType ?? null;
           let defaultProxyDisabled = null;
           let defaultProxyDisableError = null;
-          if (!config.kernelProxyId) {
+          if (!config.kernelProxyId && !config.keepDefaultProxy) {
             try {
               await withTimeout2(
                 this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
@@ -14038,9 +14047,15 @@ var init_BrowserDriver = __esm({
           await this.installEsbuildHelperShims(this.context);
           this.page = await this.context.newPage();
           await this.page.setViewportSize(config.viewport);
-          if (this.debugEnabled) {
+          if (this.debugEnabled || config.requireUsEgress) {
             this.debugSnapshot.networkLocation = await this.captureBrowserNetworkLocation();
           }
+          if (config.requireUsEgress) {
+            const country = this.debugSnapshot.networkLocation?.country ?? null;
+            if (!isUsCountry(country)) {
+              throw new CaptchaError(`Non-US egress (${country ?? "unknown"}) \u2014 retrying with a fresh session`);
+            }
+          }
           return;
         }
         const launchOpts = {
@@ -14180,6 +14195,7 @@ var init_BrowserDriver = __esm({
         if (options?.num) params.set("num", String(options.num));
         if (uule) params.set("uule", uule);
         const url = "https://www.google.com/search?" + params.toString();
+        const navUrl = serpArrivalUrl(url);
         const navDebug = options?.debug ? {
           requestedUrl: url,
           finalUrl: null,
@@ -14192,7 +14208,7 @@ var init_BrowserDriver = __esm({
         } : null;
         if (navDebug) this.debugSnapshot.serpNavigation = navDebug;
         try {
-          await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+          await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
         } catch (err) {
           await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: null, captchaDetected: null });
           const diag = await this.captureDiagnostics(url);
@@ -17339,6 +17355,9 @@ var init_schemas3 = __esm({
       device: import_zod17.z.enum(["desktop", "mobile"]).default("desktop"),
       proxyMode: import_zod17.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
       proxyZip: import_zod17.z.string().regex(/^\d{5}$/).optional(),
+      keepDefaultProxy: import_zod17.z.boolean().optional(),
+      requireUsEgress: import_zod17.z.boolean().optional(),
+      maxAttempts: import_zod17.z.number().int().min(1).max(12).optional(),
       debug: import_zod17.z.boolean().default(false),
       depth: import_zod17.z.number().int().min(1).max(30).default(3),
       maxQuestions: import_zod17.z.number().int().min(1).max(1e3).default(100),
@@ -19235,6 +19254,12 @@ function buildGoogleLocalResultsUrl(input) {
   if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
   return `https://www.google.com/search?${params.toString()}`;
 }
+function buildGoogleOrganicSearchUrl(input) {
+  const query = [input.query, input.location].filter(Boolean).join(" ");
+  const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
+  if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
+  return `https://www.google.com/search?${params.toString()}`;
+}
 var LOCAL_RESULTS_PAGE_LIMIT, LOCAL_RESULTS_WAIT_MS, MapsSearchExtractor;
 var init_MapsSearchExtractor = __esm({
   "src/extractor/MapsSearchExtractor.ts"() {
@@ -19251,13 +19276,14 @@ var init_MapsSearchExtractor = __esm({
       async extract(options) {
         const startMs = Date.now();
         const searchQuery = [options.query, options.location].filter(Boolean).join(" ");
-        const searchUrl = buildGoogleLocalResultsUrl(options);
+        const searchUrl = buildGoogleOrganicSearchUrl(options);
         const config = {
           headless: options.headless,
           kernelApiKey: options.kernelApiKey,
           kernelProxyId: options.kernelProxyId,
           kernelProxyResolution: options.kernelProxyResolution,
           proxyMode: options.proxyMode,
+          keepDefaultProxy: options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId,
           viewport: { width: 1280, height: 900 },
           locale: `${options.hl}-${options.gl.toUpperCase()}`,
           debug: options.debug
@@ -19268,6 +19294,10 @@ var init_MapsSearchExtractor = __esm({
           await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
           await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
           if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
+          const reachedFinder = await this.clickIntoLocalFinder(page);
+          if (!reachedFinder) throw new ExtractionError('Google "More businesses" control was not found on the organic results page');
+          await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
+          if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
           const results = await this.collectResults(page, options);
           return {
             query: options.query,
@@ -19284,6 +19314,17 @@ var init_MapsSearchExtractor = __esm({
           await this.driver.close();
         }
       }
+      async clickIntoLocalFinder(page) {
+        const control = page.getByRole("link", { name: /more (businesses|places|results)/i }).or(page.getByRole("button", { name: /more (businesses|places|results)/i })).first();
+        const visible = await control.isVisible({ timeout: 1e4 }).catch(() => false);
+        if (!visible) return false;
+        await control.click({ timeout: 1e4 }).catch(() => {
+        });
+        await page.waitForTimeout(5e3);
+        return page.evaluate(
+          () => /[?&]udm=1/.test(location.href) || document.querySelectorAll('a[href*="/maps/place/"]').length > 0
+        );
+      }
       async detectBlock(page) {
         return page.evaluate(() => {
           const text2 = document.body.innerText.slice(0, 2e3);
@@ -19337,29 +19378,38 @@ var init_MapsSearchExtractor = __esm({
           };
           const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
           const addressPattern = /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i;
-          const cards = Array.from(document.querySelectorAll("button")).filter((button) => {
-            const name = normalize4(button.querySelector("h3")?.textContent);
-            return Boolean(name) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText);
+          const headings = Array.from(document.querySelectorAll('div[role="heading"]')).filter((heading) => {
+            const text2 = normalize4(heading.textContent);
+            return Boolean(text2) && !/^(?:businesses|places|sponsored|people also ask|discussions and forums|more businesses|more places|custom date range|choose what you)/i.test(text2 ?? "");
           });
           const seen = /* @__PURE__ */ new Set();
           const out = [];
-          for (const [cardIndex, card] of cards.entries()) {
-            const name = normalize4(card.querySelector("h3")?.textContent);
+          let cardIndex = -1;
+          for (const heading of headings) {
+            let card = heading;
+            for (let hop = 0; hop < 8; hop++) {
+              const parent = card.parentElement;
+              if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+              card = parent;
+            }
+            if (card.closest('[role="dialog"], dialog')) continue;
+            const name = normalize4(heading.textContent);
             if (!name) continue;
-            const text2 = normalize4(card.innerText) ?? name;
-            const key = `${name.toLowerCase()}|${text2.toLowerCase()}`;
+            const lines = (card.innerText ?? "").split("\n").map((line) => normalize4(line)).filter((line) => Boolean(line));
+            if (/^sponsored$/i.test(lines[0] ?? "")) continue;
+            const ratingLine = lines.find((line) => /\d(?:\.\d)?\s*\(/.test(line)) ?? "";
+            const rating = ratingLine.match(/(\d(?:\.\d)?)\s*\(/)?.[1] ?? null;
+            const reviewCount = ratingLine.match(/\((\d[\d,]*)\)/)?.[1]?.replace(/,/g, "") ?? null;
+            const category = normalize4(ratingLine.split("\xB7").slice(1).join("\xB7"));
+            const phone = lines.map((line) => line.match(phonePattern)?.[0]).find((value) => Boolean(value)) ?? null;
+            const hoursStatus = normalize4((lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? "").split("\xB7")[0] ?? "");
+            const address = lines.find((line) => addressPattern.test(line)) ?? null;
+            const websiteUrl = Array.from(card.querySelectorAll("a[href]")).map((anchor) => anchor.href).find((href) => /^https?:/i.test(href) && !/google\.|gstatic|googleusercontent/.test(href)) ?? null;
+            if (!rating && !(phone && websiteUrl)) continue;
+            const key = `${name.toLowerCase()}|${lines.join(" ").toLowerCase()}`;
             if (seen.has(key)) continue;
             seen.add(key);
-            const lines = text2.split(/\n+/).map((line) => normalize4(line)).filter((line) => Boolean(line));
-            const aria = normalize4(card.getAttribute("aria-label"));
-            const rating = (aria ?? text2).match(/(?:rated\s+)?(\d(?:\.\d)?)\s*(?:out of 5)?/i)?.[1] ?? null;
-            const reviewCount = (aria ?? text2).match(/(?:\(|·\s*)([\d,]+)\s+(?:user\s+)?reviews?\b/i)?.[1]?.replace(/,/g, "") ?? null;
-            const phone = lines.find((line) => phonePattern.test(line)) ?? null;
-            const address = lines.find((line) => addressPattern.test(line)) ?? null;
-            const hoursStatus = lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? null;
-            const category = lines.find(
-              (line) => line !== name && line !== phone && line !== address && line !== hoursStatus && !/^(?:rated|\d[\d.,]*\s*(?:out of 5|reviews?)?)\b/i.test(line) && !/sponsored|website|directions|call|save|share|more/i.test(line)
-            ) ?? null;
+            cardIndex += 1;
             const mapsSearchUrl = `https://www.google.com/maps/search/${encodeURIComponent([name, locationHint].filter(Boolean).join(" "))}`;
             out.push({
               cardIndex,
@@ -19374,7 +19424,7 @@ var init_MapsSearchExtractor = __esm({
               address,
               phone,
               hoursStatus,
-              websiteUrl: null,
+              websiteUrl,
               directionsUrl: `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent([name, address].filter(Boolean).join(", "))}`,
               metadata: lines.slice(0, 20)
             });
@@ -19383,17 +19433,27 @@ var init_MapsSearchExtractor = __esm({
         }, fallbackLocation);
       }
       async openCardAndExtractDialog(page, card, includeServices) {
+        if (!includeServices) return null;
         const clicked = await page.evaluate((input) => {
           const normalize4 = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();
-          const cards = Array.from(document.querySelectorAll("button")).filter(
-            (button) => Boolean(button.querySelector("h3")) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText)
-          );
-          const exact = cards.find((button) => `${normalize4(button.querySelector("h3")?.textContent)}|${normalize4(button.innerText)}` === input.cardKey);
-          const target = exact ?? cards[input.cardIndex];
-          if (!target) return false;
-          target.click();
-          return true;
-        }, { cardIndex: card.cardIndex, cardKey: card.cardKey });
+          const headings = Array.from(document.querySelectorAll('div[role="heading"]'));
+          for (const heading of headings) {
+            let target = heading;
+            for (let hop = 0; hop < 8; hop++) {
+              const parent = target.parentElement;
+              if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+              target = parent;
+            }
+            if (target.closest('[role="dialog"], dialog')) continue;
+            const name = normalize4(heading.textContent);
+            if (!name) continue;
+            const lines = (target.innerText ?? "").split("\n").map((line) => normalize4(line)).filter(Boolean);
+            if (`${name}|${lines.join(" ")}` !== input.cardKey) continue;
+            target.click();
+            return true;
+          }
+          return false;
+        }, { cardKey: card.cardKey });
         if (!clicked) return null;
         const dialog = page.locator('dialog, [role="dialog"]').filter({ has: page.locator('h1, h2, [role="heading"]') }).last();
         await dialog.waitFor({ state: "visible", timeout: 6e3 }).catch(() => {
@@ -28145,6 +28205,8 @@ var init_PAAExtractor = __esm({
           kernelProxyId: options.kernelProxyId,
           kernelProxyResolution: options.kernelProxyResolution,
           proxyMode: options.proxyMode,
+          keepDefaultProxy: options.keepDefaultProxy,
+          requireUsEgress: options.requireUsEgress,
           viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
           locale: `${options.hl}-${options.gl.toUpperCase()}`,
           userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
@@ -28645,7 +28707,8 @@ async function harvest(rawOptions) {
   };
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
-  const maxAttempts = maxAttemptsForProxyMode(proxyMode) + 1;
+  const requestedMaxAttempts = raw.maxAttempts;
+  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;
@@ -29723,7 +29786,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.31.0";
+    PACKAGE_VERSION = "0.32.1";
   }
 });
 
@@ -30355,9 +30418,9 @@ var init_mcp_tool_schemas = __esm({
       gl: import_zod35.z.string().length(2).default("us").describe("Google country code inferred from location."),
       hl: import_zod35.z.string().length(2).default("en").describe("Language inferred from user request."),
       maxResults: import_zod35.z.number().int().min(1).max(50).default(10).describe("Number of candidates to return. Default 10, maximum 50."),
-      includeServices: import_zod35.z.boolean().default(false).describe("Open each returned Google Business Profile and include configured services and areas served when available. Does not collect review cards."),
-      proxyMode: import_zod35.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Leave unset for the default direct browser route. Google localization comes from the city in the query plus UULE, gl, and hl. location is an explicit residential-proxy override."),
-      proxyZip: import_zod35.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override when proxyMode is location."),
+      includeServices: import_zod35.z.boolean().default(false).describe("Open each returned business profile to include its configured services and areas served when available. Adds a page visit per business; does not collect review cards."),
+      proxyMode: import_zod35.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Leave unset for the default route (stealth browser on the managed ISP proxy, retried on a fresh session when Google blocks). Localization comes from the city in the query plus gl/hl. location forces an explicit residential proxy \u2014 not recommended for Google."),
+      proxyZip: import_zod35.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override, only used when proxyMode is location."),
       debug: import_zod35.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
     };
     DirectoryWorkflowInputSchema = {
@@ -30371,7 +30434,7 @@ var init_mcp_tool_schemas = __esm({
       includeZipGroups: import_zod35.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available (MCP_SCRAPER_USZIPS_CSV_PATH or usZipsCsvPath)."),
       usZipsCsvPath: import_zod35.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead. For ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the server, or pass this in local/test mode."),
       saveCsv: import_zod35.z.boolean().default(true).describe("Save a directory-ready CSV of results to the MCP Scraper output directory and return its path."),
-      proxyMode: import_zod35.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy behavior per city search. Leave unset for the direct localized Google route; location is an explicit residential-proxy override."),
+      proxyMode: import_zod35.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy behavior per city search. Leave unset for the default route (stealth browser on the managed ISP proxy, retried fresh on a Google block). location forces an explicit residential proxy \u2014 not recommended for Google."),
       proxyZip: import_zod35.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting; normally omitted."),
       debug: import_zod35.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
     };
@@ -32191,7 +32254,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
   server.registerTool("maps_search", {
     title: "Google Maps Business Search",
-    description: "Search Google local results for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. It paginates the rendered local-results list, opens each exact business card, then closes the profile dialog before continuing. Set includeServices for services and areas served; review cards are never collected by this tool.",
+    description: "Search Google local results for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. Reaches the local-results list from the organic page and paginates it, returning up to 50 candidates (default 10) with names, place URLs, CIDs, and ratings. Leave proxyMode unset. Set includeServices to also open each business profile for its services and areas served; review cards are never collected by this tool.",
     inputSchema: MapsSearchInputSchema,
     outputSchema: recordOutputSchema("maps_search", MapsSearchOutputSchema),
     annotations: liveWebToolAnnotations("Google Maps Business Search")