mcp-scraper 0.32.0 → 0.32.2

package/README.md

@@ -88,7 +88,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.32.0`, SHA-256 `f31500fa60ffa96d2bea43feb4f5606a40fdc59b599dd51a11b8a54e5c9bce32`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.32.2`, SHA-256 `e4bcf7d1e722ab821eaceebb5acba77e19e3e7bba1326c3d41f77cb22a70f793`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -13834,8 +13834,8 @@ function positiveIntFromEnv(name, fallback) {
 function proxyIdSuffix(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
-function serpArrivalUrl(googleUrl) {
-  const base = process.env.SERP_REDIRECT_BASE?.trim().replace(/\/+$/, "");
+function serpArrivalUrl(googleUrl, baseOverride) {
+  const base = (baseOverride ?? process.env.SERP_REDIRECT_BASE)?.trim().replace(/\/+$/, "");
   if (!base) return googleUrl;
   return `${base}/g/${Buffer.from(googleUrl, "utf8").toString("base64url")}`;
 }
@@ -17394,6 +17394,8 @@ var init_schemas3 = __esm({
       includeServices: import_zod17.z.boolean().default(false),
       proxyMode: import_zod17.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
       proxyZip: import_zod17.z.string().regex(/^\d{5}$/).optional(),
+      serpRedirect: import_zod17.z.boolean().optional(),
+      forceDirectEgress: import_zod17.z.boolean().optional(),
       debug: import_zod17.z.boolean().default(false),
       kernelApiKey: import_zod17.z.string().optional(),
       kernelProxyId: import_zod17.z.string().optional(),
@@ -19260,14 +19262,16 @@ function buildGoogleOrganicSearchUrl(input) {
   if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
   return `https://www.google.com/search?${params.toString()}`;
 }
-var LOCAL_RESULTS_PAGE_LIMIT, LOCAL_RESULTS_WAIT_MS, MapsSearchExtractor;
+var LOCAL_RESULTS_PAGE_LIMIT, LOCAL_RESULTS_WAIT_MS, MAPS_REDIRECT_BASE, MapsSearchExtractor;
 var init_MapsSearchExtractor = __esm({
   "src/extractor/MapsSearchExtractor.ts"() {
     "use strict";
     init_errors();
+    init_BrowserDriver();
     init_uule();
     LOCAL_RESULTS_PAGE_LIMIT = 8;
     LOCAL_RESULTS_WAIT_MS = 1e3;
+    MAPS_REDIRECT_BASE = "https://serp-redirector.vercel.app";
     MapsSearchExtractor = class {
       constructor(driver) {
         this.driver = driver;
@@ -19277,21 +19281,23 @@ var init_MapsSearchExtractor = __esm({
         const startMs = Date.now();
         const searchQuery = [options.query, options.location].filter(Boolean).join(" ");
         const searchUrl = buildGoogleOrganicSearchUrl(options);
+        const keepDefaultProxy = !options.forceDirectEgress && options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId;
         const config = {
           headless: options.headless,
           kernelApiKey: options.kernelApiKey,
           kernelProxyId: options.kernelProxyId,
           kernelProxyResolution: options.kernelProxyResolution,
           proxyMode: options.proxyMode,
-          keepDefaultProxy: options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId,
+          keepDefaultProxy,
           viewport: { width: 1280, height: 900 },
           locale: `${options.hl}-${options.gl.toUpperCase()}`,
           debug: options.debug
         };
+        const navUrl = options.serpRedirect ? serpArrivalUrl(searchUrl, process.env.SERP_REDIRECT_BASE?.trim() || MAPS_REDIRECT_BASE) : searchUrl;
         try {
           await this.driver.launch(config);
           const page = this.driver.getPage();
-          await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
+          await page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
           await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
           if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
           const reachedFinder = await this.clickIntoLocalFinder(page);
@@ -19378,29 +19384,38 @@ var init_MapsSearchExtractor = __esm({
           };
           const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
           const addressPattern = /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i;
-          const cards = Array.from(document.querySelectorAll("button")).filter((button) => {
-            const name = normalize4(button.querySelector("h3")?.textContent);
-            return Boolean(name) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText);
+          const headings = Array.from(document.querySelectorAll('div[role="heading"]')).filter((heading) => {
+            const text2 = normalize4(heading.textContent);
+            return Boolean(text2) && !/^(?:businesses|places|sponsored|people also ask|discussions and forums|more businesses|more places|custom date range|choose what you)/i.test(text2 ?? "");
           });
           const seen = /* @__PURE__ */ new Set();
           const out = [];
-          for (const [cardIndex, card] of cards.entries()) {
-            const name = normalize4(card.querySelector("h3")?.textContent);
+          let cardIndex = -1;
+          for (const heading of headings) {
+            let card = heading;
+            for (let hop = 0; hop < 8; hop++) {
+              const parent = card.parentElement;
+              if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+              card = parent;
+            }
+            if (card.closest('[role="dialog"], dialog')) continue;
+            const name = normalize4(heading.textContent);
             if (!name) continue;
-            const text2 = normalize4(card.innerText) ?? name;
-            const key = `${name.toLowerCase()}|${text2.toLowerCase()}`;
+            const lines = (card.innerText ?? "").split("\n").map((line) => normalize4(line)).filter((line) => Boolean(line));
+            if (/^sponsored$/i.test(lines[0] ?? "")) continue;
+            const ratingLine = lines.find((line) => /\d(?:\.\d)?\s*\(/.test(line)) ?? "";
+            const rating = ratingLine.match(/(\d(?:\.\d)?)\s*\(/)?.[1] ?? null;
+            const reviewCount = ratingLine.match(/\((\d[\d,]*)\)/)?.[1]?.replace(/,/g, "") ?? null;
+            const category = normalize4(ratingLine.split("\xB7").slice(1).join("\xB7"));
+            const phone = lines.map((line) => line.match(phonePattern)?.[0]).find((value) => Boolean(value)) ?? null;
+            const hoursStatus = normalize4((lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? "").split("\xB7")[0] ?? "");
+            const address = lines.find((line) => addressPattern.test(line)) ?? null;
+            const websiteUrl = Array.from(card.querySelectorAll("a[href]")).map((anchor) => anchor.href).find((href) => /^https?:/i.test(href) && !/google\.|gstatic|googleusercontent/.test(href)) ?? null;
+            if (!rating && !(phone && websiteUrl)) continue;
+            const key = `${name.toLowerCase()}|${lines.join(" ").toLowerCase()}`;
             if (seen.has(key)) continue;
             seen.add(key);
-            const lines = text2.split(/\n+/).map((line) => normalize4(line)).filter((line) => Boolean(line));
-            const aria = normalize4(card.getAttribute("aria-label"));
-            const rating = (aria ?? text2).match(/(?:rated\s+)?(\d(?:\.\d)?)\s*(?:out of 5)?/i)?.[1] ?? null;
-            const reviewCount = (aria ?? text2).match(/(?:\(|·\s*)([\d,]+)\s+(?:user\s+)?reviews?\b/i)?.[1]?.replace(/,/g, "") ?? null;
-            const phone = lines.find((line) => phonePattern.test(line)) ?? null;
-            const address = lines.find((line) => addressPattern.test(line)) ?? null;
-            const hoursStatus = lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? null;
-            const category = lines.find(
-              (line) => line !== name && line !== phone && line !== address && line !== hoursStatus && !/^(?:rated|\d[\d.,]*\s*(?:out of 5|reviews?)?)\b/i.test(line) && !/sponsored|website|directions|call|save|share|more/i.test(line)
-            ) ?? null;
+            cardIndex += 1;
             const mapsSearchUrl = `https://www.google.com/maps/search/${encodeURIComponent([name, locationHint].filter(Boolean).join(" "))}`;
             out.push({
               cardIndex,
@@ -19415,7 +19430,7 @@ var init_MapsSearchExtractor = __esm({
               address,
               phone,
               hoursStatus,
-              websiteUrl: null,
+              websiteUrl,
               directionsUrl: `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent([name, address].filter(Boolean).join(", "))}`,
               metadata: lines.slice(0, 20)
             });
@@ -19424,17 +19439,27 @@ var init_MapsSearchExtractor = __esm({
         }, fallbackLocation);
       }
       async openCardAndExtractDialog(page, card, includeServices) {
+        if (!includeServices) return null;
         const clicked = await page.evaluate((input) => {
           const normalize4 = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();
-          const cards = Array.from(document.querySelectorAll("button")).filter(
-            (button) => Boolean(button.querySelector("h3")) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText)
-          );
-          const exact = cards.find((button) => `${normalize4(button.querySelector("h3")?.textContent)}|${normalize4(button.innerText)}` === input.cardKey);
-          const target = exact ?? cards[input.cardIndex];
-          if (!target) return false;
-          target.click();
-          return true;
-        }, { cardIndex: card.cardIndex, cardKey: card.cardKey });
+          const headings = Array.from(document.querySelectorAll('div[role="heading"]'));
+          for (const heading of headings) {
+            let target = heading;
+            for (let hop = 0; hop < 8; hop++) {
+              const parent = target.parentElement;
+              if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+              target = parent;
+            }
+            if (target.closest('[role="dialog"], dialog')) continue;
+            const name = normalize4(heading.textContent);
+            if (!name) continue;
+            const lines = (target.innerText ?? "").split("\n").map((line) => normalize4(line)).filter(Boolean);
+            if (`${name}|${lines.join(" ")}` !== input.cardKey) continue;
+            target.click();
+            return true;
+          }
+          return false;
+        }, { cardKey: card.cardKey });
         if (!clicked) return null;
         const dialog = page.locator('dialog, [role="dialog"]').filter({ has: page.locator('h1, h2, [role="heading"]') }).last();
         await dialog.waitFor({ state: "visible", timeout: 6e3 }).catch(() => {
@@ -20145,7 +20170,9 @@ async function runMapsSearchWithRotation(options) {
         ...options,
         proxyZip,
         kernelProxyId: options.proxyMode === "none" ? void 0 : resolution2.kernelProxyId,
-        kernelProxyResolution: resolution2.resolution
+        kernelProxyResolution: resolution2.resolution,
+        serpRedirect: options.proxyMode === "none" && attemptIndex >= 1 && attemptIndex < maxAttempts - 1,
+        forceDirectEgress: options.proxyMode === "none" && attemptIndex >= 1
       };
       const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
       const baseCtx = currentCostContext();
@@ -29767,7 +29794,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.32.0";
+    PACKAGE_VERSION = "0.32.2";
   }
 });