mcp-scraper 0.32.0 → 0.32.2

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-RDCCU7CQ.js";
+} from "./chunk-AEMABLEL.js";
 import {
   recordVendorUsage
 } from "./chunk-MTSBI7ZH.js";
 import "./chunk-K443GQY5.js";
-import "./chunk-PTHY7NYD.js";
+import "./chunk-V73MPRU6.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-ICT7DDHL.js";
 

package/dist/{server-OBOTNFD6.js → server-LTZDJHFG.js}

@@ -15,8 +15,9 @@ import {
   encodeUule,
   harvest,
   normalizeLocation,
-  resolveKernelProxyId
-} from "./chunk-RDCCU7CQ.js";
+  resolveKernelProxyId,
+  serpArrivalUrl
+} from "./chunk-AEMABLEL.js";
 import {
   HttpMcpToolExecutor,
   assessTranscriptSignal,
@@ -36,7 +37,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-YOOEAXWP.js";
+} from "./chunk-XEED325T.js";
 import {
   auditImages,
   buildLinkReport,
@@ -65,7 +66,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-55M5CE5L.js";
+} from "./chunk-MKZIL4EA.js";
 import "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
@@ -76,8 +77,8 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-PTHY7NYD.js";
-import "./chunk-CPQF3PYW.js";
+} from "./chunk-V73MPRU6.js";
+import "./chunk-XLO5CZ6T.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -13083,6 +13084,7 @@ import { Hono as Hono11 } from "hono";
 // src/extractor/MapsSearchExtractor.ts
 var LOCAL_RESULTS_PAGE_LIMIT = 8;
 var LOCAL_RESULTS_WAIT_MS = 1e3;
+var MAPS_REDIRECT_BASE = "https://serp-redirector.vercel.app";
 function buildGoogleLocalResultsUrl(input) {
   const query = [input.query, input.location].filter(Boolean).join(" ");
   const params = new URLSearchParams({ q: query, udm: "1", gl: input.gl, hl: input.hl, pws: "0" });
@@ -13104,21 +13106,23 @@ var MapsSearchExtractor = class {
     const startMs = Date.now();
     const searchQuery = [options.query, options.location].filter(Boolean).join(" ");
     const searchUrl = buildGoogleOrganicSearchUrl(options);
+    const keepDefaultProxy = !options.forceDirectEgress && options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId;
     const config = {
       headless: options.headless,
       kernelApiKey: options.kernelApiKey,
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
-      keepDefaultProxy: options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId,
+      keepDefaultProxy,
       viewport: { width: 1280, height: 900 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       debug: options.debug
     };
+    const navUrl = options.serpRedirect ? serpArrivalUrl(searchUrl, process.env.SERP_REDIRECT_BASE?.trim() || MAPS_REDIRECT_BASE) : searchUrl;
     try {
       await this.driver.launch(config);
       const page = this.driver.getPage();
-      await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
+      await page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
       await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
       if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
       const reachedFinder = await this.clickIntoLocalFinder(page);
@@ -13205,29 +13209,38 @@ var MapsSearchExtractor = class {
       };
       const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
       const addressPattern = /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i;
-      const cards = Array.from(document.querySelectorAll("button")).filter((button) => {
-        const name = normalize(button.querySelector("h3")?.textContent);
-        return Boolean(name) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText);
+      const headings = Array.from(document.querySelectorAll('div[role="heading"]')).filter((heading) => {
+        const text2 = normalize(heading.textContent);
+        return Boolean(text2) && !/^(?:businesses|places|sponsored|people also ask|discussions and forums|more businesses|more places|custom date range|choose what you)/i.test(text2 ?? "");
       });
       const seen = /* @__PURE__ */ new Set();
       const out = [];
-      for (const [cardIndex, card] of cards.entries()) {
-        const name = normalize(card.querySelector("h3")?.textContent);
+      let cardIndex = -1;
+      for (const heading of headings) {
+        let card = heading;
+        for (let hop = 0; hop < 8; hop++) {
+          const parent = card.parentElement;
+          if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+          card = parent;
+        }
+        if (card.closest('[role="dialog"], dialog')) continue;
+        const name = normalize(heading.textContent);
         if (!name) continue;
-        const text2 = normalize(card.innerText) ?? name;
-        const key = `${name.toLowerCase()}|${text2.toLowerCase()}`;
+        const lines = (card.innerText ?? "").split("\n").map((line) => normalize(line)).filter((line) => Boolean(line));
+        if (/^sponsored$/i.test(lines[0] ?? "")) continue;
+        const ratingLine = lines.find((line) => /\d(?:\.\d)?\s*\(/.test(line)) ?? "";
+        const rating = ratingLine.match(/(\d(?:\.\d)?)\s*\(/)?.[1] ?? null;
+        const reviewCount = ratingLine.match(/\((\d[\d,]*)\)/)?.[1]?.replace(/,/g, "") ?? null;
+        const category = normalize(ratingLine.split("\xB7").slice(1).join("\xB7"));
+        const phone = lines.map((line) => line.match(phonePattern)?.[0]).find((value) => Boolean(value)) ?? null;
+        const hoursStatus = normalize((lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? "").split("\xB7")[0] ?? "");
+        const address = lines.find((line) => addressPattern.test(line)) ?? null;
+        const websiteUrl = Array.from(card.querySelectorAll("a[href]")).map((anchor) => anchor.href).find((href) => /^https?:/i.test(href) && !/google\.|gstatic|googleusercontent/.test(href)) ?? null;
+        if (!rating && !(phone && websiteUrl)) continue;
+        const key = `${name.toLowerCase()}|${lines.join(" ").toLowerCase()}`;
         if (seen.has(key)) continue;
         seen.add(key);
-        const lines = text2.split(/\n+/).map((line) => normalize(line)).filter((line) => Boolean(line));
-        const aria = normalize(card.getAttribute("aria-label"));
-        const rating = (aria ?? text2).match(/(?:rated\s+)?(\d(?:\.\d)?)\s*(?:out of 5)?/i)?.[1] ?? null;
-        const reviewCount = (aria ?? text2).match(/(?:\(|·\s*)([\d,]+)\s+(?:user\s+)?reviews?\b/i)?.[1]?.replace(/,/g, "") ?? null;
-        const phone = lines.find((line) => phonePattern.test(line)) ?? null;
-        const address = lines.find((line) => addressPattern.test(line)) ?? null;
-        const hoursStatus = lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? null;
-        const category = lines.find(
-          (line) => line !== name && line !== phone && line !== address && line !== hoursStatus && !/^(?:rated|\d[\d.,]*\s*(?:out of 5|reviews?)?)\b/i.test(line) && !/sponsored|website|directions|call|save|share|more/i.test(line)
-        ) ?? null;
+        cardIndex += 1;
         const mapsSearchUrl = `https://www.google.com/maps/search/${encodeURIComponent([name, locationHint].filter(Boolean).join(" "))}`;
         out.push({
           cardIndex,
@@ -13242,7 +13255,7 @@ var MapsSearchExtractor = class {
           address,
           phone,
           hoursStatus,
-          websiteUrl: null,
+          websiteUrl,
           directionsUrl: `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent([name, address].filter(Boolean).join(", "))}`,
           metadata: lines.slice(0, 20)
         });
@@ -13251,17 +13264,27 @@ var MapsSearchExtractor = class {
     }, fallbackLocation);
   }
   async openCardAndExtractDialog(page, card, includeServices) {
+    if (!includeServices) return null;
     const clicked = await page.evaluate((input) => {
       const normalize = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();
-      const cards = Array.from(document.querySelectorAll("button")).filter(
-        (button) => Boolean(button.querySelector("h3")) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText)
-      );
-      const exact = cards.find((button) => `${normalize(button.querySelector("h3")?.textContent)}|${normalize(button.innerText)}` === input.cardKey);
-      const target = exact ?? cards[input.cardIndex];
-      if (!target) return false;
-      target.click();
-      return true;
-    }, { cardIndex: card.cardIndex, cardKey: card.cardKey });
+      const headings = Array.from(document.querySelectorAll('div[role="heading"]'));
+      for (const heading of headings) {
+        let target = heading;
+        for (let hop = 0; hop < 8; hop++) {
+          const parent = target.parentElement;
+          if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+          target = parent;
+        }
+        if (target.closest('[role="dialog"], dialog')) continue;
+        const name = normalize(heading.textContent);
+        if (!name) continue;
+        const lines = (target.innerText ?? "").split("\n").map((line) => normalize(line)).filter(Boolean);
+        if (`${name}|${lines.join(" ")}` !== input.cardKey) continue;
+        target.click();
+        return true;
+      }
+      return false;
+    }, { cardKey: card.cardKey });
     if (!clicked) return null;
     const dialog = page.locator('dialog, [role="dialog"]').filter({ has: page.locator('h1, h2, [role="heading"]') }).last();
     await dialog.waitFor({ state: "visible", timeout: 6e3 }).catch(() => {
@@ -13959,7 +13982,9 @@ async function runMapsSearchWithRotation(options) {
         ...options,
         proxyZip,
         kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
-        kernelProxyResolution: resolution.resolution
+        kernelProxyResolution: resolution.resolution,
+        serpRedirect: options.proxyMode === "none" && attemptIndex >= 1 && attemptIndex < maxAttempts - 1,
+        forceDirectEgress: options.proxyMode === "none" && attemptIndex >= 1
       };
       const isHeadfulEscalation = attemptIndex === maxAttempts - 1;
       const baseCtx = currentCostContext();
@@ -27405,7 +27430,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-7DJYUACV.js");
+  const { drainQueue } = await import("./worker-ZZCZNY3R.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -27608,4 +27633,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-OBOTNFD6.js.map
+//# sourceMappingURL=server-LTZDJHFG.js.map