mcp-scraper 0.31.0 → 0.32.1

package/dist/index.d.cts

@@ -8,6 +8,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
     proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
     proxyZip: z.ZodOptional<z.ZodString>;
+    keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
+    requireUsEgress: z.ZodOptional<z.ZodBoolean>;
+    maxAttempts: z.ZodOptional<z.ZodNumber>;
     debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
@@ -38,6 +41,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     pages: number;
     location?: string | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
@@ -52,6 +58,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device?: "desktop" | "mobile" | undefined;
     proxyMode?: "none" | "location" | "configured" | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;

package/dist/index.d.ts

@@ -8,6 +8,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
     proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
     proxyZip: z.ZodOptional<z.ZodString>;
+    keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
+    requireUsEgress: z.ZodOptional<z.ZodBoolean>;
+    maxAttempts: z.ZodOptional<z.ZodNumber>;
     debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
@@ -38,6 +41,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     pages: number;
     location?: string | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
@@ -52,6 +58,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device?: "desktop" | "mobile" | undefined;
     proxyMode?: "none" | "location" | "configured" | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-YH5DRFWQ.js";
+} from "./chunk-RDCCU7CQ.js";
 import {
   recordVendorUsage
 } from "./chunk-MTSBI7ZH.js";
 import "./chunk-K443GQY5.js";
-import "./chunk-ZQSHKWV4.js";
+import "./chunk-PTHY7NYD.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-ICT7DDHL.js";
 

package/dist/{server-DS3ZSNNO.js → server-GXTYKIAX.js}

@@ -16,7 +16,7 @@ import {
   harvest,
   normalizeLocation,
   resolveKernelProxyId
-} from "./chunk-YH5DRFWQ.js";
+} from "./chunk-RDCCU7CQ.js";
 import {
   HttpMcpToolExecutor,
   assessTranscriptSignal,
@@ -36,7 +36,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-ZOXL3VWO.js";
+} from "./chunk-2FY4VBAY.js";
 import {
   auditImages,
   buildLinkReport,
@@ -65,7 +65,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-KAMKTSJQ.js";
+} from "./chunk-55M5CE5L.js";
 import "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
@@ -76,8 +76,8 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-ZQSHKWV4.js";
-import "./chunk-W3GYPHFL.js";
+} from "./chunk-PTHY7NYD.js";
+import "./chunk-VJBS6RPB.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -129,6 +129,7 @@ import {
 } from "./chunk-4767BA2O.js";
 import {
   CaptchaError,
+  ExtractionError,
   RECAPTCHA_INSTRUCTIONS,
   sanitizeVendorName
 } from "./chunk-M2S27J6Z.js";
@@ -13088,6 +13089,12 @@ function buildGoogleLocalResultsUrl(input) {
   if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
   return `https://www.google.com/search?${params.toString()}`;
 }
+function buildGoogleOrganicSearchUrl(input) {
+  const query = [input.query, input.location].filter(Boolean).join(" ");
+  const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
+  if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
+  return `https://www.google.com/search?${params.toString()}`;
+}
 var MapsSearchExtractor = class {
   constructor(driver) {
     this.driver = driver;
@@ -13096,13 +13103,14 @@ var MapsSearchExtractor = class {
   async extract(options) {
     const startMs = Date.now();
     const searchQuery = [options.query, options.location].filter(Boolean).join(" ");
-    const searchUrl = buildGoogleLocalResultsUrl(options);
+    const searchUrl = buildGoogleOrganicSearchUrl(options);
     const config = {
       headless: options.headless,
       kernelApiKey: options.kernelApiKey,
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
+      keepDefaultProxy: options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId,
       viewport: { width: 1280, height: 900 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       debug: options.debug
@@ -13113,6 +13121,10 @@ var MapsSearchExtractor = class {
       await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
       await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
       if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
+      const reachedFinder = await this.clickIntoLocalFinder(page);
+      if (!reachedFinder) throw new ExtractionError('Google "More businesses" control was not found on the organic results page');
+      await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
+      if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
       const results = await this.collectResults(page, options);
       return {
         query: options.query,
@@ -13129,6 +13141,17 @@ var MapsSearchExtractor = class {
       await this.driver.close();
     }
   }
+  async clickIntoLocalFinder(page) {
+    const control = page.getByRole("link", { name: /more (businesses|places|results)/i }).or(page.getByRole("button", { name: /more (businesses|places|results)/i })).first();
+    const visible = await control.isVisible({ timeout: 1e4 }).catch(() => false);
+    if (!visible) return false;
+    await control.click({ timeout: 1e4 }).catch(() => {
+    });
+    await page.waitForTimeout(5e3);
+    return page.evaluate(
+      () => /[?&]udm=1/.test(location.href) || document.querySelectorAll('a[href*="/maps/place/"]').length > 0
+    );
+  }
   async detectBlock(page) {
     return page.evaluate(() => {
       const text2 = document.body.innerText.slice(0, 2e3);
@@ -13182,29 +13205,38 @@ var MapsSearchExtractor = class {
       };
       const phonePattern = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}/;
       const addressPattern = /\b[A-Z]{2}\s+\d{5}(?:-\d{4})?\b|\b(?:St|Street|Ave|Avenue|Rd|Road|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pkwy|Parkway)\b/i;
-      const cards = Array.from(document.querySelectorAll("button")).filter((button) => {
-        const name = normalize(button.querySelector("h3")?.textContent);
-        return Boolean(name) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText);
+      const headings = Array.from(document.querySelectorAll('div[role="heading"]')).filter((heading) => {
+        const text2 = normalize(heading.textContent);
+        return Boolean(text2) && !/^(?:businesses|places|sponsored|people also ask|discussions and forums|more businesses|more places|custom date range|choose what you)/i.test(text2 ?? "");
       });
       const seen = /* @__PURE__ */ new Set();
       const out = [];
-      for (const [cardIndex, card] of cards.entries()) {
-        const name = normalize(card.querySelector("h3")?.textContent);
+      let cardIndex = -1;
+      for (const heading of headings) {
+        let card = heading;
+        for (let hop = 0; hop < 8; hop++) {
+          const parent = card.parentElement;
+          if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+          card = parent;
+        }
+        if (card.closest('[role="dialog"], dialog')) continue;
+        const name = normalize(heading.textContent);
         if (!name) continue;
-        const text2 = normalize(card.innerText) ?? name;
-        const key = `${name.toLowerCase()}|${text2.toLowerCase()}`;
+        const lines = (card.innerText ?? "").split("\n").map((line) => normalize(line)).filter((line) => Boolean(line));
+        if (/^sponsored$/i.test(lines[0] ?? "")) continue;
+        const ratingLine = lines.find((line) => /\d(?:\.\d)?\s*\(/.test(line)) ?? "";
+        const rating = ratingLine.match(/(\d(?:\.\d)?)\s*\(/)?.[1] ?? null;
+        const reviewCount = ratingLine.match(/\((\d[\d,]*)\)/)?.[1]?.replace(/,/g, "") ?? null;
+        const category = normalize(ratingLine.split("\xB7").slice(1).join("\xB7"));
+        const phone = lines.map((line) => line.match(phonePattern)?.[0]).find((value) => Boolean(value)) ?? null;
+        const hoursStatus = normalize((lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? "").split("\xB7")[0] ?? "");
+        const address = lines.find((line) => addressPattern.test(line)) ?? null;
+        const websiteUrl = Array.from(card.querySelectorAll("a[href]")).map((anchor) => anchor.href).find((href) => /^https?:/i.test(href) && !/google\.|gstatic|googleusercontent/.test(href)) ?? null;
+        if (!rating && !(phone && websiteUrl)) continue;
+        const key = `${name.toLowerCase()}|${lines.join(" ").toLowerCase()}`;
         if (seen.has(key)) continue;
         seen.add(key);
-        const lines = text2.split(/\n+/).map((line) => normalize(line)).filter((line) => Boolean(line));
-        const aria = normalize(card.getAttribute("aria-label"));
-        const rating = (aria ?? text2).match(/(?:rated\s+)?(\d(?:\.\d)?)\s*(?:out of 5)?/i)?.[1] ?? null;
-        const reviewCount = (aria ?? text2).match(/(?:\(|·\s*)([\d,]+)\s+(?:user\s+)?reviews?\b/i)?.[1]?.replace(/,/g, "") ?? null;
-        const phone = lines.find((line) => phonePattern.test(line)) ?? null;
-        const address = lines.find((line) => addressPattern.test(line)) ?? null;
-        const hoursStatus = lines.find((line) => /\b(?:open|closed|opens|closes)\b/i.test(line)) ?? null;
-        const category = lines.find(
-          (line) => line !== name && line !== phone && line !== address && line !== hoursStatus && !/^(?:rated|\d[\d.,]*\s*(?:out of 5|reviews?)?)\b/i.test(line) && !/sponsored|website|directions|call|save|share|more/i.test(line)
-        ) ?? null;
+        cardIndex += 1;
         const mapsSearchUrl = `https://www.google.com/maps/search/${encodeURIComponent([name, locationHint].filter(Boolean).join(" "))}`;
         out.push({
           cardIndex,
@@ -13219,7 +13251,7 @@ var MapsSearchExtractor = class {
           address,
           phone,
           hoursStatus,
-          websiteUrl: null,
+          websiteUrl,
           directionsUrl: `https://www.google.com/maps/dir/?api=1&destination=${encodeURIComponent([name, address].filter(Boolean).join(", "))}`,
           metadata: lines.slice(0, 20)
         });
@@ -13228,17 +13260,27 @@ var MapsSearchExtractor = class {
     }, fallbackLocation);
   }
   async openCardAndExtractDialog(page, card, includeServices) {
+    if (!includeServices) return null;
     const clicked = await page.evaluate((input) => {
       const normalize = (value) => (value ?? "").replace(/\s+/g, " ").trim().toLowerCase();
-      const cards = Array.from(document.querySelectorAll("button")).filter(
-        (button) => Boolean(button.querySelector("h3")) && !button.closest('[role="dialog"], dialog') && !/\bsponsored\b/i.test(button.innerText)
-      );
-      const exact = cards.find((button) => `${normalize(button.querySelector("h3")?.textContent)}|${normalize(button.innerText)}` === input.cardKey);
-      const target = exact ?? cards[input.cardIndex];
-      if (!target) return false;
-      target.click();
-      return true;
-    }, { cardIndex: card.cardIndex, cardKey: card.cardKey });
+      const headings = Array.from(document.querySelectorAll('div[role="heading"]'));
+      for (const heading of headings) {
+        let target = heading;
+        for (let hop = 0; hop < 8; hop++) {
+          const parent = target.parentElement;
+          if (!parent || parent.querySelectorAll('div[role="heading"]').length > 1) break;
+          target = parent;
+        }
+        if (target.closest('[role="dialog"], dialog')) continue;
+        const name = normalize(heading.textContent);
+        if (!name) continue;
+        const lines = (target.innerText ?? "").split("\n").map((line) => normalize(line)).filter(Boolean);
+        if (`${name}|${lines.join(" ")}` !== input.cardKey) continue;
+        target.click();
+        return true;
+      }
+      return false;
+    }, { cardKey: card.cardKey });
     if (!clicked) return null;
     const dialog = page.locator('dialog, [role="dialog"]').filter({ has: page.locator('h1, h2, [role="heading"]') }).last();
     await dialog.waitFor({ state: "visible", timeout: 6e3 }).catch(() => {
@@ -27382,7 +27424,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-RYP5EOV5.js");
+  const { drainQueue } = await import("./worker-7DJYUACV.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -27585,4 +27627,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-DS3ZSNNO.js.map
+//# sourceMappingURL=server-GXTYKIAX.js.map