mcp-scraper 0.30.0 → 0.32.0

package/dist/index.d.cts

@@ -8,6 +8,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
     proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
     proxyZip: z.ZodOptional<z.ZodString>;
+    keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
+    requireUsEgress: z.ZodOptional<z.ZodBoolean>;
+    maxAttempts: z.ZodOptional<z.ZodNumber>;
     debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
@@ -38,6 +41,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     pages: number;
     location?: string | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
@@ -52,6 +58,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device?: "desktop" | "mobile" | undefined;
     proxyMode?: "none" | "location" | "configured" | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;

package/dist/index.d.ts

@@ -8,6 +8,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device: z.ZodDefault<z.ZodEnum<["desktop", "mobile"]>>;
     proxyMode: z.ZodDefault<z.ZodEnum<["location", "configured", "none"]>>;
     proxyZip: z.ZodOptional<z.ZodString>;
+    keepDefaultProxy: z.ZodOptional<z.ZodBoolean>;
+    requireUsEgress: z.ZodOptional<z.ZodBoolean>;
+    maxAttempts: z.ZodOptional<z.ZodNumber>;
     debug: z.ZodDefault<z.ZodBoolean>;
     depth: z.ZodDefault<z.ZodNumber>;
     maxQuestions: z.ZodDefault<z.ZodNumber>;
@@ -38,6 +41,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     pages: number;
     location?: string | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     profileDir?: string | undefined;
     proxy?: string | undefined;
     kernelApiKey?: string | undefined;
@@ -52,6 +58,9 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     device?: "desktop" | "mobile" | undefined;
     proxyMode?: "none" | "location" | "configured" | undefined;
     proxyZip?: string | undefined;
+    keepDefaultProxy?: boolean | undefined;
+    requireUsEgress?: boolean | undefined;
+    maxAttempts?: number | undefined;
     debug?: boolean | undefined;
     depth?: number | undefined;
     maxQuestions?: number | undefined;

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-YH5DRFWQ.js";
+} from "./chunk-RDCCU7CQ.js";
 import {
   recordVendorUsage
 } from "./chunk-MTSBI7ZH.js";
 import "./chunk-K443GQY5.js";
-import "./chunk-ZQSHKWV4.js";
+import "./chunk-PTHY7NYD.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-ICT7DDHL.js";
 

package/dist/{server-PBPXND5G.js → server-OBOTNFD6.js}

@@ -16,7 +16,7 @@ import {
   harvest,
   normalizeLocation,
   resolveKernelProxyId
-} from "./chunk-YH5DRFWQ.js";
+} from "./chunk-RDCCU7CQ.js";
 import {
   HttpMcpToolExecutor,
   assessTranscriptSignal,
@@ -36,7 +36,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-T2CIPAOK.js";
+} from "./chunk-YOOEAXWP.js";
 import {
   auditImages,
   buildLinkReport,
@@ -65,7 +65,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-KAMKTSJQ.js";
+} from "./chunk-55M5CE5L.js";
 import "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
@@ -76,8 +76,8 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-ZQSHKWV4.js";
-import "./chunk-WQUF7AH7.js";
+} from "./chunk-PTHY7NYD.js";
+import "./chunk-CPQF3PYW.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -129,6 +129,7 @@ import {
 } from "./chunk-4767BA2O.js";
 import {
   CaptchaError,
+  ExtractionError,
   RECAPTCHA_INSTRUCTIONS,
   sanitizeVendorName
 } from "./chunk-M2S27J6Z.js";
@@ -13088,6 +13089,12 @@ function buildGoogleLocalResultsUrl(input) {
   if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
   return `https://www.google.com/search?${params.toString()}`;
 }
+function buildGoogleOrganicSearchUrl(input) {
+  const query = [input.query, input.location].filter(Boolean).join(" ");
+  const params = new URLSearchParams({ q: query, gl: input.gl, hl: input.hl, pws: "0" });
+  if (input.location) params.set("uule", encodeUule(normalizeLocation(input.location)));
+  return `https://www.google.com/search?${params.toString()}`;
+}
 var MapsSearchExtractor = class {
   constructor(driver) {
     this.driver = driver;
@@ -13096,13 +13103,14 @@ var MapsSearchExtractor = class {
   async extract(options) {
     const startMs = Date.now();
     const searchQuery = [options.query, options.location].filter(Boolean).join(" ");
-    const searchUrl = buildGoogleLocalResultsUrl(options);
+    const searchUrl = buildGoogleOrganicSearchUrl(options);
     const config = {
       headless: options.headless,
       kernelApiKey: options.kernelApiKey,
       kernelProxyId: options.kernelProxyId,
       kernelProxyResolution: options.kernelProxyResolution,
       proxyMode: options.proxyMode,
+      keepDefaultProxy: options.proxyMode !== "location" && options.proxyMode !== "configured" && !options.kernelProxyId,
       viewport: { width: 1280, height: 900 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
       debug: options.debug
@@ -13113,6 +13121,10 @@ var MapsSearchExtractor = class {
       await page.goto(searchUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
       await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
       if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
+      const reachedFinder = await this.clickIntoLocalFinder(page);
+      if (!reachedFinder) throw new ExtractionError('Google "More businesses" control was not found on the organic results page');
+      await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
+      if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
       const results = await this.collectResults(page, options);
       return {
         query: options.query,
@@ -13129,6 +13141,17 @@ var MapsSearchExtractor = class {
       await this.driver.close();
     }
   }
+  async clickIntoLocalFinder(page) {
+    const control = page.getByRole("link", { name: /more (businesses|places|results)/i }).or(page.getByRole("button", { name: /more (businesses|places|results)/i })).first();
+    const visible = await control.isVisible({ timeout: 1e4 }).catch(() => false);
+    if (!visible) return false;
+    await control.click({ timeout: 1e4 }).catch(() => {
+    });
+    await page.waitForTimeout(5e3);
+    return page.evaluate(
+      () => /[?&]udm=1/.test(location.href) || document.querySelectorAll('a[href*="/maps/place/"]').length > 0
+    );
+  }
   async detectBlock(page) {
     return page.evaluate(() => {
       const text2 = document.body.innerText.slice(0, 2e3);
@@ -20698,7 +20721,7 @@ loadChannels();
 
 // src/api/vault-routes.ts
 import { Hono as Hono22 } from "hono";
-var VAULTS = ["People", "Deals", "Projects", "Tasks", "Communications", "Calendar"];
+var VAULTS = ["People", "Organizations", "Deals", "Projects", "Tasks", "Communications", "Calendar"];
 var PROJECT_TYPES = ["codebase", "personal", "work", "client"];
 var PROJECT_KIND_BY_TYPE = {
   codebase: "codebase",
@@ -20782,6 +20805,7 @@ function normalizeRelationshipProps(vault, props) {
 }
 function defaultType(vault) {
   if (vault === "People" || vault === "Communications") return "unclassified";
+  if (vault === "Organizations") return "organization";
   if (vault === "Deals") return "deal";
   if (vault === "Projects") return "project";
   if (vault === "Tasks") return "task";
@@ -20789,7 +20813,7 @@ function defaultType(vault) {
   return "unclassified";
 }
 function defaultStatus(vault) {
-  if (vault === "People") return "active";
+  if (vault === "People" || vault === "Organizations") return "active";
   if (vault === "Deals") return "lead";
   if (vault === "Projects") return "active";
   if (vault === "Tasks") return "to_do";
@@ -20802,6 +20826,12 @@ function nonEmptyStrings(value) {
 function validateNewRecord(vault, props) {
   const type = typeof props.type === "string" ? props.type : "";
   if (vault === "People" && !["person", "organization"].includes(type)) return "People records must be a person or organization";
+  if (vault === "Organizations") {
+    if (type !== "organization") return "Organizations records must have type organization";
+    if (typeof props.organization_name !== "string" || !props.organization_name.trim()) {
+      return "Organizations require organization_name";
+    }
+  }
   if (vault === "Deals") {
     if (type !== "deal") return "Deals records must have type deal";
     if (typeof props.organization_ref !== "string" && nonEmptyStrings(props.person_refs).length === 0) {
@@ -27375,7 +27405,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-RYP5EOV5.js");
+  const { drainQueue } = await import("./worker-7DJYUACV.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -27578,4 +27608,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-PBPXND5G.js.map
+//# sourceMappingURL=server-OBOTNFD6.js.map