mcp-scraper 0.3.24 → 0.3.26

package/dist/chunk-MS2UZ6HR.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.26";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-MS2UZ6HR.js.map

package/dist/chunk-MS2UZ6HR.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.26'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js}

@@ -1,6 +1,14 @@
+import {
+  postToMemoryLibrary
+} from "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE
-} from "./chunk-LFATOGDF.js";
+} from "./chunk-BP27CZ5Q.js";
+
+// src/lib/slugify.ts
+function slugify(s) {
+  return s.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
+}
 
 // src/workflows/artifact-writer.ts
 import { mkdir, readFile, stat, writeFile } from "fs/promises";
@@ -72,11 +80,6 @@ function rowsToCsv(headers, rows) {
   ].join("\n") + "\n";
 }
 
-// src/lib/slugify.ts
-function slugify(s) {
-  return s.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
-}
-
 // src/workflows/artifact-writer.ts
 function workflowOutputBaseDir(outputDir) {
   return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
@@ -117,6 +120,7 @@ var ArtifactWriter = class _ArtifactWriter {
   async remember(kind, label, path, rows) {
     const size = await stat(path).then((s) => s.size).catch(() => void 0);
     this.artifacts.push({ kind, label, path, bytes: size, rows });
+    await postToMemoryLibrary({ title: `${this.title} ${label}`, content: `artifact: ${path}`, source: `mcp-scraper-workflow:${this.workflowId}:${this.runId}` });
     return path;
   }
   async writeJson(label, relativePath, data) {
@@ -211,7 +215,7 @@ async function openWorkflowReport(id, outputDir) {
 
 // src/workflows/registry.ts
 import { readFile as readFile2 } from "fs/promises";
-import { z as z5 } from "zod";
+import { z as z6 } from "zod";
 
 // src/workflows/http-client.ts
 var WorkflowHttpClient = class {
@@ -662,20 +666,41 @@ ${summary}
   }
 };
 
-// src/workflows/workflows/local-competitive-audit.ts
+// src/workflows/workflows/get-leads.ts
 import { z as z3 } from "zod";
-var LocalCompetitiveAuditInputSchema = z3.object({
-  query: z3.string().min(1),
-  state: z3.string().min(2).default("TN"),
-  minPopulation: z3.number().int().min(0).default(1e5),
-  maxCities: z3.number().int().min(1).max(100).default(25),
-  maxResultsPerCity: z3.number().int().min(1).max(50).default(20),
-  hydrateTop: z3.number().int().min(0).max(10).default(5),
-  maxReviews: z3.number().int().min(0).max(500).default(50),
-  concurrency: z3.number().int().min(1).max(5).default(5),
-  proxyMode: z3.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
-  returnPartial: z3.boolean().default(true)
+var GetLeadsInputSchema = z3.object({
+  query: z3.string().min(1).describe('Business category, niche, or keyword to search on Google Maps, e.g. "roofers", "med spas", "dentists". Do not include the city here.'),
+  location: z3.string().min(1).describe('City / market to search, e.g. "Houston, TX" or "Austin, Texas".'),
+  maxResults: z3.number().int().min(1).max(50).default(25).describe("How many Maps businesses to collect for the market. Maximum 50."),
+  enrichWebsites: z3.boolean().default(true).describe("Visit each business website (home + contact pages) to harvest email and social links. Uses the proxy/browser-backed extractor so blocked sites still resolve."),
+  hydrateReviewCounts: z3.boolean().default(true).describe("Deep-dive each profile to confirm the review count and booking URL that the Maps search list omits."),
+  concurrency: z3.number().int().min(1).max(4).default(3).describe("How many businesses to enrich in parallel. Keep low to respect per-account concurrency limits."),
+  proxyMode: z3.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting for the Maps search. location targets the market; none is for local debugging.")
 });
+var LEADS_CSV_HEADERS = [
+  "position",
+  "business_name",
+  "review_stars",
+  "review_count",
+  "category",
+  "address",
+  "phone",
+  "website",
+  "email",
+  "email_domain_match",
+  "email_status",
+  "socials",
+  "booking_url",
+  "place_url",
+  "cid",
+  "source_location"
+];
+var EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
+var MAILTO_RE = /mailto:([^"'?>\s]+)/gi;
+var SOCIAL_RE = /https?:\/\/(?:www\.)?(?:facebook|instagram|linkedin|twitter|youtube|tiktok)\.com\/[^\s"'<>)]+/gi;
+var EMAIL_BAD = [".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", "sentry", "wix.com", "wixpress", "example.", "godaddy", "squarespace", "schema.org", "@2x", "u003e", "core-js"];
+var EMAIL_PREFIX = ["info@", "office@", "contact@", "sales@", "hello@", "admin@", "service@", "support@"];
+var SOCIAL_SKIP = ["/sharer", "/share?", "/intent/", "plugins/", "/dialog/", "platform.", "badge", "developers."];
 async function mapLimit(items, limit, fn) {
   const out = new Array(items.length);
   let next = 0;
@@ -689,6 +714,224 @@ async function mapLimit(items, limit, fn) {
   await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
   return out;
 }
+function hostRoot(url) {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "").split(".")[0].toLowerCase();
+  } catch {
+    return "";
+  }
+}
+function cleanEmails(raw) {
+  const out = /* @__PURE__ */ new Set();
+  for (const value of raw) {
+    const email = value.trim().toLowerCase().replace(/\.$/, "");
+    if (email.split("@").length !== 2) continue;
+    if (email.length > 60) continue;
+    if (EMAIL_BAD.some((bad) => email.includes(bad))) continue;
+    out.add(email);
+  }
+  return [...out];
+}
+function pickEmail(emails, host) {
+  if (!emails.length) return "";
+  const root = hostRoot(host);
+  for (const prefix of EMAIL_PREFIX) {
+    const hit = emails.find((e) => e.startsWith(prefix) && root && e.split("@")[1].includes(root));
+    if (hit) return hit;
+  }
+  const domainHit = emails.find((e) => root && e.split("@")[1].includes(root));
+  if (domainHit) return domainHit;
+  for (const prefix of EMAIL_PREFIX) {
+    const hit = emails.find((e) => e.startsWith(prefix));
+    if (hit) return hit;
+  }
+  return emails[0];
+}
+function parseContacts(html) {
+  const mailtos = [...html.matchAll(MAILTO_RE)].map((m) => decodeURIComponent(m[1]).split("?")[0]);
+  const inline = html.match(EMAIL_RE) ?? [];
+  const emails = cleanEmails([...mailtos, ...inline]);
+  const socials = [...new Set((html.match(SOCIAL_RE) ?? []).map((s) => s.replace(/[)"'<>]+$/, "")).filter((s) => !SOCIAL_SKIP.some((skip) => s.toLowerCase().includes(skip))))].slice(0, 5);
+  return { emails, socials };
+}
+function buildAttemptUrls(website) {
+  const urls = /* @__PURE__ */ new Set();
+  urls.add(website);
+  try {
+    const origin = new URL(website).origin;
+    urls.add(`${origin}/contact`);
+    urls.add(`${origin}/contact-us`);
+  } catch {
+    return [...urls];
+  }
+  return [...urls].slice(0, 3);
+}
+var getLeadsWorkflowDefinition = {
+  id: "get-leads",
+  title: "Get Leads",
+  description: "Build an outreach-ready local lead list: Google Maps search for a niche in a market, confirm review counts + booking URLs, then visit each business website (home + contact) to harvest email and social links, with proxy/browser-backed extraction so blocked sites still resolve. Saves a leads CSV.",
+  inputSchema: GetLeadsInputSchema,
+  async run(input, ctx) {
+    await ctx.artifacts.writeManifest("running", {}, [], []);
+    const warnings = [];
+    const errors = [];
+    const search = await ctx.client.post("/maps/search", {
+      query: input.query,
+      location: input.location,
+      maxResults: input.maxResults,
+      proxyMode: input.proxyMode
+    }, 18e4);
+    await ctx.artifacts.writeJson("Maps search raw", "raw/maps-search.json", search);
+    const seen = /* @__PURE__ */ new Set();
+    const deduped = search.results.filter((result) => {
+      const key = result.cid ?? result.placeUrl ?? result.name.toLowerCase();
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    });
+    const rows = await mapLimit(deduped, input.concurrency, async (result) => {
+      let reviewCount = result.reviewCount ?? "";
+      let bookingUrl = "";
+      let website = result.websiteUrl ?? "";
+      if (input.hydrateReviewCounts) {
+        try {
+          const place = await ctx.client.post("/maps/place", {
+            businessName: result.name,
+            location: input.location,
+            includeReviews: false
+          }, 18e4);
+          reviewCount = place.reviewCount ?? reviewCount;
+          bookingUrl = place.bookingUrl ?? "";
+          website = website || (place.website ?? "");
+        } catch (err) {
+          warnings.push(`Review hydration failed for ${result.name}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      }
+      let email = "";
+      let socials = [];
+      let emailStatus = website ? "no email found" : "no website";
+      let domainMatch = "";
+      if (input.enrichWebsites && website) {
+        const attempts = buildAttemptUrls(website);
+        let reached = false;
+        const allEmails = [];
+        const allSocials = /* @__PURE__ */ new Set();
+        for (const attemptUrl of attempts) {
+          try {
+            const page = await ctx.client.post("/extract-url", { url: attemptUrl }, 12e4);
+            reached = true;
+            const parsed = parseContacts(page.bodyHtml ?? page.bodyMarkdown ?? "");
+            allEmails.push(...parsed.emails);
+            parsed.socials.forEach((s) => allSocials.add(s));
+            if (parsed.emails.length) break;
+          } catch {
+            continue;
+          }
+        }
+        socials = [...allSocials].slice(0, 5);
+        email = pickEmail([...new Set(allEmails)], website);
+        if (email) {
+          domainMatch = hostRoot(website) && email.split("@")[1].includes(hostRoot(website)) ? "yes" : "NO-verify";
+          emailStatus = domainMatch === "yes" ? "ok" : "ok (off-domain \u2014 verify)";
+        } else if (!reached) {
+          emailStatus = "site unreachable";
+        } else if (socials.length) {
+          emailStatus = "form/social only";
+        }
+      }
+      return {
+        position: result.position,
+        business_name: result.name,
+        review_stars: result.rating ?? "",
+        review_count: reviewCount,
+        category: result.category ?? "",
+        address: result.address ?? "",
+        phone: result.phone ?? "",
+        website,
+        email,
+        email_domain_match: domainMatch,
+        email_status: emailStatus,
+        socials: socials.join(" ; "),
+        booking_url: bookingUrl,
+        place_url: result.placeUrl,
+        cid: result.cid ?? "",
+        source_location: input.location
+      };
+    });
+    await ctx.artifacts.writeCsv("Leads CSV", "exports/leads.csv", LEADS_CSV_HEADERS, rows);
+    await ctx.artifacts.writeJson("Leads evidence", "evidence.json", { input, search: { query: search.searchQuery, resultCount: search.resultCount }, rows });
+    const withEmail = rows.filter((r) => r.email).length;
+    const domainMatched = rows.filter((r) => r.email_domain_match === "yes").length;
+    const withReviewCount = rows.filter((r) => r.review_count).length;
+    const withSocials = rows.filter((r) => r.socials).length;
+    const withWebsite = rows.filter((r) => r.website).length;
+    const summary = `${rows.length} ${input.query} in ${input.location}: ${withEmail} emails (${domainMatched} domain-matched), ${withReviewCount} review counts, ${withSocials} with socials.`;
+    await ctx.artifacts.writeText("Summary", "summary.md", `# Get Leads
+
+${summary}
+`);
+    await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
+      "# Get Leads \u2014 next steps",
+      "",
+      "- [ ] Verify deliverability of the harvested emails before outreach (MX/SMTP check).",
+      '- [ ] For "form/social only" rows, use the website contact form or the captured social profile.',
+      '- [ ] Treat "off-domain \u2014 verify" emails with care; they may belong to a web developer or parent brand.',
+      "- [ ] Respect CAN-SPAM / GDPR \u2014 scraped contact data is not consent to email."
+    ].join("\n"));
+    const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
+      title: "Get Leads",
+      subtitle: `${input.query} \xB7 ${input.location}`,
+      summary,
+      warnings,
+      tables: [
+        {
+          title: "Leads",
+          columns: ["position", "business_name", "review_stars", "review_count", "phone", "website", "email", "email_status", "socials", "booking_url"],
+          rows
+        }
+      ]
+    }));
+    const status = errors.length ? "failed" : warnings.length ? "partial" : "succeeded";
+    const counts = {
+      businesses: rows.length,
+      withWebsite,
+      emails: withEmail,
+      domainMatchedEmails: domainMatched,
+      reviewCounts: withReviewCount,
+      withSocials
+    };
+    await ctx.artifacts.writeManifest(status, counts, warnings, errors);
+    return { title: "Get Leads", summary, status, counts, warnings, errors, reportPath };
+  }
+};
+
+// src/workflows/workflows/local-competitive-audit.ts
+import { z as z4 } from "zod";
+var LocalCompetitiveAuditInputSchema = z4.object({
+  query: z4.string().min(1),
+  state: z4.string().min(2).default("TN"),
+  minPopulation: z4.number().int().min(0).default(1e5),
+  maxCities: z4.number().int().min(1).max(100).default(25),
+  maxResultsPerCity: z4.number().int().min(1).max(50).default(20),
+  hydrateTop: z4.number().int().min(0).max(10).default(5),
+  maxReviews: z4.number().int().min(0).max(500).default(50),
+  concurrency: z4.number().int().min(1).max(5).default(5),
+  proxyMode: z4.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
+  returnPartial: z4.boolean().default(true)
+});
+async function mapLimit2(items, limit, fn) {
+  const out = new Array(items.length);
+  let next = 0;
+  async function worker() {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      out[index] = await fn(items[index], index);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
+  return out;
+}
 function numberFrom(value) {
   if (value === null || value === void 0 || value === "") return null;
   const parsed = Number(String(value).replace(/[^\d.]/g, ""));
@@ -742,7 +985,7 @@ var localCompetitiveAuditWorkflowDefinition = {
       seen.add(key);
       return true;
     });
-    const hydrated = await mapLimit(deduped, 3, async ({ city, result }, index) => {
+    const hydrated = await mapLimit2(deduped, 3, async ({ city, result }, index) => {
       try {
         const detail = await ctx.client.post("/maps/place", {
           businessName: result.name,
@@ -862,7 +1105,7 @@ ${summary}
 };
 
 // src/workflows/workflows/comparison-briefs.ts
-import { z as z4 } from "zod";
+import { z as z5 } from "zod";
 
 // src/workflows/workflows/seo-workflow-utils.ts
 var STOP_WORDS = /* @__PURE__ */ new Set([
@@ -1005,7 +1248,7 @@ function pageSummaryRow(page, source) {
     schema_types: (page.schemaTypes ?? []).join("; ")
   };
 }
-async function mapLimit2(items, limit, fn) {
+async function mapLimit3(items, limit, fn) {
   const out = new Array(items.length);
   let next = 0;
   async function worker() {
@@ -1020,49 +1263,49 @@ async function mapLimit2(items, limit, fn) {
 }
 
 // src/workflows/workflows/comparison-briefs.ts
-var ProxyModeSchema = z4.enum(["location", "configured", "none"]);
-var MapComparisonInputSchema = z4.object({
-  query: z4.string().min(1),
-  location: z4.string().optional(),
-  state: z4.string().optional(),
-  minPopulation: z4.number().int().min(0).default(1e5),
-  maxCities: z4.number().int().min(1).max(100).default(5),
-  maxResultsPerCity: z4.number().int().min(1).max(50).default(20),
-  hydrateTop: z4.number().int().min(0).max(10).default(5),
-  maxReviews: z4.number().int().min(0).max(500).default(25),
-  concurrency: z4.number().int().min(1).max(5).default(5),
+var ProxyModeSchema = z5.enum(["location", "configured", "none"]);
+var MapComparisonInputSchema = z5.object({
+  query: z5.string().min(1),
+  location: z5.string().optional(),
+  state: z5.string().optional(),
+  minPopulation: z5.number().int().min(0).default(1e5),
+  maxCities: z5.number().int().min(1).max(100).default(5),
+  maxResultsPerCity: z5.number().int().min(1).max(50).default(20),
+  hydrateTop: z5.number().int().min(0).max(10).default(5),
+  maxReviews: z5.number().int().min(0).max(500).default(25),
+  concurrency: z5.number().int().min(1).max(5).default(5),
   proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
-  returnPartial: z4.boolean().default(true)
+  returnPartial: z5.boolean().default(true)
 }).refine((input) => input.location || input.state, {
   message: "Either location or state is required for map-comparison"
 });
-var SerpComparisonInputSchema = z4.object({
-  keyword: z4.string().min(1),
-  domain: z4.string().optional(),
-  url: z4.string().url().optional(),
-  location: z4.string().optional(),
-  maxResults: z4.number().int().min(1).max(20).default(10),
-  maxQuestions: z4.number().int().min(1).max(200).default(40),
-  extractTop: z4.number().int().min(0).max(10).default(5),
-  includePaa: z4.boolean().default(true),
-  includeAiOverview: z4.boolean().default(true),
-  returnPartial: z4.boolean().default(true)
+var SerpComparisonInputSchema = z5.object({
+  keyword: z5.string().min(1),
+  domain: z5.string().optional(),
+  url: z5.string().url().optional(),
+  location: z5.string().optional(),
+  maxResults: z5.number().int().min(1).max(20).default(10),
+  maxQuestions: z5.number().int().min(1).max(200).default(40),
+  extractTop: z5.number().int().min(0).max(10).default(5),
+  includePaa: z5.boolean().default(true),
+  includeAiOverview: z5.boolean().default(true),
+  returnPartial: z5.boolean().default(true)
 });
-var PaaExpansionBriefInputSchema = z4.object({
-  keyword: z4.string().min(1),
-  location: z4.string().optional(),
-  maxQuestions: z4.number().int().min(1).max(300).default(80),
-  depth: z4.number().int().min(1).max(6).default(3),
-  returnPartial: z4.boolean().default(true)
+var PaaExpansionBriefInputSchema = z5.object({
+  keyword: z5.string().min(1),
+  location: z5.string().optional(),
+  maxQuestions: z5.number().int().min(1).max(300).default(80),
+  depth: z5.number().int().min(1).max(6).default(3),
+  returnPartial: z5.boolean().default(true)
 });
-var AiOverviewLanguageInputSchema = z4.object({
-  keyword: z4.string().min(1),
-  domain: z4.string().optional(),
-  url: z4.string().url().optional(),
-  location: z4.string().optional(),
-  maxQuestions: z4.number().int().min(1).max(200).default(40),
-  extractTop: z4.number().int().min(0).max(8).default(3),
-  returnPartial: z4.boolean().default(true)
+var AiOverviewLanguageInputSchema = z5.object({
+  keyword: z5.string().min(1),
+  domain: z5.string().optional(),
+  url: z5.string().url().optional(),
+  location: z5.string().optional(),
+  maxQuestions: z5.number().int().min(1).max(200).default(40),
+  extractTop: z5.number().int().min(0).max(8).default(3),
+  returnPartial: z5.boolean().default(true)
 });
 function businessRowsFromMaps(location, query, results) {
   return results.map((result) => ({
@@ -1179,7 +1422,7 @@ function pageGapRows(targetPage, competitorPages) {
   return rows.slice(0, 150);
 }
 async function extractPages(ctx, sources, warnings, labelPrefix) {
-  return mapLimit2(sources.filter((source) => source.url), 2, async (source, index) => {
+  return mapLimit3(sources.filter((source) => source.url), 2, async (source, index) => {
     try {
       const page = await ctx.client.post("/extract-url", { url: source.url }, 18e4);
       await ctx.artifacts.writeJson(`${labelPrefix} ${index + 1}`, `raw/extract-url/${labelPrefix.toLowerCase()}-${index + 1}.json`, page);
@@ -1228,7 +1471,7 @@ var mapComparisonWorkflowDefinition = {
     }
     const compareRows = comparisonRows(rows);
     const selected = compareRows.slice(0, input.hydrateTop * Math.max(1, input.location ? 1 : input.maxCities));
-    const hydrated = await mapLimit2(selected, 3, async (row, index) => {
+    const hydrated = await mapLimit3(selected, 3, async (row, index) => {
       try {
         const detail = await ctx.client.post("/maps/place", {
           businessName: row.business_name,
@@ -1563,6 +1806,7 @@ var aiOverviewLanguageWorkflowDefinition = {
 // src/workflows/registry.ts
 var DEFINITIONS = [
   directoryWorkflowDefinition,
+  getLeadsWorkflowDefinition,
   agentPacketWorkflowDefinition,
   localCompetitiveAuditWorkflowDefinition,
   mapComparisonWorkflowDefinition,
@@ -1669,6 +1913,7 @@ async function runWorkflowStep(id, rawInput, opts) {
 export {
   csvRecords,
   rowsToCsv,
+  slugify,
   workflowOutputBaseDir,
   listWorkflowReports,
   findWorkflowReport,
@@ -1680,4 +1925,4 @@ export {
   runWorkflow,
   runWorkflowStep
 };
-//# sourceMappingURL=chunk-H2R232HK.js.map
+//# sourceMappingURL=chunk-NNEIXK5L.js.map