@mgsoftwarebv/mg-dashboard-mcp 2.6.0 → 2.6.3

package/dist/index.js

@@ -723,7 +723,7 @@ var AGENT_TOOLS = [
   },
   {
     name: "web-fetch",
-    description: "Fetch a web page and return its text content (HTML tags stripped). Use this to read company websites, extract contact info, about pages, etc. Returns at most 15000 characters of cleaned text.",
+    description: "Fetch a web page and return its text content (HTML tags stripped). Automatically extracts emails and phone numbers found on the page. Returns at most 15000 characters of cleaned text.",
     inputSchema: {
       type: "object",
       properties: {
@@ -732,6 +732,28 @@ var AGENT_TOOLS = [
       },
       required: ["url"]
     }
+  },
+  {
+    name: "web-find-contacts",
+    description: "POWERFUL contact finder. Crawls a company website (homepage + contact/about/team pages), extracts ALL emails, phone numbers, and LinkedIn URLs via regex + mailto/tel parsing. Also searches Google for the company email as fallback. Returns structured contact data. USE THIS for every lead instead of manually browsing contact pages.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url: {
+          type: "string",
+          description: 'Company website URL or domain (e.g. "https://example.nl" or "example.nl")'
+        },
+        company_name: {
+          type: "string",
+          description: "Company name (improves Google search accuracy)"
+        },
+        include_search: {
+          type: "boolean",
+          description: "Also search Google for emails (default: true)"
+        }
+      },
+      required: ["url"]
+    }
   }
 ];
 var AGENT_TOOL_NAMES = new Set(AGENT_TOOLS.map((t) => t.name));
@@ -747,7 +769,8 @@ var AGENT_TOOL_MODULE_MAP = {
   "agent-save-email-draft": "agent_reporting",
   "agent-complete-target": "agent_reporting",
   "web-search": "agent_reporting",
-  "web-fetch": "agent_reporting"
+  "web-fetch": "agent_reporting",
+  "web-find-contacts": "agent_reporting"
 };
 function clamp(val, min, max) {
   return Math.max(min, Math.min(max, val));
@@ -788,6 +811,242 @@ async function webSearch(query, maxResults) {
   }
   return results;
 }
+var EMAIL_REGEX = /[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g;
+var PHONE_NL_REGEX = /(?:\+31|0)[\s\-.]?\(?\d{1,3}\)?[\s\-.]?\d{3,4}[\s\-.]?\d{2,4}/g;
+var LINKEDIN_REGEX = /https?:\/\/(?:www\.)?linkedin\.com\/(?:in|company)\/[a-zA-Z0-9\-_%]+\/?/gi;
+var NOISE_EMAIL_PATTERNS = [
+  /noreply@/i,
+  /no-reply@/i,
+  /mailer-daemon@/i,
+  /postmaster@/i,
+  /@example\./i,
+  /test@/i,
+  /@wix\.com$/i,
+  /@sentry\.io$/i,
+  /@wordpress\./i,
+  /@gravatar\.com$/i,
+  /@schema\.org$/i,
+  /@w3\.org$/i,
+  /@facebook\.com$/i,
+  /@google\.com$/i,
+  /@twitter\.com$/i,
+  /@github\.com$/i,
+  /@cloudflare\./i,
+  /@vercel\./i,
+  /@netlify\./i,
+  /@cookiebot\./i,
+  /@hotjar\./i,
+  /@hubspot\./i,
+  /@mailchimp\./i,
+  /@googleusercontent/i,
+  /@gstatic/i,
+  /@youtube/i,
+  /@recaptcha/i,
+  /@privacy/i,
+  /@cookie/i,
+  /@gdpr/i,
+  /@dynamicweb\./i,
+  /@placeholder\./i,
+  /@yourcompany\./i,
+  /@company\./i,
+  /smith@/i,
+  /doe@/i,
+  /demo@/i,
+  /sample@/i,
+  /naam@/i,
+  /voorbeeld/i,
+  /your-?email/i,
+  /email@/i,
+  /@domein\./i,
+  /@bedrijf\./i,
+  /@domain\./i,
+  /@sentry/i,
+  /@wixpress/i,
+  /@lieferkassen/i,
+  /john@/i,
+  /jane@/i
+];
+var CONTACT_PATH_KEYWORDS = [
+  "contact",
+  "about",
+  "over-ons",
+  "over",
+  "team",
+  "medewerker",
+  "impressum",
+  "imprint",
+  "wie-zijn",
+  "ons-team",
+  "werknemers",
+  "organisatie",
+  "zakelijk",
+  "bedrijfsinfo",
+  "neem-contact",
+  "footer"
+];
+var CONTACT_PATHS_TO_TRY = [
+  "/contact",
+  "/over-ons",
+  "/about",
+  "/about-us",
+  "/team",
+  "/contact-us",
+  "/over",
+  "/wie-zijn-wij",
+  "/ons-team",
+  "/zakelijk/over-ons",
+  "/medewerkers",
+  "/organisatie",
+  "/bedrijfsinformatie",
+  "/impressum"
+];
+function decodeUnicodeEscapes(text) {
+  return text.replace(
+    /\\u([0-9a-fA-F]{4})/g,
+    (_, hex) => String.fromCharCode(parseInt(hex, 16))
+  );
+}
+function decodeHtmlEntities(text) {
+  return text.replace(/&#(\d+);/g, (_, num) => String.fromCharCode(parseInt(num))).replace(
+    /&#x([0-9a-fA-F]+);/g,
+    (_, hex) => String.fromCharCode(parseInt(hex, 16))
+  );
+}
+function extractEmailsFromHtml(html) {
+  const emails = /* @__PURE__ */ new Set();
+  const decoded = decodeHtmlEntities(decodeUnicodeEscapes(html));
+  const mailtoRegex = /mailto:([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})/gi;
+  let m;
+  for (const src of [html, decoded]) {
+    while ((m = mailtoRegex.exec(src)) !== null) emails.add(m[1].toLowerCase());
+  }
+  const noStyles = decoded.replace(/<style[\s\S]*?<\/style>/gi, "");
+  const textEmails = noStyles.match(EMAIL_REGEX) || [];
+  for (const e of textEmails) {
+    const lower = e.toLowerCase();
+    if (!/\.(png|jpg|jpeg|gif|svg|webp|css|js|woff|ttf|eot|ico)$/i.test(lower)) {
+      emails.add(lower);
+    }
+  }
+  return [...emails].filter((e) => !NOISE_EMAIL_PATTERNS.some((p) => p.test(e)));
+}
+function extractPhonesFromHtml(html) {
+  const phones = /* @__PURE__ */ new Set();
+  const decoded = decodeHtmlEntities(decodeUnicodeEscapes(html));
+  const telRegex = /href="tel:([^"]+)"/gi;
+  let m;
+  for (const src of [html, decoded]) {
+    while ((m = telRegex.exec(src)) !== null) {
+      const clean = m[1].replace(/[\s\-().]/g, "");
+      if (clean.length >= 10 && clean.length <= 14) phones.add(m[1].trim());
+    }
+  }
+  const stripped = decoded.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ");
+  const textPhones = stripped.match(PHONE_NL_REGEX) || [];
+  for (const p of textPhones) {
+    const clean = p.replace(/[\s\-().]/g, "");
+    if (clean.length >= 10 && clean.length <= 14) phones.add(p.trim());
+  }
+  return [...phones].slice(0, 10);
+}
+function extractLinkedInFromHtml(html) {
+  const links = /* @__PURE__ */ new Set();
+  const matches = html.match(LINKEDIN_REGEX) || [];
+  for (const l of matches) links.add(l.replace(/\/$/, ""));
+  return [...links];
+}
+function discoverContactPages(html, baseUrl) {
+  let base;
+  try {
+    base = new URL(baseUrl);
+  } catch {
+    return [];
+  }
+  const pages = /* @__PURE__ */ new Set();
+  for (const path of CONTACT_PATHS_TO_TRY) {
+    pages.add(`${base.origin}${path}`);
+  }
+  const linkRegex = /<a\s[^>]*href="([^"#]*)"[^>]*>/gi;
+  let m;
+  while ((m = linkRegex.exec(html)) !== null) {
+    try {
+      const url = new URL(m[1], baseUrl);
+      if (url.hostname !== base.hostname) continue;
+      const path = url.pathname.toLowerCase();
+      if (CONTACT_PATH_KEYWORDS.some((kw) => path.includes(kw))) {
+        pages.add(url.origin + url.pathname);
+      }
+    } catch {
+    }
+  }
+  pages.delete(base.origin + base.pathname);
+  return [...pages].slice(0, 10);
+}
+function guessCommonEmails(domain) {
+  const d = domain.replace(/^www\./, "");
+  return [`info@${d}`, `contact@${d}`, `hello@${d}`, `administratie@${d}`, `verkoop@${d}`];
+}
+var BOT_CHALLENGE_INDICATORS = [
+  "sgcaptcha",
+  "challenge-platform",
+  "cf-browser-verification",
+  "Just a moment",
+  "Checking your browser",
+  "Enable JavaScript and cookies",
+  "Attention Required",
+  "DDoS protection by"
+];
+function isBotChallengePage(html) {
+  if (html.length > 2e3) return false;
+  const lower = html.toLowerCase();
+  return BOT_CHALLENGE_INDICATORS.some((ind) => lower.includes(ind.toLowerCase()));
+}
+async function fetchRawHtml(url, timeoutMs = 1e4) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, {
+      headers: {
+        "User-Agent": WEB_USER_AGENT,
+        "Accept": "text/html,application/xhtml+xml",
+        "Accept-Language": "nl,en;q=0.9"
+      },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    const ct = res.headers.get("content-type") || "";
+    if (!ct.includes("text/html") && !ct.includes("text/plain") && !ct.includes("xhtml")) return null;
+    if (!res.ok && res.status !== 403) return null;
+    const html = await res.text();
+    if (isBotChallengePage(html)) return null;
+    return html;
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function fetchWaybackHtml(url, timeoutMs = 15e3) {
+  const cleanUrl = url.replace(/^https?:\/\//, "");
+  const wbUrl = `https://web.archive.org/web/2024/${cleanUrl}`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(wbUrl, {
+      headers: { "User-Agent": WEB_USER_AGENT },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    if (!res.ok) return null;
+    const html = await res.text();
+    if (html.length < 500) return null;
+    return html;
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
 async function webFetch(url, extractLinks) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), 15e3);
@@ -824,7 +1083,7 @@ async function webFetch(url, extractLinks) {
     if (text.length > MAX_TEXT) {
       text = text.slice(0, MAX_TEXT) + "\n\n[... truncated at 15000 chars ...]";
     }
-    return { text, links };
+    return { text, rawHtml: html, links };
   } finally {
     clearTimeout(timeout);
   }
@@ -1308,15 +1567,14 @@ ${output}` }]
       const resultsCount = clamp(Number(args2.results_count) || 0, 0, 9999);
       if (!targetId) throw new Error("target_id is required");
       const { error } = await supabase2.from("lead_generation_target").update({
-        status: "completed",
         results_count: resultsCount,
-        completed_at: (/* @__PURE__ */ new Date()).toISOString()
+        assigned_workspace_id: null
       }).eq("id", targetId);
-      if (error) throw new Error(`Failed to complete target: ${error.message}`);
+      if (error) throw new Error(`Failed to update target: ${error.message}`);
       return {
         content: [{
           type: "text",
-          text: `Target ${targetId} marked completed. ${resultsCount} leads found.`
+          text: `Target ${targetId} updated: ${resultsCount} leads found. Workspace released.`
         }]
       };
     }
@@ -1362,10 +1620,169 @@ ${result.text}`;
 
 --- Links (${result.links.length} found, showing max 50) ---
 ${linkList}`;
+      }
+      const pageEmails = extractEmailsFromHtml(result.rawHtml);
+      const pagePhones = extractPhonesFromHtml(result.rawHtml);
+      const pageLinkedIn = extractLinkedInFromHtml(result.rawHtml);
+      if (pageEmails.length > 0 || pagePhones.length > 0 || pageLinkedIn.length > 0) {
+        text += "\n\n--- Auto-extracted Contact Info ---";
+        if (pageEmails.length > 0) text += `
+Emails: ${pageEmails.join(", ")}`;
+        if (pagePhones.length > 0) text += `
+Phones: ${pagePhones.join(", ")}`;
+        if (pageLinkedIn.length > 0) text += `
+LinkedIn: ${pageLinkedIn.join(", ")}`;
       }
       return { content: [{ type: "text", text }] };
     }
     // -----------------------------------------------------------------
+    // Web Find Contacts — SOTA multi-page crawler + email extraction
+    // -----------------------------------------------------------------
+    case "web-find-contacts": {
+      const inputUrl = sanitizeString(args2.url, 2e3);
+      if (!inputUrl) throw new Error("url is required");
+      const companyName = sanitizeString(args2.company_name, 500);
+      const includeSearch = args2.include_search !== false;
+      const fullUrl = inputUrl.startsWith("http") ? inputUrl : `https://${inputUrl}`;
+      let baseUrl;
+      try {
+        baseUrl = new URL(fullUrl);
+      } catch {
+        throw new Error(`Invalid URL: ${inputUrl}`);
+      }
+      const domain = baseUrl.hostname.replace(/^www\./, "");
+      const urlsToTry = [fullUrl];
+      if (!fullUrl.includes("www.")) urlsToTry.push(fullUrl.replace("://", "://www."));
+      if (fullUrl.startsWith("https")) {
+        urlsToTry.push(fullUrl.replace("https", "http"));
+        if (!fullUrl.includes("www.")) urlsToTry.push(fullUrl.replace("https://", "http://www."));
+      }
+      let html = null;
+      let usedWayback = false;
+      for (const tryUrl of urlsToTry) {
+        html = await fetchRawHtml(tryUrl, 12e3);
+        if (html) break;
+      }
+      if (!html) {
+        html = await fetchWaybackHtml(`https://${domain}`, 15e3);
+        if (html) usedWayback = true;
+      }
+      if (!html) throw new Error(`Could not fetch ${fullUrl} (site may be down or blocking)`);
+      const contactPages = usedWayback ? [] : discoverContactPages(html, fullUrl);
+      const pagePromises = contactPages.map(async (pageUrl) => {
+        const pageHtml = await fetchRawHtml(pageUrl, 8e3);
+        return { url: pageUrl, html: pageHtml };
+      });
+      const pageResults = await Promise.allSettled(pagePromises);
+      const successPages = [usedWayback ? `(wayback) ${domain}` : fullUrl];
+      const allHtmls = [html];
+      for (const result of pageResults) {
+        if (result.status === "fulfilled" && result.value.html) {
+          allHtmls.push(result.value.html);
+          successPages.push(result.value.url);
+        }
+      }
+      if (usedWayback) {
+        const waybackContactPaths = ["/contact", "/over-ons", "/about", "/team"];
+        const wbPromises = waybackContactPaths.map(async (path) => {
+          const wbHtml = await fetchWaybackHtml(`https://${domain}${path}`, 12e3);
+          return { path, html: wbHtml };
+        });
+        const wbResults = await Promise.allSettled(wbPromises);
+        for (const wr of wbResults) {
+          if (wr.status === "fulfilled" && wr.value.html) {
+            allHtmls.push(wr.value.html);
+            successPages.push(`(wayback) ${domain}${wr.value.path}`);
+          }
+        }
+      }
+      const allEmails = /* @__PURE__ */ new Set();
+      const allPhones = /* @__PURE__ */ new Set();
+      const allLinkedIn = /* @__PURE__ */ new Set();
+      for (const pageHtml of allHtmls) {
+        for (const e of extractEmailsFromHtml(pageHtml)) allEmails.add(e);
+        for (const p of extractPhonesFromHtml(pageHtml)) allPhones.add(p);
+        for (const l of extractLinkedInFromHtml(pageHtml)) allLinkedIn.add(l);
+      }
+      let searchNote = "";
+      if (includeSearch) {
+        try {
+          const queries = companyName ? [`"${companyName}" "${domain}" email`, `"@${domain}" email`] : [`"${domain}" email contact`, `"@${domain}" email`, `site:${domain} "@"`];
+          let totalSearchResults = 0;
+          for (const searchQuery of queries) {
+            const searchResults = await webSearch(searchQuery, 8);
+            totalSearchResults += searchResults.length;
+            for (const r of searchResults) {
+              const combined = `${r.title} ${r.snippet}`;
+              const decoded = decodeHtmlEntities(decodeUnicodeEscapes(combined));
+              const snippetEmails = decoded.match(EMAIL_REGEX) || [];
+              for (const e of snippetEmails) {
+                const lower = e.toLowerCase();
+                const domainBase = domain.split(".")[0] ?? domain;
+                if (lower.includes(domainBase) && !NOISE_EMAIL_PATTERNS.some((p) => p.test(lower))) {
+                  allEmails.add(lower);
+                }
+              }
+              const snippetPhones = decoded.match(PHONE_NL_REGEX) || [];
+              for (const p of snippetPhones) {
+                const clean = p.replace(/[\s\-().]/g, "");
+                if (clean.length >= 10 && clean.length <= 14) allPhones.add(p.trim());
+              }
+              const snippetLi = decoded.match(LINKEDIN_REGEX) || [];
+              for (const l of snippetLi) allLinkedIn.add(l.replace(/\/$/, ""));
+            }
+            if (allEmails.size > 0) break;
+          }
+          searchNote = ` + ${totalSearchResults} search results`;
+        } catch {
+          searchNote = " (search failed)";
+        }
+      }
+      const foundEmails = [...allEmails];
+      const generalPatterns = /^(info|contact|hello|admin|office|receptie|secretariaat|verkoop|administratie|support|service|boekingen|reserveringen)@/i;
+      const generalEmails = foundEmails.filter((e) => generalPatterns.test(e));
+      const personalEmails = foundEmails.filter((e) => !generalPatterns.test(e));
+      const guessed = guessCommonEmails(domain);
+      const newGuesses = guessed.filter((g) => !allEmails.has(g));
+      const lines = [
+        `=== CONTACT SCAN: ${domain} ===`,
+        `Pages scanned: ${successPages.length}${searchNote}`,
+        ""
+      ];
+      if (foundEmails.length > 0) {
+        lines.push(`EMAILS FOUND (${foundEmails.length}):`);
+        if (generalEmails.length > 0) {
+          lines.push("  General:");
+          for (const e of generalEmails) lines.push(`    * ${e}`);
+        }
+        if (personalEmails.length > 0) {
+          lines.push("  Personal/Department:");
+          for (const e of personalEmails) lines.push(`    * ${e}`);
+        }
+      } else {
+        lines.push("EMAILS FOUND: None on website");
+        lines.push("SUGGESTED EMAILS (common patterns, verify before use):");
+        for (const e of newGuesses) lines.push(`    ? ${e}`);
+      }
+      lines.push("");
+      if ([...allPhones].length > 0) {
+        lines.push(`PHONE NUMBERS (${allPhones.size}):`);
+        for (const p of allPhones) lines.push(`    ${p}`);
+      } else {
+        lines.push("PHONE NUMBERS: None found");
+      }
+      lines.push("");
+      if ([...allLinkedIn].length > 0) {
+        lines.push(`LINKEDIN (${allLinkedIn.size}):`);
+        for (const l of allLinkedIn) lines.push(`    ${l}`);
+      }
+      lines.push("", "PAGES CHECKED:");
+      for (const p of successPages) lines.push(`  [OK] ${p}`);
+      const failedPages = contactPages.filter((p) => !successPages.includes(p));
+      for (const p of failedPages) lines.push(`  [--] ${p}`);
+      return { content: [{ type: "text", text: lines.join("\n") }] };
+    }
+    // -----------------------------------------------------------------
     default:
       return { content: [{ type: "text", text: `Unknown agent tool: ${name}` }] };
   }