@mgsoftwarebv/mg-dashboard-mcp 2.6.2 → 2.6.3

package/dist/index.js

@@ -852,7 +852,19 @@ var NOISE_EMAIL_PATTERNS = [
   /smith@/i,
   /doe@/i,
   /demo@/i,
-  /sample@/i
+  /sample@/i,
+  /naam@/i,
+  /voorbeeld/i,
+  /your-?email/i,
+  /email@/i,
+  /@domein\./i,
+  /@bedrijf\./i,
+  /@domain\./i,
+  /@sentry/i,
+  /@wixpress/i,
+  /@lieferkassen/i,
+  /john@/i,
+  /jane@/i
 ];
 var CONTACT_PATH_KEYWORDS = [
   "contact",
@@ -974,6 +986,21 @@ function guessCommonEmails(domain) {
   const d = domain.replace(/^www\./, "");
   return [`info@${d}`, `contact@${d}`, `hello@${d}`, `administratie@${d}`, `verkoop@${d}`];
 }
+var BOT_CHALLENGE_INDICATORS = [
+  "sgcaptcha",
+  "challenge-platform",
+  "cf-browser-verification",
+  "Just a moment",
+  "Checking your browser",
+  "Enable JavaScript and cookies",
+  "Attention Required",
+  "DDoS protection by"
+];
+function isBotChallengePage(html) {
+  if (html.length > 2e3) return false;
+  const lower = html.toLowerCase();
+  return BOT_CHALLENGE_INDICATORS.some((ind) => lower.includes(ind.toLowerCase()));
+}
 async function fetchRawHtml(url, timeoutMs = 1e4) {
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort(), timeoutMs);
@@ -987,10 +1014,33 @@ async function fetchRawHtml(url, timeoutMs = 1e4) {
       redirect: "follow",
       signal: controller.signal
     });
-    if (!res.ok) return null;
     const ct = res.headers.get("content-type") || "";
     if (!ct.includes("text/html") && !ct.includes("text/plain") && !ct.includes("xhtml")) return null;
-    return await res.text();
+    if (!res.ok && res.status !== 403) return null;
+    const html = await res.text();
+    if (isBotChallengePage(html)) return null;
+    return html;
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function fetchWaybackHtml(url, timeoutMs = 15e3) {
+  const cleanUrl = url.replace(/^https?:\/\//, "");
+  const wbUrl = `https://web.archive.org/web/2024/${cleanUrl}`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(wbUrl, {
+      headers: { "User-Agent": WEB_USER_AGENT },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    if (!res.ok) return null;
+    const html = await res.text();
+    if (html.length < 500) return null;
+    return html;
   } catch {
     return null;
   } finally {
@@ -1608,18 +1658,23 @@ LinkedIn: ${pageLinkedIn.join(", ")}`;
         if (!fullUrl.includes("www.")) urlsToTry.push(fullUrl.replace("https://", "http://www."));
       }
       let html = null;
+      let usedWayback = false;
       for (const tryUrl of urlsToTry) {
         html = await fetchRawHtml(tryUrl, 12e3);
         if (html) break;
       }
+      if (!html) {
+        html = await fetchWaybackHtml(`https://${domain}`, 15e3);
+        if (html) usedWayback = true;
+      }
       if (!html) throw new Error(`Could not fetch ${fullUrl} (site may be down or blocking)`);
-      const contactPages = discoverContactPages(html, fullUrl);
+      const contactPages = usedWayback ? [] : discoverContactPages(html, fullUrl);
       const pagePromises = contactPages.map(async (pageUrl) => {
         const pageHtml = await fetchRawHtml(pageUrl, 8e3);
         return { url: pageUrl, html: pageHtml };
       });
       const pageResults = await Promise.allSettled(pagePromises);
-      const successPages = [fullUrl];
+      const successPages = [usedWayback ? `(wayback) ${domain}` : fullUrl];
       const allHtmls = [html];
       for (const result of pageResults) {
         if (result.status === "fulfilled" && result.value.html) {
@@ -1627,6 +1682,20 @@ LinkedIn: ${pageLinkedIn.join(", ")}`;
           successPages.push(result.value.url);
         }
       }
+      if (usedWayback) {
+        const waybackContactPaths = ["/contact", "/over-ons", "/about", "/team"];
+        const wbPromises = waybackContactPaths.map(async (path) => {
+          const wbHtml = await fetchWaybackHtml(`https://${domain}${path}`, 12e3);
+          return { path, html: wbHtml };
+        });
+        const wbResults = await Promise.allSettled(wbPromises);
+        for (const wr of wbResults) {
+          if (wr.status === "fulfilled" && wr.value.html) {
+            allHtmls.push(wr.value.html);
+            successPages.push(`(wayback) ${domain}${wr.value.path}`);
+          }
+        }
+      }
       const allEmails = /* @__PURE__ */ new Set();
       const allPhones = /* @__PURE__ */ new Set();
       const allLinkedIn = /* @__PURE__ */ new Set();