mcp-scraper 0.65.1 → 0.66.1

package/dist/chunk-VXLU74YZ.js

@@ -0,0 +1,293 @@
+import {
+  isPrivateIpAddress,
+  loadHtmlDocument
+} from "./chunk-GGZEC22A.js";
+
+// src/lib/contact-evidence.ts
+var MAX_EMAILS = 20;
+var MAX_SOCIAL_PROFILES = 20;
+var MAX_CONTACT_FORMS = 10;
+var MAX_CANDIDATE_PAGE_URLS = 20;
+var MAX_CONTEXT_CHARS = 280;
+var EMAIL_PATTERN = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
+var EMAIL_NOISE = [
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".svg",
+  ".webp",
+  ".ico",
+  "@2x",
+  "sentry",
+  "wix.com",
+  "wixpress",
+  "example.",
+  "godaddy",
+  "squarespace",
+  "schema.org",
+  "u003e",
+  "core-js"
+];
+var DEVELOPER_CONTEXT = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
+var CONTACT_PATH = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
+var SOCIAL_HOSTS = {
+  "facebook.com": "facebook",
+  "fb.com": "facebook",
+  "instagram.com": "instagram",
+  "linkedin.com": "linkedin",
+  "twitter.com": "twitter",
+  "x.com": "x",
+  "youtube.com": "youtube",
+  "youtu.be": "youtube",
+  "tiktok.com": "tiktok",
+  "pinterest.com": "pinterest"
+};
+var SOCIAL_NOISE = /(?:\/shar(?:e|er)|\/intent\/|\/dialog\/|\/plugins\/|\/share\?|\/developers?\/|platform\.)/i;
+function compact(value) {
+  return (value ?? "").replace(/\s+/g, " ").trim();
+}
+function safeHttpUrl(value, base) {
+  try {
+    const parsed = new URL(value, base);
+    if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+function normalizeEmail(value) {
+  let decoded = value;
+  try {
+    decoded = decodeURIComponent(value);
+  } catch {
+  }
+  const email = decoded.split("?")[0].trim().toLowerCase().replace(/[),.;:]+$/, "");
+  if (email.length > 254 || email.split("@").length !== 2) return null;
+  if (!/^[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9.-]+\.[a-z]{2,}$/i.test(email)) return null;
+  if (EMAIL_NOISE.some((part) => email.includes(part))) return null;
+  return email;
+}
+function nearbyContext(node) {
+  const parent = node.closest("p,li,address,section,article,div").first();
+  return compact(parent.length ? parent.text() : node.text()).slice(0, MAX_CONTEXT_CHARS);
+}
+function addEmail(target, raw, sourceUrl, context) {
+  const value = normalizeEmail(raw);
+  if (!value || DEVELOPER_CONTEXT.test(context)) return;
+  if (!target.has(value)) {
+    target.set(value, {
+      value,
+      sourceUrl,
+      context: compact(context).slice(0, MAX_CONTEXT_CHARS),
+      sourceType: "first_party_page"
+    });
+  }
+}
+function normalizedSocialUrl(parsed) {
+  parsed.hash = "";
+  for (const key of [...parsed.searchParams.keys()]) {
+    if (/^(?:utm_|fbclid|gclid)/i.test(key)) parsed.searchParams.delete(key);
+  }
+  return parsed.href;
+}
+function socialNetwork(hostname) {
+  const host = hostname.toLowerCase().replace(/^www\./, "");
+  const match = Object.entries(SOCIAL_HOSTS).find(([domain]) => host === domain || host.endsWith(`.${domain}`));
+  return match?.[1] ?? null;
+}
+function extractContactEvidence(html, pageUrl) {
+  const source = safeHttpUrl(pageUrl);
+  if (!source) throw new Error("pageUrl must be a public HTTP(S) URL");
+  const { $ } = loadHtmlDocument(html);
+  const emails = /* @__PURE__ */ new Map();
+  const socialProfiles = /* @__PURE__ */ new Map();
+  const contactForms = /* @__PURE__ */ new Map();
+  const candidatePageUrls = /* @__PURE__ */ new Map();
+  $('a[href^="mailto:" i]').each((_index, element) => {
+    const node = $(element);
+    addEmail(emails, (node.attr("href") ?? "").slice(7), source.href, nearbyContext(node));
+  });
+  $("body").find("*").addBack().contents().filter((_index, node) => node.type === "text").each((_index, node) => {
+    const parent = $(node.parent ?? node);
+    const text = String(node.data ?? "");
+    for (const match of text.match(EMAIL_PATTERN) ?? []) addEmail(emails, match, source.href, nearbyContext(parent));
+  });
+  $("a[href]").each((_index, element) => {
+    const node = $(element);
+    const parsed = safeHttpUrl(node.attr("href") ?? "", source.href);
+    if (!parsed) return;
+    const context = compact(`${node.text()} ${nearbyContext(node)}`).slice(0, MAX_CONTEXT_CHARS);
+    const network = socialNetwork(parsed.hostname);
+    if (network && !SOCIAL_NOISE.test(`${parsed.pathname}${parsed.search}`)) {
+      const url2 = normalizedSocialUrl(parsed);
+      if (!socialProfiles.has(url2)) socialProfiles.set(url2, { url: url2, network, sourceUrl: source.href, context });
+      return;
+    }
+    if (parsed.origin !== source.origin || !CONTACT_PATH.test(parsed.pathname)) return;
+    parsed.hash = "";
+    const url = parsed.href;
+    if (!candidatePageUrls.has(url)) candidatePageUrls.set(url, { url, sourceUrl: source.href, context });
+  });
+  $("form").each((_index, element) => {
+    const node = $(element);
+    const action = safeHttpUrl(node.attr("action") || source.href, source.href);
+    if (!action || action.origin !== source.origin) return;
+    action.hash = "";
+    const url = action.href;
+    if (!contactForms.has(url)) contactForms.set(url, { url, sourceUrl: source.href, context: nearbyContext(node) });
+  });
+  return {
+    emails: [...emails.values()].slice(0, MAX_EMAILS),
+    socialProfiles: [...socialProfiles.values()].slice(0, MAX_SOCIAL_PROFILES),
+    contactForms: [...contactForms.values()].slice(0, MAX_CONTACT_FORMS),
+    candidatePageUrls: [...candidatePageUrls.values()].slice(0, MAX_CANDIDATE_PAGE_URLS)
+  };
+}
+
+// src/leads/contact-enrichment.ts
+var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
+  "admin",
+  "contact",
+  "customerservice",
+  "hello",
+  "help",
+  "info",
+  "inquiries",
+  "mail",
+  "office",
+  "reception",
+  "sales",
+  "service",
+  "support",
+  "team"
+]);
+var CONTACT_PATH2 = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
+var TEAM_CONTEXT = /\b(?:our team|meet the team|staff|agents?|brokers?|realtors?|people)\b/i;
+var OFFICE_CONTEXT = /\b(?:brokerage|company|corporate|front desk|main office|office|organization|reception)\b/i;
+function normalizeWords(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter(Boolean);
+}
+function safeHttpUrl2(value, base) {
+  try {
+    const parsed = new URL(value, base);
+    if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+function classifyEmailScope(evidence, entity) {
+  const local = evidence.value.split("@")[0].toLowerCase().replace(/[^a-z0-9]+/g, " ");
+  const nameWords = normalizeWords(entity.name).filter((word) => word.length > 1);
+  const pagePath = safeHttpUrl2(evidence.sourceUrl)?.pathname.toLowerCase() ?? "";
+  const combinedContext = `${evidence.context} ${pagePath}`.toLowerCase();
+  const generic = GENERIC_EMAIL_PREFIXES.has(local.replace(/\s+/g, ""));
+  const nameHits = nameWords.filter((word) => new RegExp(`(?:^|[^a-z0-9])${word}(?:$|[^a-z0-9])`, "i").test(`${local} ${combinedContext}`)).length;
+  const hasExactPersonContext = entity.entityType === "person" && nameWords.length >= 2 && nameHits >= Math.min(2, nameWords.length) && (local.includes(nameWords[0]) || pagePath.includes(nameWords.join("-")) || pagePath.includes(nameWords.join("/")));
+  if (!generic && hasExactPersonContext) return { scope: "person", reasons: ["named_person_page_context"] };
+  if (generic || OFFICE_CONTEXT.test(combinedContext)) return { scope: "office", reasons: generic ? ["generic_mailbox_prefix"] : ["organization_page_context"] };
+  if (TEAM_CONTEXT.test(combinedContext)) return { scope: "team", reasons: ["team_page_context"] };
+  if (entity.entityType === "business") {
+    const organizationWords = normalizeWords(entity.organization || entity.name).filter((word) => word.length > 2);
+    if (organizationWords.some((word) => combinedContext.includes(word))) return { scope: "team", reasons: ["business_context"] };
+  }
+  return { scope: "unknown", reasons: ["public_occurrence_without_identity_evidence"] };
+}
+function selectPrimaryEmail(evidence, entity) {
+  const scoped = evidence.map((item) => ({ ...item, ...classifyEmailScope(item, entity) }));
+  const allowed = entity.entityType === "person" ? ["person", "team"] : ["person", "team", "office"];
+  const sourcePriority = ["provided", "first_party_page", "structured_data", "serp_snippet"];
+  const priority = ["person", "team", "office"];
+  return scoped.filter((item) => allowed.includes(item.scope)).sort((a, b) => {
+    const sourceDelta = sourcePriority.indexOf(a.sourceType) - sourcePriority.indexOf(b.sourceType);
+    return sourceDelta || priority.indexOf(a.scope) - priority.indexOf(b.scope);
+  })[0] ?? null;
+}
+function contactAttemptUrls(seedUrl, discoveredLinks, maxPages) {
+  if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 4) throw new Error("maxPages must be an integer from 1 to 4");
+  const seed = safeHttpUrl2(seedUrl);
+  if (!seed) return [];
+  seed.hash = "";
+  const urls = /* @__PURE__ */ new Set([seed.href]);
+  const discovered = discoveredLinks.map((item) => typeof item === "string" ? item : item.url).map((item) => safeHttpUrl2(item, seed.href)).filter((item) => Boolean(item && item.origin === seed.origin && CONTACT_PATH2.test(item.pathname)));
+  for (const item of discovered) {
+    item.hash = "";
+    urls.add(item.href);
+  }
+  for (const pathname of ["/contact", "/contact-us"]) urls.add(new URL(pathname, seed.origin).href);
+  return [...urls].slice(0, maxPages);
+}
+
+// src/leads/serp-email-evidence.ts
+var EMAIL_CANDIDATE = /[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
+var DEVELOPER_CONTEXT2 = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
+var MAX_CONTEXT_CHARS2 = 280;
+var MAX_EMAILS2 = 10;
+function normalizeHostname(hostname) {
+  return hostname.toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
+}
+function isLocalHostname(hostname) {
+  const host = normalizeHostname(hostname);
+  return host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || !host.includes(".") && !host.includes(":");
+}
+function parsePublicLiteralUrl(websiteUrl) {
+  try {
+    const parsed = new URL(websiteUrl.trim());
+    if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
+    if (isLocalHostname(parsed.hostname) || isPrivateIpAddress(parsed.hostname)) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+function compactContext(title, snippet) {
+  return `${title} ${snippet ?? ""}`.replace(/\s+/g, " ").trim().slice(0, MAX_CONTEXT_CHARS2);
+}
+function buildSerpEmailQuery(websiteUrl) {
+  const parsed = parsePublicLiteralUrl(websiteUrl);
+  if (!parsed) return null;
+  parsed.hostname = normalizeHostname(parsed.hostname);
+  return `email for ${parsed.protocol}//${parsed.host}/`;
+}
+function extractSerpEmailEvidence({
+  websiteUrl,
+  results
+}) {
+  const parsedWebsite = parsePublicLiteralUrl(websiteUrl);
+  if (!parsedWebsite) return [];
+  const websiteDomain = normalizeHostname(parsedWebsite.hostname);
+  const accepted = /* @__PURE__ */ new Map();
+  const orderedResults = results.map((result, index) => ({ result, index })).sort((left, right) => {
+    const leftPosition = Number.isFinite(left.result.position) ? left.result.position : Number.MAX_SAFE_INTEGER;
+    const rightPosition = Number.isFinite(right.result.position) ? right.result.position : Number.MAX_SAFE_INTEGER;
+    return leftPosition - rightPosition || left.index - right.index;
+  });
+  for (const { result } of orderedResults) {
+    const searchableText = `${result.title} ${result.snippet ?? ""}`;
+    const context = compactContext(result.title, result.snippet);
+    for (const rawCandidate of searchableText.match(EMAIL_CANDIDATE) ?? []) {
+      const value = normalizeEmail(rawCandidate);
+      if (!value || DEVELOPER_CONTEXT2.test(searchableText) || normalizeHostname(value.slice(value.lastIndexOf("@") + 1)) !== websiteDomain) continue;
+      if (accepted.has(value)) continue;
+      accepted.set(value, {
+        value,
+        sourceUrl: result.url,
+        sourceTitle: result.title,
+        sourcePosition: result.position,
+        context,
+        sourceType: "serp_snippet"
+      });
+    }
+  }
+  return [...accepted.values()].slice(0, MAX_EMAILS2);
+}
+
+export {
+  extractContactEvidence,
+  classifyEmailScope,
+  selectPrimaryEmail,
+  contactAttemptUrls,
+  buildSerpEmailQuery,
+  extractSerpEmailEvidence
+};

package/dist/{chunk-DCWXVAQT.js → chunk-WID3MFRS.js}

@@ -1,3 +1,8 @@
+// release-message.json
+var release_message_default = {
+  message: "Long PAA research now continues as a durable MCP Task with preserved progress, while transient browser interruptions return specific retry guidance instead of a generic outage."
+};
+
 // src/install-terminal.ts
 var CODES = {
   reset: "\x1B[0m",
@@ -48,7 +53,7 @@ function renderInstallTerminal(options) {
     colorize(ascii, "amber", color),
     `${colorize("MCP Scraper Agent", "cyan", color)}  .  v${options.version}  .  mcpscraper.dev`,
     "1/1 install surfaces ready",
-    colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
+    colorize(`Newest in v${options.version}: ${release_message_default.message}`, "lime", color),
     "",
     `${colorize("Tools", "cyan", color)}  ${colorize("(259 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
@@ -106,4 +111,3 @@ function renderInstallTerminal(options) {
 export {
   renderInstallTerminal
 };
-//# sourceMappingURL=chunk-DCWXVAQT.js.map

package/dist/{db-BRI465XQ.js → db-PO7N7ZP3.js}

@@ -126,7 +126,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-GTGSD4YS.js";
 export {
   CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH,
@@ -256,4 +256,3 @@ export {
   upsertPageSnapshot,
   verifyPassword
 };
-//# sourceMappingURL=db-BRI465XQ.js.map

package/dist/{extract-bundle-UWKJT4MU.js → extract-bundle-SIBFUA6I.js}

@@ -4,27 +4,26 @@ import {
   downloadAsset,
   embedCommonsTexts,
   publicizeExtractionFailure
-} from "./chunk-AMLFKPLL.js";
-import "./chunk-UN4LZVCB.js";
+} from "./chunk-75VW7CWE.js";
+import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
   renderImageSection,
   renderIssueReport,
   renderLinkReport
-} from "./chunk-342OUCSG.js";
-import "./chunk-MQGBCWZD.js";
-import "./chunk-UVFB24RZ.js";
+} from "./chunk-PGJQDMC2.js";
+import "./chunk-GGZEC22A.js";
 import {
   createSiteExtractBundleArtifactStream,
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-5PIO7QBG.js";
-import "./chunk-EKJ65E3Z.js";
-import "./chunk-2XUXVKT4.js";
+} from "./chunk-3PIXKLGZ.js";
+import "./chunk-DNM65UCK.js";
+import "./chunk-AYJHI2FK.js";
 import {
   getDb
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-GTGSD4YS.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
@@ -1002,4 +1001,3 @@ export {
   SITE_EXTRACT_PAGE_CHUNK,
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-UWKJT4MU.js.map

package/dist/index.cjs

@@ -4073,6 +4073,9 @@ function classifyAttemptError(err) {
   if (err instanceof RequestAbortedError) return "request_aborted";
   if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return "timeout";
   const message = err instanceof Error ? err.message : String(err);
+  if (/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message)) {
+    return "browser_session_interrupted";
+  }
   if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
   if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
   return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message) ? "timeout" : "error";
@@ -4120,6 +4123,9 @@ function looksLikeProxyUnavailable(message) {
 function retryableLocationProxyError(outcome) {
   return outcome === "captcha" || outcome === "proxy_tunnel_failed" || outcome === "proxy_unavailable";
 }
+function retryableFreshBrowserError(outcome) {
+  return outcome === "browser_session_interrupted";
+}
 function locationMismatchMessage(result) {
   const evidence = result.diagnostics.debug?.locationEvidence;
   const expected = evidence?.expected?.canonicalLocation ?? result.location ?? "requested location";
@@ -4303,7 +4309,7 @@ async function harvest(rawOptions) {
           proxyFailures++;
           if (proxyFailures >= 2) effectiveProxyMode = "none";
         }
-        const willRetry2 = i < maxAttempts - 1 && (outcome2 === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome2));
+        const willRetry2 = i < maxAttempts - 1 && (outcome2 === "captcha" || retryableFreshBrowserError(outcome2) || proxyMode === "location" && retryableLocationProxyError(outcome2));
         if (outcome2 === "captcha") {
           console.warn(JSON.stringify({
             event: "harvest_attempt_captcha",
@@ -4421,7 +4427,7 @@ async function harvest(rawOptions) {
         proxyFailures++;
         if (proxyFailures >= 2) effectiveProxyMode = "none";
       }
-      const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
+      const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || retryableFreshBrowserError(outcome) || proxyMode === "location" && retryableLocationProxyError(outcome));
       if (outcome === "captcha") {
         console.warn(JSON.stringify({
           event: "harvest_attempt_captcha",
@@ -4803,4 +4809,3 @@ var VideoGenerator = class {
   buildClipPrompts,
   harvest
 });
-//# sourceMappingURL=index.cjs.map

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-ORKXE7VX.js";
+} from "./chunk-4P3PHFPQ.js";
 import {
   recordVendorUsage
-} from "./chunk-4LINSJNI.js";
-import "./chunk-FG6LJGJ7.js";
-import "./chunk-4VG3NUUY.js";
+} from "./chunk-TNLGHNLX.js";
+import "./chunk-77LQV73B.js";
+import "./chunk-GTGSD4YS.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";
@@ -333,4 +333,3 @@ export {
   buildClipPrompts,
   harvest
 };
-//# sourceMappingURL=index.js.map

package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-OTAUQONJ.js}

@@ -30,8 +30,8 @@ import {
   saveLeadListEnrichmentRowResult,
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
-} from "./chunk-N54JS5WP.js";
-import "./chunk-4VG3NUUY.js";
+} from "./chunk-TNJIPV74.js";
+import "./chunk-GTGSD4YS.js";
 export {
   LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
   LEAD_LIST_RUNNING_STALE_MINUTES,
@@ -65,4 +65,3 @@ export {
   setInitializingLeadListEnrichmentConcurrencyLock,
   stableCanonicalJson
 };
-//# sourceMappingURL=lead-list-enrichment-repository-ODWGRVCQ.js.map

package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-RM2KJ3H5.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-DKIG2TEE.js";
-import "./chunk-G3P3ZDB4.js";
-import "./chunk-4VG3NUUY.js";
+} from "./chunk-AQKJOSIB.js";
+import "./chunk-RJ7JVYKU.js";
+import "./chunk-GTGSD4YS.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,
@@ -32,4 +32,3 @@ export {
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
 };
-//# sourceMappingURL=location-data-repository-67MMEVZQ.js.map