@agent-pattern-labs/leads-rig 0.1.5 → 0.1.7

package/lib/leadharness-crawler.mjs

@@ -1,14 +1,27 @@
 import { createHash } from 'crypto';
 import { lookup, resolveMx } from 'dns/promises';
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
+import { dirname } from 'path';
 import { setTimeout as sleep } from 'timers/promises';
-import { cleanDomains, dedupeLeadRecords, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain } from './leadharness-leads.mjs';
+import { cleanDomains, dedupeLeadRecords, isGoodLead, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain, resolveProjectPath } from './leadharness-leads.mjs';
 
 const DEFAULT_MAX_PAGES = 10;
 const HARD_MAX_PAGES = 25;
+const DEFAULT_DOMAIN_CONCURRENCY = 2;
+const HARD_DOMAIN_CONCURRENCY = 10;
+const DEFAULT_PAGE_CONCURRENCY = 1;
+const HARD_PAGE_CONCURRENCY = 2;
+const DEFAULT_DNS_CONCURRENCY = 8;
+const HARD_DNS_CONCURRENCY = 25;
 const DEFAULT_TIMEOUT_MS = 14_000;
 const DEFAULT_DELAY_MS = 200;
 const DEFAULT_USER_AGENT = 'PublicLeadsBot/0.1 (+https://example.com/public-leads)';
 const MAX_BODY_CHARS = 2 * 1024 * 1024;
+const DEFAULT_CACHE_PATH = '.leadharness-cache/crawler-cache.json';
+const DEFAULT_ROBOTS_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_DNS_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_PAGE_CACHE_TTL_MS = 60 * 60 * 1000;
+const MAX_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const EMAIL_RE = /\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b/gi;
 const NAME_RE = /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2})\b/g;
 const PHONE_RE = /(?:\+?1[\s.-]?)?(?:\([2-9]\d{2}\)|[2-9]\d{2})[\s.-]?[2-9]\d{2}[\s.-]?\d{4}\b/g;
@@ -35,27 +48,43 @@ export async function crawlDomains(inputs, options = {}) {
   const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, HARD_MAX_PAGES);
   const minConfidence = boundedInteger(options.minConfidence, 30, 0, 100);
   const includeBlocked = Boolean(options.includeBlocked);
+  const concurrency = crawlerConcurrency(options);
+  const context = createCrawlContext(options);
   const results = [];
   const errors = [];
   const leadsByKey = new Map();
 
-  for (const target of targets) {
-    try {
-      const result = await crawlDomainTarget(target, {
-        ...options,
-        maxPages,
-        minConfidence,
-        includeBlocked,
-      });
-      results.push(result);
-      for (const lead of result.leads) {
+  try {
+    const settled = await mapLimit(targets, concurrency, async (target) => {
+      try {
+        return {
+          target,
+          result: await crawlDomainTarget(target, {
+            ...options,
+            maxPages,
+            minConfidence,
+            includeBlocked,
+          }, context),
+        };
+      } catch (error) {
+        return { target, error };
+      }
+    });
+
+    for (const item of settled) {
+      if (item.error) {
+        errors.push(`${item.target.domain}: ${item.error instanceof Error ? item.error.message : String(item.error)}`);
+        continue;
+      }
+      results.push(item.result);
+      for (const lead of item.result.leads) {
         const key = leadKey(lead);
         const current = leadsByKey.get(key);
         leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
       }
-    } catch (error) {
-      errors.push(`${target.domain}: ${error instanceof Error ? error.message : String(error)}`);
     }
+  } finally {
+    saveCrawlerCache(context.cache);
   }
 
   const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
@@ -77,7 +106,12 @@ export async function crawlDomains(inputs, options = {}) {
 export async function crawlDomain(input, options = {}) {
   const [target] = normalizeCrawlTargets([input]);
   if (!target) throw new Error('domain is required');
-  return crawlDomainTarget(target, options);
+  const context = createCrawlContext(options);
+  try {
+    return await crawlDomainTarget(target, options, context);
+  } finally {
+    saveCrawlerCache(context.cache);
+  }
 }
 
 export function normalizeCrawlTargets(inputs) {
@@ -124,73 +158,110 @@ function normalizeCrawlTarget(input) {
   };
 }
 
-async function crawlDomainTarget(target, options) {
+function createCrawlContext(options) {
+  return {
+    cache: loadCrawlerCache(options),
+    dnsInFlight: new Map(),
+    originPaces: new Map(),
+    robotsInFlight: new Map(),
+    dnsLimiter: createLimiter(boundedInteger(options.dnsConcurrency, DEFAULT_DNS_CONCURRENCY, 1, HARD_DNS_CONCURRENCY)),
+  };
+}
+
+async function crawlDomainTarget(target, options, context) {
   const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, HARD_MAX_PAGES);
   const minConfidence = boundedInteger(options.minConfidence, 30, 0, 100);
+  const pageConcurrency = boundedInteger(options.pageConcurrency, DEFAULT_PAGE_CONCURRENCY, 1, HARD_PAGE_CONCURRENCY);
   const queue = [...target.starts];
   const queued = new Set(queue);
   const seen = new Set();
+  const inFlight = new Set();
   const leadsByKey = new Map();
   const warnings = [];
   const pages = [];
-  const robotsCache = new Map();
+  const stopAfterGoodLeads = boundedInteger(options.stopAfterGoodLeads, 0, 0, 1000);
+  const stopAfterContactPath = optionFlag(options.stopAfterContactPath);
+  let contactPathFound = false;
   let websiteUrl = '';
 
-  while (queue.length > 0 && pages.length < maxPages) {
-    const pageUrl = queue.shift();
-    if (!pageUrl || seen.has(pageUrl)) continue;
-    seen.add(pageUrl);
-
-    const allowed = await allowedByRobots(pageUrl, robotsCache, options, warnings);
-    if (!allowed) {
-      pages.push({ url: pageUrl, title: '', statusCode: 0, emailsFound: 0, error: 'blocked by robots.txt' });
-      continue;
-    }
-
-    const page = await fetchPage(pageUrl, options).catch((error) => ({
-      error: error instanceof Error ? error.message : String(error),
-    }));
-    if (page.error) {
-      pages.push({ url: pageUrl, title: '', statusCode: 0, emailsFound: 0, error: page.error });
-      continue;
+  const shouldStop = () => {
+    if (stopAfterContactPath && contactPathFound) return true;
+    if (stopAfterGoodLeads > 0 && countGoodLeads(leadsByKey) >= stopAfterGoodLeads) return true;
+    return false;
+  };
+  const schedulePages = () => {
+    while (!shouldStop() && inFlight.size < pageConcurrency && queue.length > 0 && pages.length + inFlight.size < maxPages) {
+      const pageUrl = queue.shift();
+      if (!pageUrl || seen.has(pageUrl)) continue;
+      seen.add(pageUrl);
+      let task;
+      task = crawlPage(pageUrl, context, options, warnings)
+        .then((result) => ({ task, result }))
+        .catch((error) => ({
+          task,
+          result: {
+            pageUrl,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        }));
+      inFlight.add(task);
     }
+  };
 
-    if (!websiteUrl) websiteUrl = originOf(page.url);
-    pages.push({
-      url: page.url,
-      title: page.title,
-      statusCode: page.statusCode,
-      emailsFound: page.emails.length,
-      phonesFound: page.phones.length,
-      socialUrlsFound: page.socialUrls.length,
-      contactUrlsFound: page.contactUrls.length,
-    });
+  schedulePages();
+  while (inFlight.size > 0) {
+    const { task, result } = await Promise.race(inFlight);
+    inFlight.delete(task);
 
-    for (const email of page.emails) {
-      const lead = await buildEmailLead(email, target.domain, websiteUrl, page);
-      if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
-      const key = leadKey(lead);
-      const current = leadsByKey.get(key);
-      leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
-    }
+    if (result.blocked) {
+      pages.push({ url: result.pageUrl, title: '', statusCode: 0, emailsFound: 0, error: 'blocked by robots.txt' });
+    } else if (result.error) {
+      pages.push({ url: result.pageUrl, title: '', statusCode: 0, emailsFound: 0, error: result.error });
+    } else {
+      const page = result.page;
+      if (!websiteUrl) websiteUrl = originOf(page.url);
+      pages.push({
+        url: page.url,
+        title: page.title,
+        statusCode: page.statusCode,
+        emailsFound: page.emails.length,
+        phonesFound: page.phones.length,
+        socialUrlsFound: page.socialUrls.length,
+        contactUrlsFound: page.contactUrls.length,
+      });
 
-    if (page.hasForm && isContactLikeURL(page.url)) {
-      const lead = buildContactPath(target.domain, websiteUrl, page);
-      if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
+      const emailLeads = await Promise.all(
+        page.emails.map((email) => buildEmailLead(email, target.domain, websiteUrl, page, context)),
+      );
+      for (const lead of emailLeads) {
+        if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
         const key = leadKey(lead);
         const current = leadsByKey.get(key);
         leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
       }
-    }
 
-    for (const link of page.links) {
-      if (queued.has(link) || seen.has(link)) continue;
-      if (!sameDomain(target.domain, link) || !isHighSignalPage(link)) continue;
-      queued.add(link);
-      queue.push(link);
+      if (page.hasForm && isContactLikeURL(page.url)) {
+        const lead = buildContactPath(target.domain, websiteUrl, page);
+        if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
+          const key = leadKey(lead);
+          const current = leadsByKey.get(key);
+          leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
+          contactPathFound = true;
+        }
+      }
+
+      if (!shouldStop()) {
+        for (const link of page.links) {
+          if (queued.has(link) || seen.has(link)) continue;
+          if (!sameDomain(target.domain, link) || !isHighSignalPage(link)) continue;
+          queued.add(link);
+          queue.push(link);
+        }
+        queue.sort((a, b) => pagePriority(b) - pagePriority(a));
+      }
     }
-    queue.sort((a, b) => pagePriority(b) - pagePriority(a));
-    await sleep(boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000));
+
+    schedulePages();
   }
 
   const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
@@ -208,62 +279,88 @@ async function crawlDomainTarget(target, options) {
   };
 }
 
-async function fetchPage(pageUrl, options) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), boundedInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1_000, 60_000));
-  try {
-    const response = await fetch(pageUrl, {
-      headers: {
-        Accept: 'text/html,application/xhtml+xml',
-        'User-Agent': options.userAgent || DEFAULT_USER_AGENT,
-      },
-      redirect: 'follow',
-      signal: controller.signal,
-    });
-    if (response.status >= 400) throw new Error(`HTTP ${response.status}`);
-    const contentType = String(response.headers.get('content-type') || '').toLowerCase();
-    if (contentType && !contentType.includes('html')) {
-      throw new Error(`skipped non-HTML content type ${contentType}`);
-    }
-    const contentLength = Number(response.headers.get('content-length') || 0);
-    if (contentLength > MAX_BODY_CHARS) {
-      throw new Error(`skipped large response (${contentLength} bytes)`);
+async function crawlPage(pageUrl, context, options, warnings) {
+  const allowed = await allowedByRobots(pageUrl, context, options, warnings);
+  if (!allowed) return { pageUrl, blocked: true };
+  const page = await fetchPage(pageUrl, options, context);
+  return { pageUrl, page };
+}
+
+async function fetchPage(pageUrl, options, context) {
+  const cached = getCachedPage(context.cache, pageUrl);
+  if (cached) return cached;
+
+  return withOriginPace(context, pageUrl, options, async () => {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), boundedInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1_000, 60_000));
+    try {
+      const response = await fetch(pageUrl, {
+        headers: {
+          Accept: 'text/html,application/xhtml+xml',
+          'User-Agent': options.userAgent || DEFAULT_USER_AGENT,
+        },
+        redirect: 'follow',
+        signal: controller.signal,
+      });
+      if (response.status >= 400) throw new Error(`HTTP ${response.status}`);
+      const contentType = String(response.headers.get('content-type') || '').toLowerCase();
+      if (contentType && !contentType.includes('html')) {
+        throw new Error(`skipped non-HTML content type ${contentType}`);
+      }
+      const contentLength = Number(response.headers.get('content-length') || 0);
+      if (contentLength > MAX_BODY_CHARS) {
+        throw new Error(`skipped large response (${contentLength} bytes)`);
+      }
+
+      const html = (await response.text()).slice(0, MAX_BODY_CHARS);
+      const finalUrl = response.url || pageUrl;
+      const page = parsePage(finalUrl, response.status, html);
+      setCachedPage(context.cache, pageUrl, page);
+      return page;
+    } finally {
+      clearTimeout(timeout);
     }
+  });
+}
 
-    const html = (await response.text()).slice(0, MAX_BODY_CHARS);
-    const decoded = decodeEntities(html);
-    const text = compactText(stripHtml(decoded));
-    const finalUrl = response.url || pageUrl;
-    const phones = extractPhones(decoded, text);
-    const socialUrls = extractSocialUrls(finalUrl, decoded);
-    const contactUrls = extractContactUrls(finalUrl, decoded);
-    return {
-      url: finalUrl,
-      statusCode: response.status,
-      title: compactText(extractTitle(decoded)),
-      text,
-      links: extractLinks(finalUrl, decoded),
-      emails: extractEmails(decoded),
-      phones,
-      socialUrls,
-      contactUrls,
-      hasForm: /<form\b/i.test(decoded),
-    };
-  } finally {
-    clearTimeout(timeout);
-  }
+function parsePage(finalUrl, statusCode, html) {
+  const decoded = decodeEntities(html);
+  const text = compactText(stripHtml(decoded));
+  const refs = extractPageRefs(finalUrl, decoded);
+  const phones = extractPhones(text, refs.phoneValues);
+  return {
+    url: finalUrl,
+    statusCode,
+    title: compactText(extractTitle(decoded)),
+    text,
+    links: refs.links,
+    emails: extractEmails(decoded, refs.emailValues),
+    phones,
+    socialUrls: refs.socialUrls,
+    contactUrls: refs.contactUrls,
+    hasForm: /<form\b/i.test(decoded),
+  };
 }
 
-async function allowedByRobots(pageUrl, cache, options, warnings) {
+async function allowedByRobots(pageUrl, context, options, warnings) {
   const origin = originOf(pageUrl);
   if (!origin) return true;
-  if (!cache.has(origin)) {
-    cache.set(origin, await fetchRobots(origin, options).catch((error) => {
+  let rules = getCachedRobots(context.cache, origin);
+  if (!rules) {
+    try {
+      if (!context.robotsInFlight.has(origin)) {
+        const promise = fetchRobots(origin, options, context).finally(() => {
+          context.robotsInFlight.delete(origin);
+        });
+        context.robotsInFlight.set(origin, promise);
+      }
+      rules = await context.robotsInFlight.get(origin);
+      setCachedRobots(context.cache, origin, rules);
+    } catch (error) {
       warnings.push(`robots.txt check failed for ${origin}: ${error instanceof Error ? error.message : String(error)}`);
-      return { disallow: [] };
-    }));
+      rules = { disallow: [] };
+    }
   }
-  const rules = cache.get(origin);
   let path = '/';
   try {
     const parsed = new URL(pageUrl);
@@ -274,11 +371,12 @@ async function allowedByRobots(pageUrl, cache, options, warnings) {
   return !rules.disallow.some((rule) => rule && path.startsWith(rule));
 }
 
-async function fetchRobots(origin, options) {
-  const response = await fetch(`${origin.replace(/\/+$/, '')}/robots.txt`, {
+async function fetchRobots(origin, options, context) {
+  const robotsUrl = `${origin.replace(/\/+$/, '')}/robots.txt`;
+  const response = await withOriginPace(context, robotsUrl, options, () => fetch(robotsUrl, {
     headers: { 'User-Agent': options.userAgent || DEFAULT_USER_AGENT },
     signal: AbortSignal.timeout?.(5_000),
-  });
+  }));
   if (response.status === 404) return { disallow: [] };
   if (response.status >= 400) throw new Error(`robots.txt returned HTTP ${response.status}`);
   return parseRobots(await response.text());
@@ -317,51 +415,64 @@ function extractTitle(html) {
   return decodeEntities(String(html || '').match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '');
 }
 
-function extractLinks(baseUrl, html) {
+function extractPageRefs(baseUrl, html) {
   const links = [];
-  const seen = new Set();
+  const linkSeen = new Set();
+  const hrefUrls = [];
+  const hrefSeen = new Set();
+  const emailValues = [];
+  const phoneValues = [];
   const hrefRe = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
   let match;
-  while ((match = hrefRe.exec(html))) {
-    const href = match[1] || match[2] || match[3] || '';
-    const link = normalizeLink(baseUrl, href);
-    if (!link || seen.has(link)) continue;
-    seen.add(link);
-    links.push(link);
-  }
-  links.sort((a, b) => pagePriority(b) - pagePriority(a));
-  return links;
-}
+  while ((match = hrefRe.exec(String(html || '')))) {
+    const raw = decodeEntities(match[1] || match[2] || match[3] || '').trim();
+    if (!raw) continue;
+    const lower = raw.toLowerCase();
+    if (lower.startsWith('mailto:')) {
+      emailValues.push(decodeURIComponentSafe(raw.slice('mailto:'.length).split('?')[0]));
+      continue;
+    }
+    if (lower.startsWith('tel:')) {
+      phoneValues.push(decodeURIComponentSafe(raw.slice('tel:'.length).split('?')[0]));
+      continue;
+    }
+    if (lower.startsWith('javascript:') || lower.startsWith('#')) continue;
 
-function normalizeLink(baseUrl, href) {
-  const raw = String(href || '').trim();
-  if (!raw) return '';
-  const lower = raw.toLowerCase();
-  if (lower.startsWith('mailto:') || lower.startsWith('tel:') || lower.startsWith('javascript:') || lower.startsWith('#')) {
-    return '';
-  }
-  let url;
-  try {
-    url = new URL(raw, baseUrl);
-  } catch {
-    return '';
-  }
-  if (!['http:', 'https:'].includes(url.protocol)) return '';
-  url.hash = '';
-  url.search = '';
-  if (skipByExtension(url.pathname)) return '';
-  return url.toString();
-}
+    let url;
+    try {
+      url = new URL(raw, baseUrl);
+    } catch {
+      continue;
+    }
+    if (!['http:', 'https:'].includes(url.protocol)) continue;
 
-function extractEmails(html) {
-  const decoded = decodeEntities(String(html || ''));
-  const values = [decoded];
-  const mailtoRe = /href\s*=\s*(?:"mailto:([^"]+)"|'mailto:([^']+)'|mailto:([^\s>]+))/gi;
-  let match;
-  while ((match = mailtoRe.exec(decoded))) {
-    values.push(decodeURIComponentSafe(match[1] || match[2] || match[3] || ''));
+    url.hash = '';
+    const hrefUrl = url.toString();
+    if (!hrefSeen.has(hrefUrl)) {
+      hrefSeen.add(hrefUrl);
+      hrefUrls.push(hrefUrl);
+    }
+
+    url.search = '';
+    if (skipByExtension(url.pathname)) continue;
+    const link = url.toString();
+    if (!linkSeen.has(link)) {
+      linkSeen.add(link);
+      links.push(link);
+    }
   }
+  links.sort((a, b) => pagePriority(b) - pagePriority(a));
+  return {
+    links,
+    emailValues,
+    phoneValues,
+    socialUrls: hrefUrls.filter((url) => isPublicProfileUrl(url)).slice(0, 8),
+    contactUrls: hrefUrls.filter((url) => isSchedulingOrContactUrl(url)).slice(0, 8),
+  };
+}
 
+function extractEmails(html, extraValues = []) {
+  const values = [String(html || ''), ...extraValues];
   const emails = new Set();
   for (const value of values) {
     for (const found of String(value || '').matchAll(EMAIL_RE)) {
@@ -372,15 +483,8 @@ function extractEmails(html) {
   return [...emails].sort();
 }
 
-function extractPhones(html, text) {
-  const decoded = decodeEntities(String(html || ''));
-  const values = [String(text || '')];
-  const telRe = /href\s*=\s*(?:"tel:([^"]+)"|'tel:([^']+)'|tel:([^\s>]+))/gi;
-  let match;
-  while ((match = telRe.exec(decoded))) {
-    values.push(decodeURIComponentSafe(match[1] || match[2] || match[3] || ''));
-  }
-
+function extractPhones(text, extraValues = []) {
+  const values = [String(text || ''), ...extraValues];
   const phones = new Set();
   for (const value of values) {
     for (const found of String(value || '').matchAll(PHONE_RE)) {
@@ -391,44 +495,7 @@ function extractPhones(html, text) {
   return [...phones].sort().slice(0, 5);
 }
 
-function extractSocialUrls(baseUrl, html) {
-  return extractHrefUrls(baseUrl, html)
-    .filter((url) => isPublicProfileUrl(url))
-    .slice(0, 8);
-}
-
-function extractContactUrls(baseUrl, html) {
-  return extractHrefUrls(baseUrl, html)
-    .filter((url) => isSchedulingOrContactUrl(url))
-    .slice(0, 8);
-}
-
-function extractHrefUrls(baseUrl, html) {
-  const urls = [];
-  const seen = new Set();
-  const hrefRe = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
-  let match;
-  while ((match = hrefRe.exec(String(html || '')))) {
-    const raw = decodeEntities(match[1] || match[2] || match[3] || '');
-    const lower = raw.toLowerCase();
-    if (!raw || lower.startsWith('mailto:') || lower.startsWith('tel:') || lower.startsWith('javascript:') || lower.startsWith('#')) continue;
-    let url;
-    try {
-      url = new URL(raw, baseUrl);
-    } catch {
-      continue;
-    }
-    if (!['http:', 'https:'].includes(url.protocol)) continue;
-    url.hash = '';
-    const value = url.toString();
-    if (seen.has(value)) continue;
-    seen.add(value);
-    urls.push(value);
-  }
-  return urls;
-}
-
-async function buildEmailLead(email, companyDomain, websiteUrl, page) {
+async function buildEmailLead(email, companyDomain, websiteUrl, page, context) {
   const sourceUrl = page.url;
   const pageTitle = page.title;
   const text = page.text;
@@ -481,7 +548,7 @@ async function buildEmailLead(email, companyDomain, websiteUrl, page) {
     confidence = Math.max(confidence, 38);
   }
 
-  const verificationStatus = await emailVerificationStatus(emailDomain);
+  const verificationStatus = await emailVerificationStatus(emailDomain, context);
   if (verificationStatus !== 'mx_verified' && emailType !== 'blocked') {
     warnings.push('email domain MX could not be verified');
   }
@@ -551,8 +618,24 @@ function buildContactPath(domain, websiteUrl, page) {
   return lead;
 }
 
-async function emailVerificationStatus(domain) {
+async function emailVerificationStatus(domain, context) {
   if (!domain) return 'unknown';
+  const cached = getCachedDnsStatus(context.cache, domain);
+  if (cached) return cached;
+  if (context.dnsInFlight.has(domain)) return context.dnsInFlight.get(domain);
+
+  const promise = context.dnsLimiter(async () => {
+    const status = await resolveEmailVerificationStatus(domain);
+    setCachedDnsStatus(context.cache, domain, status);
+    return status;
+  }).finally(() => {
+    context.dnsInFlight.delete(domain);
+  });
+  context.dnsInFlight.set(domain, promise);
+  return promise;
+}
+
+async function resolveEmailVerificationStatus(domain) {
   try {
     const mx = await resolveMx(domain);
     if (mx.length > 0) return 'mx_verified';
@@ -575,10 +658,19 @@ function classifyEmailLocal(local) {
 }
 
 function isRelevantLead(lead, options) {
+  if (lead.email && !isGoodLead(lead)) return false;
   if (!options.includeBlocked && lead.emailType === 'blocked') return false;
   return lead.confidence >= options.minConfidence;
 }
 
+function countGoodLeads(leadsByKey) {
+  let count = 0;
+  for (const lead of leadsByKey.values()) {
+    if (isGoodLead(lead)) count++;
+  }
+  return count;
+}
+
 function sameDomain(domain, link) {
   try {
     const host = new URL(link).hostname.toLowerCase().replace(/^www\./, '');
@@ -904,6 +996,205 @@ function decodeURIComponentSafe(value) {
   }
 }
 
+async function mapLimit(items, limit, worker) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+  const workerCount = Math.min(limit, items.length);
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex++;
+      results[index] = await worker(items[index], index);
+    }
+  }));
+  return results;
+}
+
+function createLimiter(limit) {
+  let active = 0;
+  const queue = [];
+  const runNext = () => {
+    if (active >= limit || queue.length === 0) return;
+    const item = queue.shift();
+    active++;
+    Promise.resolve()
+      .then(item.task)
+      .then(item.resolve, item.reject)
+      .finally(() => {
+        active--;
+        runNext();
+      });
+  };
+  return (task) => new Promise((resolve, reject) => {
+    queue.push({ task, resolve, reject });
+    runNext();
+  });
+}
+
+async function withOriginPace(context, url, options, task) {
+  const delayMs = boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000);
+  if (delayMs <= 0) return task();
+  const origin = originOf(url) || String(url || '');
+  if (!origin) return task();
+
+  let pace = context.originPaces.get(origin);
+  if (!pace) {
+    pace = { gate: Promise.resolve(), nextAt: 0 };
+    context.originPaces.set(origin, pace);
+  }
+
+  let releaseGate;
+  const previousGate = pace.gate;
+  pace.gate = new Promise((resolve) => {
+    releaseGate = resolve;
+  });
+  await previousGate;
+
+  const now = Date.now();
+  const waitMs = Math.max(0, pace.nextAt - now);
+  pace.nextAt = Math.max(now, pace.nextAt) + delayMs;
+  releaseGate();
+
+  if (waitMs > 0) await sleep(waitMs);
+  return task();
+}
+
+function crawlerConcurrency(options) {
+  return boundedInteger(
+    options.concurrency ?? options.maxConcurrency ?? options.maxConcurrentDomains,
+    DEFAULT_DOMAIN_CONCURRENCY,
+    1,
+    HARD_DOMAIN_CONCURRENCY,
+  );
+}
+
+function loadCrawlerCache(options = {}) {
+  const enabled = !(options.noCache || options.disableCache);
+  const cache = {
+    enabled,
+    path: resolveProjectPath(options.cachePath || DEFAULT_CACHE_PATH),
+    robots: new Map(),
+    dns: new Map(),
+    pages: new Map(),
+    dirty: false,
+    robotsTtlMs: boundedInteger(options.robotsCacheTtlMs, DEFAULT_ROBOTS_CACHE_TTL_MS, 0, MAX_CACHE_TTL_MS),
+    dnsTtlMs: boundedInteger(options.dnsCacheTtlMs, DEFAULT_DNS_CACHE_TTL_MS, 0, MAX_CACHE_TTL_MS),
+    pageTtlMs: pageCacheTtlMs(options),
+  };
+  if (!enabled || !existsSync(cache.path)) return cache;
+
+  const value = readCrawlerCacheFile(cache.path);
+  for (const [origin, record] of Object.entries(value.robots || {})) {
+    if (origin && record?.rules) cache.robots.set(origin, record);
+  }
+  for (const [domain, record] of Object.entries(value.dns || {})) {
+    if (domain && record?.status) cache.dns.set(domain, record);
+  }
+  for (const [pageUrl, record] of Object.entries(value.pages || {})) {
+    if (pageUrl && record?.page) cache.pages.set(pageUrl, record);
+  }
+  return cache;
+}
+
+function saveCrawlerCache(cache) {
+  if (!cache?.enabled || !cache.dirty) return;
+  const current = readCrawlerCacheFile(cache.path);
+  const value = {
+    version: 1,
+    generatedAt: new Date().toISOString(),
+    robots: mergeTimedRecords(current.robots, Object.fromEntries(cache.robots), 'fetchedAt'),
+    dns: mergeTimedRecords(current.dns, Object.fromEntries(cache.dns), 'checkedAt'),
+    pages: mergeTimedRecords(current.pages, Object.fromEntries(cache.pages), 'fetchedAt'),
+  };
+  mkdirSync(dirname(cache.path), { recursive: true });
+  const tempPath = `${cache.path}.${process.pid}.${Date.now()}.tmp`;
+  writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+  renameSync(tempPath, cache.path);
+  cache.dirty = false;
+}
+
+function readCrawlerCacheFile(path) {
+  try {
+    const value = JSON.parse(readFileSync(path, 'utf8'));
+    return value && typeof value === 'object' ? value : {};
+  } catch {
+    return {};
+  }
+}
+
+function mergeTimedRecords(current = {}, incoming = {}, timestampKey) {
+  const out = { ...(current && typeof current === 'object' ? current : {}) };
+  for (const [key, record] of Object.entries(incoming || {})) {
+    const currentTime = Date.parse(out[key]?.[timestampKey] || '') || 0;
+    const nextTime = Date.parse(record?.[timestampKey] || '') || 0;
+    if (!out[key] || nextTime >= currentTime) out[key] = record;
+  }
+  return out;
+}
+
+function getCachedRobots(cache, origin) {
+  const record = freshRecord(cache.robots.get(origin), 'fetchedAt', cache.robotsTtlMs);
+  return record?.rules || null;
+}
+
+function setCachedRobots(cache, origin, rules) {
+  if (!cache.enabled || !origin) return;
+  cache.robots.set(origin, {
+    fetchedAt: new Date().toISOString(),
+    rules: {
+      disallow: Array.isArray(rules?.disallow) ? rules.disallow.map((item) => String(item)) : [],
+    },
+  });
+  cache.dirty = true;
+}
+
+function getCachedDnsStatus(cache, domain) {
+  const record = freshRecord(cache.dns.get(domain), 'checkedAt', cache.dnsTtlMs);
+  return record?.status || '';
+}
+
+function setCachedDnsStatus(cache, domain, status) {
+  if (!cache.enabled || !domain) return;
+  cache.dns.set(domain, {
+    checkedAt: new Date().toISOString(),
+    status,
+  });
+  cache.dirty = true;
+}
+
+function getCachedPage(cache, pageUrl) {
+  const record = freshRecord(cache.pages.get(pageUrl), 'fetchedAt', cache.pageTtlMs);
+  return record?.page || null;
+}
+
+function setCachedPage(cache, pageUrl, page) {
+  if (!cache.enabled || cache.pageTtlMs <= 0 || !pageUrl || !page) return;
+  cache.pages.set(pageUrl, {
+    fetchedAt: new Date().toISOString(),
+    page,
+  });
+  cache.dirty = true;
+}
+
+function freshRecord(record, timestampKey, ttlMs) {
+  if (!record || ttlMs <= 0) return null;
+  const timestamp = Date.parse(record[timestampKey] || '');
+  if (!Number.isFinite(timestamp)) return null;
+  return Date.now() - timestamp <= ttlMs ? record : null;
+}
+
+function pageCacheTtlMs(options) {
+  const configured = Number(options.pageCacheTtlMs);
+  if (Number.isFinite(configured) && configured > 0) {
+    return boundedInteger(configured, DEFAULT_PAGE_CACHE_TTL_MS, 1_000, MAX_CACHE_TTL_MS);
+  }
+  return optionFlag(options.pageCache) ? DEFAULT_PAGE_CACHE_TTL_MS : 0;
+}
+
+function optionFlag(value) {
+  if (typeof value === 'boolean') return value;
+  return /^(1|true|yes|on)$/i.test(String(value || '').trim());
+}
+
 function boundedInteger(value, fallback, min, max) {
   const number = Number(value);
   if (!Number.isFinite(number)) return fallback;