@agent-pattern-labs/leads-rig 0.1.6 → 0.1.8

package/batch/README.md

@@ -17,10 +17,11 @@ id	domain	company	notes
 ```bash
 batch/batch-runner.sh --dry-run
 batch/batch-runner.sh --parallel 2
+batch/batch-runner.sh --parallel 2 --timeout-ms 8000
 batch/batch-runner.sh --runner codex --parallel 2
 ```
 
-`--parallel` is capped at `2` to keep browser-heavy work bounded. Worker CLI permission-bypass flags are disabled by default; pass `--allow-unsafe-workers` only in a trusted local workspace when you explicitly want the old unsafe behavior.
+`--parallel` is capped at `2` to keep browser-heavy work bounded. `--timeout-ms` controls the deterministic crawler timeout suggested to workers; the batch default is `8000` to avoid spending too long on dead sites. Worker CLI permission-bypass flags are disabled by default; pass `--allow-unsafe-workers` only in a trusted local workspace when you explicitly want the old unsafe behavior.
 
 The runner writes:
 

package/config/profile.example.yml

@@ -18,10 +18,23 @@ api:
 
 crawl:
   max_pages: 10
+  timeout_ms: 14000
   max_domains_per_batch: 50
   concurrency: 2
+  page_concurrency: 1
+  dns_concurrency: 8
+  cache_path: ".leadharness-cache/crawler-cache.json"
+  robots_cache_ttl_ms: 86400000
+  dns_cache_ttl_ms: 604800000
+  page_cache: false
+  page_cache_ttl_ms: 0
+  stop_after_good_leads: 0
+  stop_after_contact_path: false
   user_agent: "PublicLeadsBot/0.1 (+https://example.com/public-leads)"
 
+batch:
+  batch_timeout_ms: 8000
+
 lead_policy:
   min_confidence: 30
   allowed_email_types:

package/docs/ARCHITECTURE.md

@@ -82,4 +82,8 @@ The package also carries the `@agent-pattern-labs/iso-*` helper ecosystem for tr
 - page visits
 - ingest requests
 
+Lead records preserve optional enrichment fields such as `linkedinUrl`, `phone`,
+`address`, `streetAddress`, `location`, `city`, `region`, `postalCode`, and
+`country` through normalization, deduplication, and ingest.
+
 The validator intentionally accepts the same defaults the local runtime normalizes, but it fails missing source evidence, invalid URL fields, invalid `emailType`, invalid `verificationStatus`, invalid confidence, person leads without email, and generic catch-all inboxes such as `info@`, `hello@`, or similar organizational aliases. Any email record must be a named `person` lead with a non-generic email and a human owner visible in the evidence; role inboxes, unknown-owner emails, blocked emails, unsupported emails, and unnamed person-like emails fail validation. Summaries expose `goodLeadCount`, which only counts those high-quality named human email records.

package/lib/leadharness-crawler.mjs

@@ -1,14 +1,27 @@
 import { createHash } from 'crypto';
 import { lookup, resolveMx } from 'dns/promises';
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs';
+import { dirname } from 'path';
 import { setTimeout as sleep } from 'timers/promises';
-import { cleanDomains, dedupeLeadRecords, isGoodLead, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain } from './leadharness-leads.mjs';
+import { cleanDomains, dedupeLeadRecords, isGoodLead, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain, resolveProjectPath } from './leadharness-leads.mjs';
 
 const DEFAULT_MAX_PAGES = 10;
 const HARD_MAX_PAGES = 25;
+const DEFAULT_DOMAIN_CONCURRENCY = 2;
+const HARD_DOMAIN_CONCURRENCY = 10;
+const DEFAULT_PAGE_CONCURRENCY = 1;
+const HARD_PAGE_CONCURRENCY = 2;
+const DEFAULT_DNS_CONCURRENCY = 8;
+const HARD_DNS_CONCURRENCY = 25;
 const DEFAULT_TIMEOUT_MS = 14_000;
 const DEFAULT_DELAY_MS = 200;
 const DEFAULT_USER_AGENT = 'PublicLeadsBot/0.1 (+https://example.com/public-leads)';
 const MAX_BODY_CHARS = 2 * 1024 * 1024;
+const DEFAULT_CACHE_PATH = '.leadharness-cache/crawler-cache.json';
+const DEFAULT_ROBOTS_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_DNS_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_PAGE_CACHE_TTL_MS = 60 * 60 * 1000;
+const MAX_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const EMAIL_RE = /\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b/gi;
 const NAME_RE = /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,2})\b/g;
 const PHONE_RE = /(?:\+?1[\s.-]?)?(?:\([2-9]\d{2}\)|[2-9]\d{2})[\s.-]?[2-9]\d{2}[\s.-]?\d{4}\b/g;
@@ -35,27 +48,43 @@ export async function crawlDomains(inputs, options = {}) {
   const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, HARD_MAX_PAGES);
   const minConfidence = boundedInteger(options.minConfidence, 30, 0, 100);
   const includeBlocked = Boolean(options.includeBlocked);
+  const concurrency = crawlerConcurrency(options);
+  const context = createCrawlContext(options);
   const results = [];
   const errors = [];
   const leadsByKey = new Map();
 
-  for (const target of targets) {
-    try {
-      const result = await crawlDomainTarget(target, {
-        ...options,
-        maxPages,
-        minConfidence,
-        includeBlocked,
-      });
-      results.push(result);
-      for (const lead of result.leads) {
+  try {
+    const settled = await mapLimit(targets, concurrency, async (target) => {
+      try {
+        return {
+          target,
+          result: await crawlDomainTarget(target, {
+            ...options,
+            maxPages,
+            minConfidence,
+            includeBlocked,
+          }, context),
+        };
+      } catch (error) {
+        return { target, error };
+      }
+    });
+
+    for (const item of settled) {
+      if (item.error) {
+        errors.push(`${item.target.domain}: ${item.error instanceof Error ? item.error.message : String(item.error)}`);
+        continue;
+      }
+      results.push(item.result);
+      for (const lead of item.result.leads) {
         const key = leadKey(lead);
         const current = leadsByKey.get(key);
         leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
       }
-    } catch (error) {
-      errors.push(`${target.domain}: ${error instanceof Error ? error.message : String(error)}`);
     }
+  } finally {
+    saveCrawlerCache(context.cache);
   }
 
   const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
@@ -77,7 +106,12 @@ export async function crawlDomains(inputs, options = {}) {
 export async function crawlDomain(input, options = {}) {
   const [target] = normalizeCrawlTargets([input]);
   if (!target) throw new Error('domain is required');
-  return crawlDomainTarget(target, options);
+  const context = createCrawlContext(options);
+  try {
+    return await crawlDomainTarget(target, options, context);
+  } finally {
+    saveCrawlerCache(context.cache);
+  }
 }
 
 export function normalizeCrawlTargets(inputs) {
@@ -124,73 +158,110 @@ function normalizeCrawlTarget(input) {
   };
 }
 
-async function crawlDomainTarget(target, options) {
+function createCrawlContext(options) {
+  return {
+    cache: loadCrawlerCache(options),
+    dnsInFlight: new Map(),
+    originPaces: new Map(),
+    robotsInFlight: new Map(),
+    dnsLimiter: createLimiter(boundedInteger(options.dnsConcurrency, DEFAULT_DNS_CONCURRENCY, 1, HARD_DNS_CONCURRENCY)),
+  };
+}
+
+async function crawlDomainTarget(target, options, context) {
   const maxPages = boundedInteger(options.maxPages, DEFAULT_MAX_PAGES, 1, HARD_MAX_PAGES);
   const minConfidence = boundedInteger(options.minConfidence, 30, 0, 100);
+  const pageConcurrency = boundedInteger(options.pageConcurrency, DEFAULT_PAGE_CONCURRENCY, 1, HARD_PAGE_CONCURRENCY);
   const queue = [...target.starts];
   const queued = new Set(queue);
   const seen = new Set();
+  const inFlight = new Set();
   const leadsByKey = new Map();
   const warnings = [];
   const pages = [];
-  const robotsCache = new Map();
+  const stopAfterGoodLeads = boundedInteger(options.stopAfterGoodLeads, 0, 0, 1000);
+  const stopAfterContactPath = optionFlag(options.stopAfterContactPath);
+  let contactPathFound = false;
   let websiteUrl = '';
 
-  while (queue.length > 0 && pages.length < maxPages) {
-    const pageUrl = queue.shift();
-    if (!pageUrl || seen.has(pageUrl)) continue;
-    seen.add(pageUrl);
-
-    const allowed = await allowedByRobots(pageUrl, robotsCache, options, warnings);
-    if (!allowed) {
-      pages.push({ url: pageUrl, title: '', statusCode: 0, emailsFound: 0, error: 'blocked by robots.txt' });
-      continue;
-    }
-
-    const page = await fetchPage(pageUrl, options).catch((error) => ({
-      error: error instanceof Error ? error.message : String(error),
-    }));
-    if (page.error) {
-      pages.push({ url: pageUrl, title: '', statusCode: 0, emailsFound: 0, error: page.error });
-      continue;
+  const shouldStop = () => {
+    if (stopAfterContactPath && contactPathFound) return true;
+    if (stopAfterGoodLeads > 0 && countGoodLeads(leadsByKey) >= stopAfterGoodLeads) return true;
+    return false;
+  };
+  const schedulePages = () => {
+    while (!shouldStop() && inFlight.size < pageConcurrency && queue.length > 0 && pages.length + inFlight.size < maxPages) {
+      const pageUrl = queue.shift();
+      if (!pageUrl || seen.has(pageUrl)) continue;
+      seen.add(pageUrl);
+      let task;
+      task = crawlPage(pageUrl, context, options, warnings)
+        .then((result) => ({ task, result }))
+        .catch((error) => ({
+          task,
+          result: {
+            pageUrl,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        }));
+      inFlight.add(task);
     }
+  };
 
-    if (!websiteUrl) websiteUrl = originOf(page.url);
-    pages.push({
-      url: page.url,
-      title: page.title,
-      statusCode: page.statusCode,
-      emailsFound: page.emails.length,
-      phonesFound: page.phones.length,
-      socialUrlsFound: page.socialUrls.length,
-      contactUrlsFound: page.contactUrls.length,
-    });
+  schedulePages();
+  while (inFlight.size > 0) {
+    const { task, result } = await Promise.race(inFlight);
+    inFlight.delete(task);
 
-    for (const email of page.emails) {
-      const lead = await buildEmailLead(email, target.domain, websiteUrl, page);
-      if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
-      const key = leadKey(lead);
-      const current = leadsByKey.get(key);
-      leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
-    }
+    if (result.blocked) {
+      pages.push({ url: result.pageUrl, title: '', statusCode: 0, emailsFound: 0, error: 'blocked by robots.txt' });
+    } else if (result.error) {
+      pages.push({ url: result.pageUrl, title: '', statusCode: 0, emailsFound: 0, error: result.error });
+    } else {
+      const page = result.page;
+      if (!websiteUrl) websiteUrl = originOf(page.url);
+      pages.push({
+        url: page.url,
+        title: page.title,
+        statusCode: page.statusCode,
+        emailsFound: page.emails.length,
+        phonesFound: page.phones.length,
+        socialUrlsFound: page.socialUrls.length,
+        contactUrlsFound: page.contactUrls.length,
+      });
 
-    if (page.hasForm && isContactLikeURL(page.url)) {
-      const lead = buildContactPath(target.domain, websiteUrl, page);
-      if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
+      const emailLeads = await Promise.all(
+        page.emails.map((email) => buildEmailLead(email, target.domain, websiteUrl, page, context)),
+      );
+      for (const lead of emailLeads) {
+        if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
         const key = leadKey(lead);
         const current = leadsByKey.get(key);
         leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
       }
-    }
 
-    for (const link of page.links) {
-      if (queued.has(link) || seen.has(link)) continue;
-      if (!sameDomain(target.domain, link) || !isHighSignalPage(link)) continue;
-      queued.add(link);
-      queue.push(link);
+      if (page.hasForm && isContactLikeURL(page.url)) {
+        const lead = buildContactPath(target.domain, websiteUrl, page);
+        if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
+          const key = leadKey(lead);
+          const current = leadsByKey.get(key);
+          leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
+          contactPathFound = true;
+        }
+      }
+
+      if (!shouldStop()) {
+        for (const link of page.links) {
+          if (queued.has(link) || seen.has(link)) continue;
+          if (!sameDomain(target.domain, link) || !isHighSignalPage(link)) continue;
+          queued.add(link);
+          queue.push(link);
+        }
+        queue.sort((a, b) => pagePriority(b) - pagePriority(a));
+      }
     }
-    queue.sort((a, b) => pagePriority(b) - pagePriority(a));
-    await sleep(boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000));
+
+    schedulePages();
   }
 
   const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
@@ -208,62 +279,88 @@ async function crawlDomainTarget(target, options) {
   };
 }
 
-async function fetchPage(pageUrl, options) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), boundedInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1_000, 60_000));
-  try {
-    const response = await fetch(pageUrl, {
-      headers: {
-        Accept: 'text/html,application/xhtml+xml',
-        'User-Agent': options.userAgent || DEFAULT_USER_AGENT,
-      },
-      redirect: 'follow',
-      signal: controller.signal,
-    });
-    if (response.status >= 400) throw new Error(`HTTP ${response.status}`);
-    const contentType = String(response.headers.get('content-type') || '').toLowerCase();
-    if (contentType && !contentType.includes('html')) {
-      throw new Error(`skipped non-HTML content type ${contentType}`);
-    }
-    const contentLength = Number(response.headers.get('content-length') || 0);
-    if (contentLength > MAX_BODY_CHARS) {
-      throw new Error(`skipped large response (${contentLength} bytes)`);
+async function crawlPage(pageUrl, context, options, warnings) {
+  const allowed = await allowedByRobots(pageUrl, context, options, warnings);
+  if (!allowed) return { pageUrl, blocked: true };
+  const page = await fetchPage(pageUrl, options, context);
+  return { pageUrl, page };
+}
+
+async function fetchPage(pageUrl, options, context) {
+  const cached = getCachedPage(context.cache, pageUrl);
+  if (cached) return cached;
+
+  return withOriginPace(context, pageUrl, options, async () => {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), boundedInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1_000, 60_000));
+    try {
+      const response = await fetch(pageUrl, {
+        headers: {
+          Accept: 'text/html,application/xhtml+xml',
+          'User-Agent': options.userAgent || DEFAULT_USER_AGENT,
+        },
+        redirect: 'follow',
+        signal: controller.signal,
+      });
+      if (response.status >= 400) throw new Error(`HTTP ${response.status}`);
+      const contentType = String(response.headers.get('content-type') || '').toLowerCase();
+      if (contentType && !contentType.includes('html')) {
+        throw new Error(`skipped non-HTML content type ${contentType}`);
+      }
+      const contentLength = Number(response.headers.get('content-length') || 0);
+      if (contentLength > MAX_BODY_CHARS) {
+        throw new Error(`skipped large response (${contentLength} bytes)`);
+      }
+
+      const html = (await response.text()).slice(0, MAX_BODY_CHARS);
+      const finalUrl = response.url || pageUrl;
+      const page = parsePage(finalUrl, response.status, html);
+      setCachedPage(context.cache, pageUrl, page);
+      return page;
+    } finally {
+      clearTimeout(timeout);
     }
+  });
+}
 
-    const html = (await response.text()).slice(0, MAX_BODY_CHARS);
-    const decoded = decodeEntities(html);
-    const text = compactText(stripHtml(decoded));
-    const finalUrl = response.url || pageUrl;
-    const phones = extractPhones(decoded, text);
-    const socialUrls = extractSocialUrls(finalUrl, decoded);
-    const contactUrls = extractContactUrls(finalUrl, decoded);
-    return {
-      url: finalUrl,
-      statusCode: response.status,
-      title: compactText(extractTitle(decoded)),
-      text,
-      links: extractLinks(finalUrl, decoded),
-      emails: extractEmails(decoded),
-      phones,
-      socialUrls,
-      contactUrls,
-      hasForm: /<form\b/i.test(decoded),
-    };
-  } finally {
-    clearTimeout(timeout);
-  }
+function parsePage(finalUrl, statusCode, html) {
+  const decoded = decodeEntities(html);
+  const text = compactText(stripHtml(decoded));
+  const refs = extractPageRefs(finalUrl, decoded);
+  const phones = extractPhones(text, refs.phoneValues);
+  return {
+    url: finalUrl,
+    statusCode,
+    title: compactText(extractTitle(decoded)),
+    text,
+    links: refs.links,
+    emails: extractEmails(decoded, refs.emailValues),
+    phones,
+    socialUrls: refs.socialUrls,
+    contactUrls: refs.contactUrls,
+    hasForm: /<form\b/i.test(decoded),
+  };
 }
 
-async function allowedByRobots(pageUrl, cache, options, warnings) {
+async function allowedByRobots(pageUrl, context, options, warnings) {
   const origin = originOf(pageUrl);
   if (!origin) return true;
-  if (!cache.has(origin)) {
-    cache.set(origin, await fetchRobots(origin, options).catch((error) => {
+  let rules = getCachedRobots(context.cache, origin);
+  if (!rules) {
+    try {
+      if (!context.robotsInFlight.has(origin)) {
+        const promise = fetchRobots(origin, options, context).finally(() => {
+          context.robotsInFlight.delete(origin);
+        });
+        context.robotsInFlight.set(origin, promise);
+      }
+      rules = await context.robotsInFlight.get(origin);
+      setCachedRobots(context.cache, origin, rules);
+    } catch (error) {
       warnings.push(`robots.txt check failed for ${origin}: ${error instanceof Error ? error.message : String(error)}`);
-      return { disallow: [] };
-    }));
+      rules = { disallow: [] };
+    }
   }
-  const rules = cache.get(origin);
   let path = '/';
   try {
     const parsed = new URL(pageUrl);
@@ -274,11 +371,12 @@ async function allowedByRobots(pageUrl, cache, options, warnings) {
   return !rules.disallow.some((rule) => rule && path.startsWith(rule));
 }
 
-async function fetchRobots(origin, options) {
-  const response = await fetch(`${origin.replace(/\/+$/, '')}/robots.txt`, {
+async function fetchRobots(origin, options, context) {
+  const robotsUrl = `${origin.replace(/\/+$/, '')}/robots.txt`;
+  const response = await withOriginPace(context, robotsUrl, options, () => fetch(robotsUrl, {
     headers: { 'User-Agent': options.userAgent || DEFAULT_USER_AGENT },
     signal: AbortSignal.timeout?.(5_000),
-  });
+  }));
   if (response.status === 404) return { disallow: [] };
   if (response.status >= 400) throw new Error(`robots.txt returned HTTP ${response.status}`);
   return parseRobots(await response.text());
@@ -317,51 +415,64 @@ function extractTitle(html) {
   return decodeEntities(String(html || '').match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '');
 }
 
-function extractLinks(baseUrl, html) {
+function extractPageRefs(baseUrl, html) {
   const links = [];
-  const seen = new Set();
+  const linkSeen = new Set();
+  const hrefUrls = [];
+  const hrefSeen = new Set();
+  const emailValues = [];
+  const phoneValues = [];
   const hrefRe = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
   let match;
-  while ((match = hrefRe.exec(html))) {
-    const href = match[1] || match[2] || match[3] || '';
-    const link = normalizeLink(baseUrl, href);
-    if (!link || seen.has(link)) continue;
-    seen.add(link);
-    links.push(link);
-  }
-  links.sort((a, b) => pagePriority(b) - pagePriority(a));
-  return links;
-}
+  while ((match = hrefRe.exec(String(html || '')))) {
+    const raw = decodeEntities(match[1] || match[2] || match[3] || '').trim();
+    if (!raw) continue;
+    const lower = raw.toLowerCase();
+    if (lower.startsWith('mailto:')) {
+      emailValues.push(decodeURIComponentSafe(raw.slice('mailto:'.length).split('?')[0]));
+      continue;
+    }
+    if (lower.startsWith('tel:')) {
+      phoneValues.push(decodeURIComponentSafe(raw.slice('tel:'.length).split('?')[0]));
+      continue;
+    }
+    if (lower.startsWith('javascript:') || lower.startsWith('#')) continue;
 
-function normalizeLink(baseUrl, href) {
-  const raw = String(href || '').trim();
-  if (!raw) return '';
-  const lower = raw.toLowerCase();
-  if (lower.startsWith('mailto:') || lower.startsWith('tel:') || lower.startsWith('javascript:') || lower.startsWith('#')) {
-    return '';
-  }
-  let url;
-  try {
-    url = new URL(raw, baseUrl);
-  } catch {
-    return '';
-  }
-  if (!['http:', 'https:'].includes(url.protocol)) return '';
-  url.hash = '';
-  url.search = '';
-  if (skipByExtension(url.pathname)) return '';
-  return url.toString();
-}
+    let url;
+    try {
+      url = new URL(raw, baseUrl);
+    } catch {
+      continue;
+    }
+    if (!['http:', 'https:'].includes(url.protocol)) continue;
 
-function extractEmails(html) {
-  const decoded = decodeEntities(String(html || ''));
-  const values = [decoded];
-  const mailtoRe = /href\s*=\s*(?:"mailto:([^"]+)"|'mailto:([^']+)'|mailto:([^\s>]+))/gi;
-  let match;
-  while ((match = mailtoRe.exec(decoded))) {
-    values.push(decodeURIComponentSafe(match[1] || match[2] || match[3] || ''));
+    url.hash = '';
+    const hrefUrl = url.toString();
+    if (!hrefSeen.has(hrefUrl)) {
+      hrefSeen.add(hrefUrl);
+      hrefUrls.push(hrefUrl);
+    }
+
+    url.search = '';
+    if (skipByExtension(url.pathname)) continue;
+    const link = url.toString();
+    if (!linkSeen.has(link)) {
+      linkSeen.add(link);
+      links.push(link);
+    }
   }
+  links.sort((a, b) => pagePriority(b) - pagePriority(a));
+  return {
+    links,
+    emailValues,
+    phoneValues,
+    socialUrls: hrefUrls.filter((url) => isPublicProfileUrl(url)).slice(0, 8),
+    contactUrls: hrefUrls.filter((url) => isSchedulingOrContactUrl(url)).slice(0, 8),
+  };
+}
 
+function extractEmails(html, extraValues = []) {
+  const values = [String(html || ''), ...extraValues];
   const emails = new Set();
   for (const value of values) {
     for (const found of String(value || '').matchAll(EMAIL_RE)) {
@@ -372,15 +483,8 @@ function extractEmails(html) {
   return [...emails].sort();
 }
 
-function extractPhones(html, text) {
-  const decoded = decodeEntities(String(html || ''));
-  const values = [String(text || '')];
-  const telRe = /href\s*=\s*(?:"tel:([^"]+)"|'tel:([^']+)'|tel:([^\s>]+))/gi;
-  let match;
-  while ((match = telRe.exec(decoded))) {
-    values.push(decodeURIComponentSafe(match[1] || match[2] || match[3] || ''));
-  }
-
+function extractPhones(text, extraValues = []) {
+  const values = [String(text || ''), ...extraValues];
   const phones = new Set();
   for (const value of values) {
     for (const found of String(value || '').matchAll(PHONE_RE)) {
@@ -391,44 +495,7 @@ function extractPhones(html, text) {
   return [...phones].sort().slice(0, 5);
 }
 
-function extractSocialUrls(baseUrl, html) {
-  return extractHrefUrls(baseUrl, html)
-    .filter((url) => isPublicProfileUrl(url))
-    .slice(0, 8);
-}
-
-function extractContactUrls(baseUrl, html) {
-  return extractHrefUrls(baseUrl, html)
-    .filter((url) => isSchedulingOrContactUrl(url))
-    .slice(0, 8);
-}
-
-function extractHrefUrls(baseUrl, html) {
-  const urls = [];
-  const seen = new Set();
-  const hrefRe = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
-  let match;
-  while ((match = hrefRe.exec(String(html || '')))) {
-    const raw = decodeEntities(match[1] || match[2] || match[3] || '');
-    const lower = raw.toLowerCase();
-    if (!raw || lower.startsWith('mailto:') || lower.startsWith('tel:') || lower.startsWith('javascript:') || lower.startsWith('#')) continue;
-    let url;
-    try {
-      url = new URL(raw, baseUrl);
-    } catch {
-      continue;
-    }
-    if (!['http:', 'https:'].includes(url.protocol)) continue;
-    url.hash = '';
-    const value = url.toString();
-    if (seen.has(value)) continue;
-    seen.add(value);
-    urls.push(value);
-  }
-  return urls;
-}
-
-async function buildEmailLead(email, companyDomain, websiteUrl, page) {
+async function buildEmailLead(email, companyDomain, websiteUrl, page, context) {
   const sourceUrl = page.url;
   const pageTitle = page.title;
   const text = page.text;
@@ -481,7 +548,7 @@ async function buildEmailLead(email, companyDomain, websiteUrl, page) {
     confidence = Math.max(confidence, 38);
   }
 
-  const verificationStatus = await emailVerificationStatus(emailDomain);
+  const verificationStatus = await emailVerificationStatus(emailDomain, context);
   if (verificationStatus !== 'mx_verified' && emailType !== 'blocked') {
     warnings.push('email domain MX could not be verified');
   }
@@ -551,8 +618,24 @@ function buildContactPath(domain, websiteUrl, page) {
   return lead;
 }
 
-async function emailVerificationStatus(domain) {
+async function emailVerificationStatus(domain, context) {
   if (!domain) return 'unknown';
+  const cached = getCachedDnsStatus(context.cache, domain);
+  if (cached) return cached;
+  if (context.dnsInFlight.has(domain)) return context.dnsInFlight.get(domain);
+
+  const promise = context.dnsLimiter(async () => {
+    const status = await resolveEmailVerificationStatus(domain);
+    setCachedDnsStatus(context.cache, domain, status);
+    return status;
+  }).finally(() => {
+    context.dnsInFlight.delete(domain);
+  });
+  context.dnsInFlight.set(domain, promise);
+  return promise;
+}
+
+async function resolveEmailVerificationStatus(domain) {
   try {
     const mx = await resolveMx(domain);
     if (mx.length > 0) return 'mx_verified';
@@ -580,6 +663,14 @@ function isRelevantLead(lead, options) {
   return lead.confidence >= options.minConfidence;
 }
 
+function countGoodLeads(leadsByKey) {
+  let count = 0;
+  for (const lead of leadsByKey.values()) {
+    if (isGoodLead(lead)) count++;
+  }
+  return count;
+}
+
 function sameDomain(domain, link) {
   try {
     const host = new URL(link).hostname.toLowerCase().replace(/^www\./, '');
@@ -905,6 +996,205 @@ function decodeURIComponentSafe(value) {
   }
 }
 
+async function mapLimit(items, limit, worker) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+  const workerCount = Math.min(limit, items.length);
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex++;
+      results[index] = await worker(items[index], index);
+    }
+  }));
+  return results;
+}
+
+function createLimiter(limit) {
+  let active = 0;
+  const queue = [];
+  const runNext = () => {
+    if (active >= limit || queue.length === 0) return;
+    const item = queue.shift();
+    active++;
+    Promise.resolve()
+      .then(item.task)
+      .then(item.resolve, item.reject)
+      .finally(() => {
+        active--;
+        runNext();
+      });
+  };
+  return (task) => new Promise((resolve, reject) => {
+    queue.push({ task, resolve, reject });
+    runNext();
+  });
+}
+
+async function withOriginPace(context, url, options, task) {
+  const delayMs = boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000);
+  if (delayMs <= 0) return task();
+  const origin = originOf(url) || String(url || '');
+  if (!origin) return task();
+
+  let pace = context.originPaces.get(origin);
+  if (!pace) {
+    pace = { gate: Promise.resolve(), nextAt: 0 };
+    context.originPaces.set(origin, pace);
+  }
+
+  let releaseGate;
+  const previousGate = pace.gate;
+  pace.gate = new Promise((resolve) => {
+    releaseGate = resolve;
+  });
+  await previousGate;
+
+  const now = Date.now();
+  const waitMs = Math.max(0, pace.nextAt - now);
+  pace.nextAt = Math.max(now, pace.nextAt) + delayMs;
+  releaseGate();
+
+  if (waitMs > 0) await sleep(waitMs);
+  return task();
+}
+
+function crawlerConcurrency(options) {
+  return boundedInteger(
+    options.concurrency ?? options.maxConcurrency ?? options.maxConcurrentDomains,
+    DEFAULT_DOMAIN_CONCURRENCY,
+    1,
+    HARD_DOMAIN_CONCURRENCY,
+  );
+}
+
+function loadCrawlerCache(options = {}) {
+  const enabled = !(options.noCache || options.disableCache);
+  const cache = {
+    enabled,
+    path: resolveProjectPath(options.cachePath || DEFAULT_CACHE_PATH),
+    robots: new Map(),
+    dns: new Map(),
+    pages: new Map(),
+    dirty: false,
+    robotsTtlMs: boundedInteger(options.robotsCacheTtlMs, DEFAULT_ROBOTS_CACHE_TTL_MS, 0, MAX_CACHE_TTL_MS),
+    dnsTtlMs: boundedInteger(options.dnsCacheTtlMs, DEFAULT_DNS_CACHE_TTL_MS, 0, MAX_CACHE_TTL_MS),
+    pageTtlMs: pageCacheTtlMs(options),
+  };
+  if (!enabled || !existsSync(cache.path)) return cache;
+
+  const value = readCrawlerCacheFile(cache.path);
+  for (const [origin, record] of Object.entries(value.robots || {})) {
+    if (origin && record?.rules) cache.robots.set(origin, record);
+  }
+  for (const [domain, record] of Object.entries(value.dns || {})) {
+    if (domain && record?.status) cache.dns.set(domain, record);
+  }
+  for (const [pageUrl, record] of Object.entries(value.pages || {})) {
+    if (pageUrl && record?.page) cache.pages.set(pageUrl, record);
+  }
+  return cache;
+}
+
+function saveCrawlerCache(cache) {
+  if (!cache?.enabled || !cache.dirty) return;
+  const current = readCrawlerCacheFile(cache.path);
+  const value = {
+    version: 1,
+    generatedAt: new Date().toISOString(),
+    robots: mergeTimedRecords(current.robots, Object.fromEntries(cache.robots), 'fetchedAt'),
+    dns: mergeTimedRecords(current.dns, Object.fromEntries(cache.dns), 'checkedAt'),
+    pages: mergeTimedRecords(current.pages, Object.fromEntries(cache.pages), 'fetchedAt'),
+  };
+  mkdirSync(dirname(cache.path), { recursive: true });
+  const tempPath = `${cache.path}.${process.pid}.${Date.now()}.tmp`;
+  writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+  renameSync(tempPath, cache.path);
+  cache.dirty = false;
+}
+
+function readCrawlerCacheFile(path) {
+  try {
+    const value = JSON.parse(readFileSync(path, 'utf8'));
+    return value && typeof value === 'object' ? value : {};
+  } catch {
+    return {};
+  }
+}
+
+function mergeTimedRecords(current = {}, incoming = {}, timestampKey) {
+  const out = { ...(current && typeof current === 'object' ? current : {}) };
+  for (const [key, record] of Object.entries(incoming || {})) {
+    const currentTime = Date.parse(out[key]?.[timestampKey] || '') || 0;
+    const nextTime = Date.parse(record?.[timestampKey] || '') || 0;
+    if (!out[key] || nextTime >= currentTime) out[key] = record;
+  }
+  return out;
+}
+
+function getCachedRobots(cache, origin) {
+  const record = freshRecord(cache.robots.get(origin), 'fetchedAt', cache.robotsTtlMs);
+  return record?.rules || null;
+}
+
+function setCachedRobots(cache, origin, rules) {
+  if (!cache.enabled || !origin) return;
+  cache.robots.set(origin, {
+    fetchedAt: new Date().toISOString(),
+    rules: {
+      disallow: Array.isArray(rules?.disallow) ? rules.disallow.map((item) => String(item)) : [],
+    },
+  });
+  cache.dirty = true;
+}
+
+function getCachedDnsStatus(cache, domain) {
+  const record = freshRecord(cache.dns.get(domain), 'checkedAt', cache.dnsTtlMs);
+  return record?.status || '';
+}
+
+function setCachedDnsStatus(cache, domain, status) {
+  if (!cache.enabled || !domain) return;
+  cache.dns.set(domain, {
+    checkedAt: new Date().toISOString(),
+    status,
+  });
+  cache.dirty = true;
+}
+
+function getCachedPage(cache, pageUrl) {
+  const record = freshRecord(cache.pages.get(pageUrl), 'fetchedAt', cache.pageTtlMs);
+  return record?.page || null;
+}
+
+function setCachedPage(cache, pageUrl, page) {
+  if (!cache.enabled || cache.pageTtlMs <= 0 || !pageUrl || !page) return;
+  cache.pages.set(pageUrl, {
+    fetchedAt: new Date().toISOString(),
+    page,
+  });
+  cache.dirty = true;
+}
+
+function freshRecord(record, timestampKey, ttlMs) {
+  if (!record || ttlMs <= 0) return null;
+  const timestamp = Date.parse(record[timestampKey] || '');
+  if (!Number.isFinite(timestamp)) return null;
+  return Date.now() - timestamp <= ttlMs ? record : null;
+}
+
+function pageCacheTtlMs(options) {
+  const configured = Number(options.pageCacheTtlMs);
+  if (Number.isFinite(configured) && configured > 0) {
+    return boundedInteger(configured, DEFAULT_PAGE_CACHE_TTL_MS, 1_000, MAX_CACHE_TTL_MS);
+  }
+  return optionFlag(options.pageCache) ? DEFAULT_PAGE_CACHE_TTL_MS : 0;
+}
+
+function optionFlag(value) {
+  if (typeof value === 'boolean') return value;
+  return /^(1|true|yes|on)$/i.test(String(value || '').trim());
+}
+
 function boundedInteger(value, fallback, min, max) {
   const number = Number(value);
   if (!Number.isFinite(number)) return fallback;

package/lib/leadharness-leads.mjs

@@ -162,7 +162,15 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
   const verificationStatus = normalizeVerificationStatus(input.verificationStatus, emailType);
   const confidence = clampInteger(input.confidence, 0, 100, 0);
   const foundAt = normalizeDateTime(input.foundAt, now);
+  const linkedinUrl = normalizeLinkedInUrl(input.linkedinUrl);
   const phone = stringValue(input.phone);
+  const address = compactWhitespace(input.address);
+  const streetAddress = compactWhitespace(input.streetAddress);
+  const location = compactWhitespace(input.location);
+  const city = compactWhitespace(input.city);
+  const region = compactWhitespace(input.region);
+  const postalCode = compactWhitespace(input.postalCode);
+  const country = compactWhitespace(input.country);
   const socialUrls = normalizeUrlList(input.socialUrls);
   const contactUrls = normalizeUrlList(input.contactUrls);
 
@@ -187,7 +195,15 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
     verificationStatus,
     confidence,
     warnings: normalizeWarnings(input.warnings),
+    linkedinUrl,
     phone,
+    address,
+    streetAddress,
+    location,
+    city,
+    region,
+    postalCode,
+    country,
     socialUrls,
     contactUrls,
     foundAt,
@@ -471,7 +487,18 @@ export function loadProfileConfig(root = PROJECT_DIR) {
     adminTokenEnv: yamlScalar(text, 'admin_token_env'),
     maxPages: yamlScalar(text, 'max_pages'),
     maxDomainsPerBatch: yamlScalar(text, 'max_domains_per_batch'),
+    timeoutMs: yamlScalar(text, 'timeout_ms'),
     concurrency: yamlScalar(text, 'concurrency'),
+    pageConcurrency: yamlScalar(text, 'page_concurrency'),
+    dnsConcurrency: yamlScalar(text, 'dns_concurrency'),
+    cachePath: yamlScalar(text, 'cache_path'),
+    robotsCacheTtlMs: yamlScalar(text, 'robots_cache_ttl_ms'),
+    dnsCacheTtlMs: yamlScalar(text, 'dns_cache_ttl_ms'),
+    pageCache: yamlScalar(text, 'page_cache'),
+    pageCacheTtlMs: yamlScalar(text, 'page_cache_ttl_ms'),
+    stopAfterGoodLeads: yamlScalar(text, 'stop_after_good_leads'),
+    stopAfterContactPath: yamlScalar(text, 'stop_after_contact_path'),
+    batchTimeoutMs: yamlScalar(text, 'batch_timeout_ms'),
     userAgent: yamlScalar(text, 'user_agent'),
     minConfidence: yamlScalar(text, 'min_confidence'),
   };
@@ -536,6 +563,15 @@ function normalizeLeadRecord(lead) {
     sources,
     evidence: sources[0]?.evidence || compactWhitespace(lead?.evidence),
     warnings: normalizeWarnings(lead?.warnings),
+    linkedinUrl: normalizeLinkedInUrl(lead?.linkedinUrl),
+    phone: stringValue(lead?.phone),
+    address: compactWhitespace(lead?.address),
+    streetAddress: compactWhitespace(lead?.streetAddress),
+    location: compactWhitespace(lead?.location),
+    city: compactWhitespace(lead?.city),
+    region: compactWhitespace(lead?.region),
+    postalCode: compactWhitespace(lead?.postalCode),
+    country: compactWhitespace(lead?.country),
     socialUrls: normalizeUrlList(lead?.socialUrls),
     contactUrls: normalizeUrlList(lead?.contactUrls),
   };
@@ -563,7 +599,15 @@ export function mergeLeadRecords(current, candidate) {
       ? secondary.verificationStatus
       : primary.verificationStatus,
     confidence: Math.max(primary.confidence || 0, secondary.confidence || 0),
+    linkedinUrl: primary.linkedinUrl || secondary.linkedinUrl,
     phone: primary.phone || secondary.phone,
+    address: longerString(primary.address, secondary.address),
+    streetAddress: primary.streetAddress || secondary.streetAddress,
+    location: longerString(primary.location, secondary.location),
+    city: primary.city || secondary.city,
+    region: primary.region || secondary.region,
+    postalCode: primary.postalCode || secondary.postalCode,
+    country: primary.country || secondary.country,
     socialUrls: normalizeUrlList([...(primary.socialUrls || []), ...(secondary.socialUrls || [])]),
     contactUrls: normalizeUrlList([...(primary.contactUrls || []), ...(secondary.contactUrls || [])]),
     warnings: normalizeWarnings([...(primary.warnings || []), ...(secondary.warnings || [])]),
@@ -831,6 +875,25 @@ function normalizeUrlList(value) {
   return urls;
 }
 
+function normalizeLinkedInUrl(value) {
+  let raw = stringValue(value);
+  if (!raw) return '';
+  if (!raw.includes('://')) raw = `https://${raw}`;
+
+  try {
+    const url = new URL(raw);
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') return '';
+    const host = url.hostname.toLowerCase();
+    if (host !== 'linkedin.com' && !host.endsWith('.linkedin.com')) return '';
+
+    const path = url.pathname.replace(/\/+$/, '');
+    if (!/^\/(in|pub|company|school|showcase)\//i.test(path)) return '';
+    return `https://www.linkedin.com${path}`;
+  } catch {
+    return '';
+  }
+}
+
 function normalizeDateTime(value, fallback) {
   const raw = stringValue(value);
   if (!raw) return fallback;
@@ -865,6 +928,12 @@ function compactWhitespace(value) {
   return stringValue(value).replace(/\s+/g, ' ');
 }
 
+function longerString(left, right) {
+  const first = stringValue(left);
+  const second = stringValue(right);
+  return second.length > first.length ? second : first;
+}
+
 function displayCompany(domain) {
   const clean = normalizeDomain(domain);
   if (!clean) return '';

package/modes/batch.md

@@ -25,9 +25,10 @@ id	domain	company	notes
 
 1. Run `batch/batch-runner.sh --dry-run` first unless the user explicitly asks to start immediately.
 2. Use `--parallel 2` by default. Do not exceed 2 browser-heavy workers per round.
-3. Each worker writes `batch/lead-results-{id}.json`.
-4. Validate every artifact and update `data/lead-manifest.json`.
-5. Run `npx public-leads verify`.
+3. For broad queues, keep the batch deterministic crawler timeout near the default `--timeout-ms 8000` unless target quality is known to be high.
+4. Each worker writes `batch/lead-results-{id}.json`.
+5. Validate every artifact and update `data/lead-manifest.json`.
+6. Run `npx public-leads verify`.
 
 ## Output
 

package/modes/crawl.md

@@ -7,7 +7,7 @@ Use this for one or more assigned domains.
 1. Normalize the domain to lowercase without protocol or `www.`.
 2. Visit only bounded public pages from the official site.
 3. Prefer the deterministic crawler when no browser-only blocker exists:
-   `npx public-leads crawl --input data/domains.tsv --out data/lead-results.json`
+   `npx public-leads crawl --input data/domains.tsv --out data/lead-results.json --concurrency 2 --page-concurrency 1`
 4. Extract public email addresses, person names/titles when visible nearby, contact forms, source URLs, page titles, evidence snippets, and warnings.
 5. Emit a JSON payload matching `templates/lead-schema.json`.
 6. Save the payload to `data/lead-results.json` for single-domain runs or `batch/lead-results-{id}.json` for batch workers.

package/modes/pipeline.md

@@ -15,7 +15,7 @@ Read domains from the first available source:
 1. Build a deduped candidate list.
 2. Drop domains already represented in the manifest unless the user asks to retry.
 3. Prefer the deterministic end-to-end command for normal public pages:
-   `npx public-leads pipeline --input data/domains.tsv --out data/lead-results.json`
+   `npx public-leads pipeline --input data/domains.tsv --out data/lead-results.json --concurrency 2 --page-concurrency 1`
 4. Use browser/MCP workers only when the deterministic crawler cannot reach a public page or the site requires rendering.
 5. For browser-heavy batch work, use `batch/batch-runner.sh --parallel 2`.
 6. Validate every artifact with `npx public-leads validate`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-pattern-labs/leads-rig",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Agentic public-web lead discovery harness with portable ingest artifacts",
   "type": "module",
   "author": "Razroo",

package/scripts/batch-orchestrator.mjs

@@ -7,6 +7,7 @@ import { mkdir, readFile, rm, writeFile } from 'fs/promises';
 import { dirname, join, resolve } from 'path';
 import { fileURLToPath } from 'url';
 import { runWorkflow } from '@agent-pattern-labs/iso-orchestrator';
+import { loadProfileConfig } from '../lib/leadharness-leads.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PKG_ROOT = resolve(__dirname, '..');
@@ -37,6 +38,7 @@ Options:
   --retry-failed       Only retry rows marked failed
   --start-from N       Start from numeric id N
   --max-retries N      Max failed retries per domain (default: 2)
+  --timeout-ms N       Deterministic crawler page timeout in worker prompt (default: 8000)
   --workflow-id ID     Durable workflow id (default: public-leads-batch)
   -h, --help           Show help
 
@@ -113,6 +115,7 @@ async function main(opts) {
 }
 
 function parseArgs(argv) {
+  const profile = loadProfileConfig();
   const opts = {
     runner: process.env.PUBLIC_LEADS_BATCH_RUNNER || process.env.LEAD_HARNESS_BATCH_RUNNER || 'opencode',
     parallel: 1,
@@ -121,6 +124,7 @@ function parseArgs(argv) {
     retryFailed: false,
     startFrom: 0,
     maxRetries: 2,
+    timeoutMs: positiveInt(process.env.PUBLIC_LEADS_BATCH_TIMEOUT_MS || process.env.LEAD_HARNESS_BATCH_TIMEOUT_MS || profile.batchTimeoutMs || 8_000, 'batch timeout'),
     workflowId: 'public-leads-batch',
   };
 
@@ -138,6 +142,7 @@ function parseArgs(argv) {
     else if (arg === '--retry-failed') opts.retryFailed = true;
     else if (arg === '--start-from') opts.startFrom = nonNegativeInt(next(), '--start-from');
     else if (arg === '--max-retries') opts.maxRetries = positiveInt(next(), '--max-retries');
+    else if (arg === '--timeout-ms') opts.timeoutMs = positiveInt(next(), '--timeout-ms');
     else if (arg === '--workflow-id') opts.workflowId = sanitizeWorkflowId(next());
     else if (arg === '-h' || arg === '--help') opts.help = true;
     else throw new Error(`unknown option: ${arg}`);
@@ -299,7 +304,7 @@ async function processDomain(workflow, item, opts) {
     retries,
   });
 
-  const prompt = buildWorkerPrompt(item, artifact);
+  const prompt = buildWorkerPrompt(item, artifact, opts);
   const run = await withWorkerLiveness(workflow, item, logFile, () => runWorker(opts, prompt, logFile));
   const statuses = parseStatusLines(run.output);
   const status = statuses.get(item.id);
@@ -356,12 +361,15 @@ async function processDomain(workflow, item, opts) {
   return { id: item.id, status: 'completed', artifact: status.artifact || artifact, leadCount: status.leadCount ?? 0 };
 }
 
-function buildWorkerPrompt(item, artifact) {
+function buildWorkerPrompt(item, artifact, opts) {
   return `Process this assigned public-leads domain.
 
 Assignment:
 ${JSON.stringify({ ...item, artifact }, null, 2)}
 
+Prefer the deterministic crawler first:
+npx public-leads crawl --domain ${item.domain} --out ${artifact} --timeout-ms ${opts.timeoutMs} --stop-after-good-leads 1 --stop-after-contact-path
+
 Write the artifact exactly to ${artifact}. Validate it with:
 npx public-leads validate --input ${artifact}
 

package/scripts/crawl.mjs

@@ -17,7 +17,10 @@ const USAGE = `public-leads crawl -- crawl public company pages into a lead arti
 Usage:
   public-leads crawl [--domain example.com | --domains example.com,example.org]
                      [--input data/domains.tsv] [--out data/lead-results.json]
-                     [--max-pages 10] [--min-confidence 30]
+                     [--max-pages 10] [--timeout-ms 14000] [--min-confidence 30]
+                     [--concurrency 2] [--page-concurrency 1] [--dns-concurrency 8]
+                     [--stop-after-good-leads 1] [--stop-after-contact-path]
+                     [--cache-path .leadharness-cache/crawler-cache.json] [--page-cache] [--page-cache-ttl-ms 3600000] [--no-cache]
                      [--include-blocked] [--allow-empty] [--json]
 
 When no domain or input is supplied, the first existing file is used:
@@ -44,10 +47,21 @@ async function main() {
   const payload = await crawlDomains(domains, {
     maxPages: opts.maxPages || profile.maxPages,
     minConfidence: opts.minConfidence || profile.minConfidence,
+    timeoutMs: opts.timeoutMs || profile.timeoutMs,
+    concurrency: opts.concurrency || profile.concurrency,
+    pageConcurrency: opts.pageConcurrency || profile.pageConcurrency,
+    dnsConcurrency: opts.dnsConcurrency || profile.dnsConcurrency,
     userAgent: opts.userAgent || profile.userAgent,
     includeBlocked: Boolean(opts.includeBlocked),
     delayMs: opts.delayMs,
-    timeoutMs: opts.timeoutMs,
+    cachePath: opts.cachePath || profile.cachePath,
+    noCache: Boolean(opts.noCache),
+    robotsCacheTtlMs: opts.robotsCacheTtlMs || profile.robotsCacheTtlMs,
+    dnsCacheTtlMs: opts.dnsCacheTtlMs || profile.dnsCacheTtlMs,
+    pageCache: opts.pageCache || profile.pageCache,
+    pageCacheTtlMs: opts.pageCacheTtlMs || profile.pageCacheTtlMs,
+    stopAfterGoodLeads: opts.stopAfterGoodLeads || profile.stopAfterGoodLeads,
+    stopAfterContactPath: opts.stopAfterContactPath || profile.stopAfterContactPath,
     jobId: opts.jobId,
   });
   writeJson(resolveProjectPath(out), payload);

package/scripts/pipeline.mjs

@@ -21,7 +21,10 @@ Usage:
   public-leads pipeline [--domain example.com | --domains example.com,example.org]
                         [--input data/domains.tsv] [--out data/lead-results.json]
                         [--manifest data/lead-manifest.json]
-                        [--max-pages 10] [--min-confidence 30]
+                        [--max-pages 10] [--timeout-ms 14000] [--min-confidence 30]
+                        [--concurrency 2] [--page-concurrency 1] [--dns-concurrency 8]
+                        [--stop-after-good-leads 1] [--stop-after-contact-path]
+                        [--cache-path .leadharness-cache/crawler-cache.json] [--page-cache] [--page-cache-ttl-ms 3600000] [--no-cache]
                         [--ingest | --upload] [--dry-run]
                         [--api https://cold-agent-leads.example.com]
 
@@ -47,10 +50,21 @@ try {
   const payload = await crawlDomains(domains, {
     maxPages: opts.maxPages || profile.maxPages,
     minConfidence: opts.minConfidence || profile.minConfidence,
+    timeoutMs: opts.timeoutMs || profile.timeoutMs,
+    concurrency: opts.concurrency || profile.concurrency,
+    pageConcurrency: opts.pageConcurrency || profile.pageConcurrency,
+    dnsConcurrency: opts.dnsConcurrency || profile.dnsConcurrency,
     userAgent: opts.userAgent || profile.userAgent,
     includeBlocked: Boolean(opts.includeBlocked),
     delayMs: opts.delayMs,
-    timeoutMs: opts.timeoutMs,
+    cachePath: opts.cachePath || profile.cachePath,
+    noCache: Boolean(opts.noCache),
+    robotsCacheTtlMs: opts.robotsCacheTtlMs || profile.robotsCacheTtlMs,
+    dnsCacheTtlMs: opts.dnsCacheTtlMs || profile.dnsCacheTtlMs,
+    pageCache: opts.pageCache || profile.pageCache,
+    pageCacheTtlMs: opts.pageCacheTtlMs || profile.pageCacheTtlMs,
+    stopAfterGoodLeads: opts.stopAfterGoodLeads || profile.stopAfterGoodLeads,
+    stopAfterContactPath: opts.stopAfterContactPath || profile.stopAfterContactPath,
     jobId: opts.jobId,
   });
   writeJson(resolveProjectPath(out), payload);

package/templates/contracts.json

@@ -30,7 +30,15 @@
         },
         { "name": "confidence", "type": "integer", "required": true, "min": 0, "max": 100 },
         { "name": "warnings", "type": "json" },
+        { "name": "linkedinUrl", "type": "url" },
         { "name": "phone", "type": "string" },
+        { "name": "address", "type": "string" },
+        { "name": "streetAddress", "type": "string" },
+        { "name": "location", "type": "string" },
+        { "name": "city", "type": "string" },
+        { "name": "region", "type": "string" },
+        { "name": "postalCode", "type": "string" },
+        { "name": "country", "type": "string" },
         { "name": "socialUrls", "type": "json" },
         { "name": "contactUrls", "type": "json" },
         { "name": "foundAt", "type": "datetime" }

package/templates/lead-schema.json

@@ -74,10 +74,30 @@
           "items": { "type": "string" },
           "default": []
         },
+        "linkedinUrl": {
+          "type": "string",
+          "description": "Optional canonical public LinkedIn profile or organization URL."
+        },
         "phone": {
           "type": "string",
           "description": "Optional normalized phone number found near the public contact evidence."
         },
+        "address": {
+          "type": "string",
+          "description": "Optional complete public business address."
+        },
+        "streetAddress": {
+          "type": "string",
+          "description": "Optional street-address component."
+        },
+        "location": {
+          "type": "string",
+          "description": "Optional display location for the lead."
+        },
+        "city": { "type": "string" },
+        "region": { "type": "string" },
+        "postalCode": { "type": "string" },
+        "country": { "type": "string" },
         "socialUrls": {
           "type": "array",
           "items": { "type": "string" },