lacspace-leads 0.2.1 → 1.2.1

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // src/cli.ts
-import { writeFileSync } from "fs";
+import { writeFileSync, existsSync } from "fs";
 import { resolve as resolve2 } from "path";
 import { createInterface } from "readline/promises";
 import { stdin, stdout, stderr, argv, exit } from "process";
@@ -23,13 +23,44 @@ var ALL_FIELDS = [
   "facebook",
   "instagram",
   "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram",
+  "emailStatus",
   "plusCode",
   "latitude",
   "longitude",
   "hours",
   "mapsUrl"
 ];
-var ENRICHED_FIELDS = ["email", "facebook", "instagram", "whatsapp"];
+var ENRICHED_FIELDS = [
+  "email",
+  "facebook",
+  "instagram",
+  "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram",
+  "emailStatus"
+];
+var FIELD_PRESETS = {
+  /** Just enough to place a business: name, phone, website. */
+  minimal: ["name", "phone", "website"],
+  /** Cold-outreach columns: who, how to reach them, where. */
+  outreach: ["name", "phone", "email", "website", "address"],
+  /** Contact-book columns incl. socials (implies enrichment). */
+  contact: ["name", "phone", "email", "website", "facebook", "instagram", "whatsapp"],
+  /** Map/geo columns for plotting. */
+  geo: ["name", "address", "latitude", "longitude", "plusCode", "mapsUrl"],
+  /** Everything Maps shows, no website enrichment. */
+  full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f)),
+  /** Every field, including enriched ones. */
+  everything: [...ALL_FIELDS]
+};
 var DEFAULT_FIELDS = ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f));
 
 // src/query.ts
@@ -40,8 +71,38 @@ function composeQuery(opts) {
   if (!type) throw new Error("A business `type` (or an explicit `query`) is required.");
   return where ? `${type} in ${where}` : type;
 }
-function mapsSearchUrl(query) {
-  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?hl=en`;
+function mapsSearchUrl(query, opts = {}) {
+  const hl = (opts.hl ?? "en").split("-")[0] || "en";
+  const params = new URLSearchParams({ hl });
+  if (opts.gl) params.set("gl", opts.gl.toLowerCase());
+  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?${params.toString()}`;
+}
+function expandQueries(opts) {
+  if (opts.query && opts.query.trim()) return [{ query: opts.query.trim() }];
+  const split = (s) => {
+    const parts = (s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+    return parts.length ? parts : [void 0];
+  };
+  const types = split(opts.type);
+  const cities = split(opts.city);
+  const areas = split(opts.area);
+  const out = [];
+  for (const type of types) {
+    for (const city of cities) {
+      for (const area of areas) {
+        const entry = {};
+        if (type) entry.type = type;
+        if (city) entry.city = city;
+        if (area) entry.area = area;
+        out.push(entry);
+      }
+    }
+  }
+  return out;
+}
+function resolvePreset(name) {
+  if (!name) return void 0;
+  return FIELD_PRESETS[name.trim().toLowerCase()];
 }
 function normalizeFields(input) {
   if (!input) return [...ALL_FIELDS];
@@ -82,12 +143,23 @@ function extractSocials(html) {
   const hrefs = [...html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1] ?? "");
   for (const href of hrefs) {
     const u = href.trim();
-    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?/i.test(u)) {
+    if (!u) continue;
+    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?|dialog\//i.test(u)) {
       out.facebook = absolutize(u);
-    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u)) {
+    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u) && !/\/(share|p)\//i.test(u)) {
       out.instagram = absolutize(u);
     } else if (!out.whatsapp && /(wa\.me\/|api\.whatsapp\.com\/|whatsapp:\/\/)/i.test(u)) {
       out.whatsapp = u.startsWith("http") || u.startsWith("whatsapp") ? u : `https:${u}`;
+    } else if (!out.linkedin && /(^|\/\/|\.)linkedin\.com\/(company|in|school)\//i.test(u) && !/sharing|shareArticle/i.test(u)) {
+      out.linkedin = absolutize(u);
+    } else if (!out.twitter && /(^|\/\/|\.)(twitter|x)\.com\//i.test(u) && !/(intent|share|widgets|hashtag)\b/i.test(u)) {
+      out.twitter = absolutize(u);
+    } else if (!out.youtube && /(youtube\.com\/(channel|c|user|@)|youtu\.be\/)/i.test(u)) {
+      out.youtube = absolutize(u);
+    } else if (!out.tiktok && /(^|\/\/|\.)tiktok\.com\/@/i.test(u)) {
+      out.tiktok = absolutize(u);
+    } else if (!out.telegram && /(t\.me\/|telegram\.me\/)/i.test(u) && !/\/share\b/i.test(u)) {
+      out.telegram = absolutize(u);
     }
   }
   return out;
@@ -138,11 +210,9 @@ async function enrichContacts(website, opts = {}) {
         const email = extractEmails(page);
         if (email) {
           contacts.email = email;
-          if (!contacts.facebook || !contacts.instagram || !contacts.whatsapp) {
-            const s = extractSocials(page);
-            if (!contacts.facebook && s.facebook) contacts.facebook = s.facebook;
-            if (!contacts.instagram && s.instagram) contacts.instagram = s.instagram;
-            if (!contacts.whatsapp && s.whatsapp) contacts.whatsapp = s.whatsapp;
+          const s = extractSocials(page);
+          for (const k of Object.keys(s)) {
+            if (!contacts[k] && s[k]) contacts[k] = s[k];
           }
           break;
         }
@@ -163,6 +233,7 @@ function filterLeads(leads, filters = {}) {
     if (filters.hasPhone && !l.phone) return false;
     if (filters.hasWebsite && !l.website) return false;
     if (filters.hasEmail && !l.email) return false;
+    if (filters.hasValidEmail && l.emailStatus !== "valid") return false;
     return true;
   });
 }
@@ -170,7 +241,8 @@ function dedupeKey(lead, by) {
   if (by === "none") return void 0;
   if (by === "website") return normalizeHost(lead.website);
   if (by === "phone") return lead.phone?.replace(/[^0-9]/g, "") || void 0;
-  return lead.name?.trim().toLowerCase() || void 0;
+  if (by === "name") return lead.name?.trim().toLowerCase() || void 0;
+  return normalizeHost(lead.website) ?? (lead.phone?.replace(/[^0-9]/g, "") || void 0) ?? (lead.name?.trim().toLowerCase() || void 0);
 }
 function normalizeHost(url) {
   if (!url) return void 0;
@@ -197,7 +269,300 @@ function dedupeLeads(leads, by = "website") {
   return out;
 }
 
+// src/normalize.ts
+var TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|gbraid$|wbraid$|mc_|_hs|ref$|ref_src$|source$|igshid$|yclid$)/i;
+function cleanWebsite(url) {
+  if (!url) return void 0;
+  let raw = url.trim();
+  if (!raw) return void 0;
+  const g = raw.match(/[?&](?:q|url)=([^&]+)/i);
+  if (g && /google\.[a-z.]+\/url/i.test(raw)) {
+    try {
+      raw = decodeURIComponent(g[1]);
+    } catch {
+    }
+  }
+  let u;
+  try {
+    u = new URL(raw.startsWith("http") ? raw : `https://${raw}`);
+  } catch {
+    return raw;
+  }
+  u.hash = "";
+  u.host = u.host.toLowerCase();
+  for (const key of [...u.searchParams.keys()]) {
+    if (TRACKING_PARAMS.test(key)) u.searchParams.delete(key);
+  }
+  let href = u.toString();
+  if (u.pathname === "/" && !u.search) href = href.replace(/\/$/, "");
+  return href;
+}
+var CALLING_CODES = {
+  NP: "977",
+  IN: "91",
+  US: "1",
+  CA: "1",
+  GB: "44",
+  AU: "61",
+  NZ: "64",
+  SG: "65",
+  MY: "60",
+  ID: "62",
+  TH: "66",
+  PH: "63",
+  VN: "84",
+  BD: "880",
+  PK: "92",
+  LK: "94",
+  AE: "971",
+  SA: "966",
+  QA: "974",
+  KW: "965",
+  BH: "973",
+  OM: "968",
+  CN: "86",
+  HK: "852",
+  JP: "81",
+  KR: "82",
+  DE: "49",
+  FR: "33",
+  ES: "34",
+  IT: "39",
+  NL: "31",
+  BE: "32",
+  CH: "41",
+  AT: "43",
+  SE: "46",
+  NO: "47",
+  DK: "45",
+  FI: "358",
+  IE: "353",
+  PT: "351",
+  PL: "48",
+  ZA: "27",
+  NG: "234",
+  KE: "254",
+  EG: "20",
+  BR: "55",
+  MX: "52",
+  AR: "54",
+  TR: "90"
+};
+function callingCode(country) {
+  if (!country) return void 0;
+  const c2 = country.trim().replace(/^\+/, "");
+  if (/^\d{1,4}$/.test(c2)) return c2;
+  return CALLING_CODES[c2.toUpperCase()];
+}
+function normalizePhone(raw, country) {
+  if (!raw) return void 0;
+  const trimmed = raw.trim();
+  if (!trimmed) return void 0;
+  const isIntl = /^\+/.test(trimmed) || /^00\d/.test(trimmed);
+  let digits = trimmed.replace(/[^0-9]/g, "");
+  if (/^00\d/.test(trimmed)) digits = digits.replace(/^00/, "");
+  const cc = callingCode(country);
+  if (isIntl) {
+    return plausible(digits) ? `+${digits}` : trimmed;
+  }
+  if (!cc) return trimmed;
+  if (digits.startsWith(cc)) return plausible(digits) ? `+${digits}` : trimmed;
+  const local = digits.replace(/^0+/, "");
+  const e164 = `${cc}${local}`;
+  return plausible(e164) ? `+${e164}` : trimmed;
+}
+function plausible(digits) {
+  return digits.length >= 8 && digits.length <= 15;
+}
+var priceRank = (p) => p ? p.replace(/[^$€£₹¥₩]/g, "").length : 0;
+function sortLeads(leads, by, dir) {
+  if (!by) return [...leads];
+  const desc = dir ? dir === "desc" : by !== "name";
+  const val = (l) => {
+    if (by === "name") return l.name?.toLowerCase();
+    if (by === "priceLevel") return l.priceLevel ? priceRank(l.priceLevel) : void 0;
+    return l[by];
+  };
+  return [...leads].map((lead, i) => ({ lead, i })).sort((a, b) => {
+    const av = val(a.lead);
+    const bv = val(b.lead);
+    const aMissing = av === void 0 || av === "";
+    const bMissing = bv === void 0 || bv === "";
+    if (aMissing && bMissing) return a.i - b.i;
+    if (aMissing) return 1;
+    if (bMissing) return -1;
+    let cmp = 0;
+    if (typeof av === "number" && typeof bv === "number") cmp = av - bv;
+    else cmp = String(av).localeCompare(String(bv));
+    if (desc) cmp = -cmp;
+    return cmp !== 0 ? cmp : a.i - b.i;
+  }).map((x) => x.lead);
+}
+
+// src/verify.ts
+import { resolveMx } from "dns/promises";
+var EMAIL_SHAPE = /^[^\s@]+@([^\s@.]+\.)+[^\s@.]{2,}$/;
+function emailFormatValid(email) {
+  return !!email && EMAIL_SHAPE.test(email.trim());
+}
+function emailDomain(email) {
+  if (!email) return void 0;
+  const at = email.lastIndexOf("@");
+  return at >= 0 ? email.slice(at + 1).trim().toLowerCase() || void 0 : void 0;
+}
+async function domainHasMail(domain, cache, timeoutMs) {
+  const cached = cache.get(domain);
+  if (cached !== void 0) return cached;
+  const timer = new Promise((res) => setTimeout(() => res(false), timeoutMs));
+  const lookup = resolveMx(domain).then((records) => Array.isArray(records) && records.length > 0).catch(() => false);
+  const ok = await Promise.race([lookup, timer]);
+  cache.set(domain, ok);
+  return ok;
+}
+async function verifyEmail(email, opts = {}) {
+  if (!email) return "unknown";
+  if (!emailFormatValid(email)) return "invalid-format";
+  const domain = emailDomain(email);
+  if (!domain) return "invalid-format";
+  const has = await domainHasMail(domain, opts.cache ?? /* @__PURE__ */ new Map(), opts.timeoutMs ?? 5e3);
+  return has ? "valid" : "no-mx";
+}
+async function verifyEmails(leads, opts = {}) {
+  const cache = /* @__PURE__ */ new Map();
+  const targets = leads.filter((l) => l.email);
+  const n = Math.max(1, Math.min(opts.concurrency ?? 5, targets.length || 1));
+  let next = 0;
+  let done = 0;
+  const verifyOpts = { cache };
+  if (opts.timeoutMs !== void 0) verifyOpts.timeoutMs = opts.timeoutMs;
+  const workers = Array.from({ length: n }, async () => {
+    for (; ; ) {
+      const i = next++;
+      if (i >= targets.length) return;
+      const lead = targets[i];
+      lead.emailStatus = await verifyEmail(lead.email, verifyOpts);
+      opts.onProgress?.(`verifying email ${++done}/${targets.length}: ${lead.email}\u2026`);
+    }
+  });
+  await Promise.all(workers);
+  return leads;
+}
+
+// src/export.ts
+import { stringify as csvStringify } from "@lacspace/csv";
+import { jsonToXlsx } from "@lacspace/xlsx";
+var HEADERS = {
+  name: "Name",
+  category: "Category",
+  rating: "Rating",
+  reviews: "Reviews",
+  priceLevel: "Price",
+  address: "Address",
+  phone: "Phone",
+  website: "Website",
+  email: "Email",
+  facebook: "Facebook",
+  instagram: "Instagram",
+  whatsapp: "WhatsApp",
+  linkedin: "LinkedIn",
+  twitter: "Twitter/X",
+  youtube: "YouTube",
+  tiktok: "TikTok",
+  telegram: "Telegram",
+  emailStatus: "Email Status",
+  plusCode: "Plus Code",
+  latitude: "Latitude",
+  longitude: "Longitude",
+  hours: "Hours",
+  mapsUrl: "Maps URL"
+};
+function toRows(leads, fields = ALL_FIELDS) {
+  return leads.map((lead) => {
+    const row = {};
+    for (const f of fields) {
+      const v = lead[f];
+      row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
+    }
+    return row;
+  });
+}
+function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
+  if (format === "json" || format === "ndjson") {
+    const picked = leads.map((lead) => {
+      const o = {};
+      for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
+      return o;
+    });
+    const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
+    return { data, binary: false };
+  }
+  const rows = toRows(leads, fields);
+  if (format === "csv") {
+    const csvRows = rows;
+    return { data: csvStringify(csvRows, { escapeFormulas: true }), binary: false };
+  }
+  const columns = fields.map((f) => {
+    const header = HEADERS[f];
+    const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
+    return { header, width: Math.min(60, Math.max(8, widest + 2)) };
+  });
+  return {
+    data: jsonToXlsx(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
+    binary: true
+  };
+}
+function computeStats(leads) {
+  const socialKeys = ["facebook", "instagram", "whatsapp", "linkedin", "twitter", "youtube", "tiktok", "telegram"];
+  const ratings = leads.map((l) => l.rating).filter((r) => typeof r === "number");
+  const stats = {
+    total: leads.length,
+    withPhone: leads.filter((l) => l.phone).length,
+    withWebsite: leads.filter((l) => l.website).length,
+    withEmail: leads.filter((l) => l.email).length,
+    withValidEmail: leads.filter((l) => l.emailStatus === "valid").length,
+    withSocial: leads.filter((l) => socialKeys.some((k) => l[k])).length
+  };
+  if (ratings.length) stats.avgRating = Math.round(ratings.reduce((a, b) => a + b, 0) / ratings.length * 10) / 10;
+  return stats;
+}
+var FIELD_BY_KEY = (() => {
+  const m = /* @__PURE__ */ new Map();
+  for (const f of ALL_FIELDS) {
+    m.set(f.toLowerCase(), f);
+    m.set(HEADERS[f].toLowerCase(), f);
+  }
+  return m;
+})();
+var NUMERIC_FIELDS = /* @__PURE__ */ new Set(["rating", "reviews", "latitude", "longitude"]);
+function rowsToLeads(rows) {
+  return rows.map((row) => {
+    const lead = {};
+    for (const [key, raw] of Object.entries(row)) {
+      const field = FIELD_BY_KEY.get(key.trim().toLowerCase());
+      if (!field || raw === null || raw === void 0 || raw === "") continue;
+      if (NUMERIC_FIELDS.has(field)) {
+        const n = typeof raw === "number" ? raw : parseFloat(String(raw));
+        if (Number.isFinite(n)) lead[field] = n;
+      } else {
+        lead[field] = String(raw);
+      }
+    }
+    return lead;
+  });
+}
+
 // src/scrape.ts
+async function pool(items, n, fn, signal) {
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => {
+    for (; ; ) {
+      const i = next++;
+      if (i >= items.length || signal?.aborted) return;
+      await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+}
 var LeadsError = class extends Error {
   code;
   cause;
@@ -235,17 +600,19 @@ function parseLatLng(url) {
   if (at) return { latitude: parseFloat(at[1]), longitude: parseFloat(at[2]) };
   return {};
 }
-async function launchBrowser(headless) {
+async function launchBrowser(headless, proxy) {
   let lastErr;
+  const base = { headless };
+  if (proxy) base.proxy = { server: proxy };
   for (const channel of ["chrome", "msedge"]) {
     try {
-      return await chromium.launch({ headless, channel });
+      return await chromium.launch({ ...base, channel });
     } catch (e) {
       lastErr = e;
     }
   }
   try {
-    return await chromium.launch({ headless });
+    return await chromium.launch(base);
   } catch (e) {
     lastErr = e;
   }
@@ -322,7 +689,13 @@ async function extractDetail(page, fields, fallbackName) {
       }
       if (fields.has("reviews")) {
         const rv = await box.locator('span[aria-label*="review"]').first().getAttribute("aria-label").catch(() => null);
-        lead.reviews = parseReviewCount(rv);
+        let count = parseReviewCount(rv);
+        if (count === void 0) {
+          const boxText = await box.innerText().catch(() => "");
+          const m = boxText.match(/\(([\d.,\s]+)\)/);
+          if (m) count = parseReviewCount(m[1]);
+        }
+        lead.reviews = count;
       }
     }
   }
@@ -362,24 +735,31 @@ async function scrapeLeads(opts) {
   const fields = new Set(normalizeFields(opts.fields ?? DEFAULT_FIELDS));
   const delayMs = Math.max(0, Math.trunc(opts.delayMs ?? 700));
   const headless = opts.headless ?? false;
+  const retries = Math.max(0, Math.trunc(opts.retries ?? 1));
   const onProgress = opts.onProgress;
+  const onLead = opts.onLead;
   const signal = opts.signal;
   const startedAt = Date.now();
   const overBudget = () => opts.maxMs !== void 0 && Date.now() - startedAt > opts.maxMs;
-  const wantEnrich = Boolean(opts.enrich) || ENRICHED_FIELDS.some((f) => fields.has(f)) || Boolean(opts.filters?.hasEmail);
+  const pauseMs = () => opts.jitter ? Math.round(delayMs * (0.6 + Math.random() * 0.8)) : delayMs;
+  const wantVerify = Boolean(opts.verifyEmails) || Boolean(opts.filters?.hasValidEmail) || fields.has("emailStatus");
+  const wantEnrich = Boolean(opts.enrich) || ENRICHED_FIELDS.some((f) => fields.has(f)) || Boolean(opts.filters?.hasEmail) || wantVerify;
   const collect = new Set(fields);
   if (wantEnrich) collect.add("website");
   const detailOnly = ["name", "mapsUrl"];
   const wantDetails = (opts.details ?? true) && ([...collect].some((f) => !detailOnly.includes(f)) || wantEnrich);
   onProgress?.(`searching Google Maps for "${query}"\u2026`);
-  const browser = await launchBrowser(headless);
+  const browser = await launchBrowser(headless, opts.proxy);
   try {
+    const locale = opts.locale ?? "en-US";
     const context = await browser.newContext({
       viewport: { width: 1280, height: 900 },
-      locale: "en-US"
+      locale
     });
     const page = await context.newPage();
-    await page.goto(mapsSearchUrl(query), { waitUntil: "domcontentloaded", timeout: 45e3 });
+    const urlOpts = { hl: locale };
+    if (opts.region) urlOpts.gl = opts.region;
+    await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
     await dismissConsent(page);
     await loadResults(page, limit, delayMs, onProgress, signal);
     if (signal?.aborted) throw new LeadsError("Search aborted.", "ABORTED");
@@ -407,44 +787,95 @@ async function scrapeLeads(opts) {
         }
         return lead;
       });
+      for (const lead of leads) {
+        try {
+          onLead?.(lead);
+        } catch {
+        }
+      }
     } else {
       for (let i = 0; i < cards.length; i++) {
         if (signal?.aborted || overBudget()) break;
         const card = cards[i];
         onProgress?.(`reading ${i + 1}/${cards.length}: ${card.name ?? "listing"}\u2026`);
-        try {
-          await page.goto(card.href, { waitUntil: "domcontentloaded", timeout: 3e4 });
-          await page.locator("h1.DUwDvf").first().waitFor({ timeout: 8e3 }).catch(() => {
-          });
-          leads.push(await extractDetail(page, collect, card.name));
-        } catch {
+        let lead;
+        for (let attempt = 0; attempt <= retries; attempt++) {
+          try {
+            await page.goto(card.href, { waitUntil: "domcontentloaded", timeout: 3e4 });
+            await page.locator("h1.DUwDvf").first().waitFor({ timeout: 8e3 }).catch(() => {
+            });
+            lead = await extractDetail(page, collect, card.name);
+            break;
+          } catch {
+            if (attempt < retries && !signal?.aborted) {
+              onProgress?.(`  retrying ${i + 1}/${cards.length} (attempt ${attempt + 2})\u2026`);
+              if (delayMs) await page.waitForTimeout(pauseMs());
+            }
+          }
+        }
+        if (!lead) {
           if (card.name && (collect.has("name") || collect.has("mapsUrl"))) {
-            const partial = {};
-            if (collect.has("name")) partial.name = card.name;
-            if (collect.has("mapsUrl")) partial.mapsUrl = card.href;
-            leads.push(partial);
+            lead = {};
+            if (collect.has("name")) lead.name = card.name;
+            if (collect.has("mapsUrl")) lead.mapsUrl = card.href;
           }
         }
-        if (delayMs) await page.waitForTimeout(delayMs);
+        if (lead) {
+          leads.push(lead);
+          try {
+            onLead?.(lead);
+          } catch {
+          }
+        }
+        if (delayMs) await page.waitForTimeout(pauseMs());
       }
     }
     onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
+    if (opts.cleanUrls ?? true) {
+      for (const lead of leads) {
+        if (lead.website) lead.website = cleanWebsite(lead.website);
+      }
+    }
     leads = dedupeLeads(leads, opts.dedupe ?? "website");
     if (wantEnrich) {
-      for (let i = 0; i < leads.length; i++) {
-        if (signal?.aborted || overBudget()) break;
-        const site = leads[i].website;
-        if (!site) continue;
-        onProgress?.(`enriching ${i + 1}/${leads.length}: ${leads[i].name ?? site}\u2026`);
-        const c2 = await enrichContacts(site);
-        if (fields.has("email") && c2.email) leads[i].email = c2.email;
-        if (fields.has("facebook") && c2.facebook) leads[i].facebook = c2.facebook;
-        if (fields.has("instagram") && c2.instagram) leads[i].instagram = c2.instagram;
-        if (fields.has("whatsapp") && c2.whatsapp) leads[i].whatsapp = c2.whatsapp;
-        if (!fields.has("website")) delete leads[i].website;
+      const socialFields = ENRICHED_FIELDS.filter((f) => f !== "email");
+      let done = 0;
+      await pool(
+        leads,
+        Math.max(1, Math.trunc(opts.concurrency ?? 3)),
+        async (lead) => {
+          if (overBudget()) return;
+          const site = lead.website;
+          if (site) {
+            onProgress?.(`enriching ${++done}/${leads.length}: ${lead.name ?? site}\u2026`);
+            const c2 = await enrichContacts(site);
+            if (fields.has("email") && c2.email) lead.email = c2.email;
+            for (const f of socialFields) {
+              const v = c2[f];
+              if (fields.has(f) && v) lead[f] = v;
+            }
+          }
+          if (!fields.has("website")) delete lead.website;
+        },
+        signal
+      );
+    }
+    if (wantVerify && !signal?.aborted) {
+      onProgress?.("verifying email domains\u2026");
+      const vOpts = {
+        concurrency: Math.max(1, Math.trunc(opts.concurrency ?? 5))
+      };
+      if (onProgress) vOpts.onProgress = (m) => onProgress(m);
+      await verifyEmails(leads, vOpts);
+    }
+    if (opts.country && fields.has("phone")) {
+      for (const lead of leads) {
+        if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
       }
     }
     if (opts.filters) leads = filterLeads(leads, opts.filters);
+    if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
+    if (wantVerify && !fields.has("emailStatus")) for (const l of leads) delete l.emailStatus;
     onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
     return leads;
   } finally {
@@ -453,61 +884,58 @@ async function scrapeLeads(opts) {
   }
 }
 
-// src/export.ts
-import { stringify as csvStringify } from "@lacspace/csv";
-import { jsonToXlsx } from "@lacspace/xlsx";
-var HEADERS = {
-  name: "Name",
-  category: "Category",
-  rating: "Rating",
-  reviews: "Reviews",
-  priceLevel: "Price",
-  address: "Address",
-  phone: "Phone",
-  website: "Website",
-  email: "Email",
-  facebook: "Facebook",
-  instagram: "Instagram",
-  whatsapp: "WhatsApp",
-  plusCode: "Plus Code",
-  latitude: "Latitude",
-  longitude: "Longitude",
-  hours: "Hours",
-  mapsUrl: "Maps URL"
-};
-function toRows(leads, fields = ALL_FIELDS) {
-  return leads.map((lead) => {
-    const row = {};
-    for (const f of fields) {
-      const v = lead[f];
-      row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
+// src/batch.ts
+async function searchLeadsBatch(queries, opts = {}) {
+  const {
+    total,
+    onProgress,
+    signal,
+    type: _t,
+    city: _c,
+    area: _a,
+    query: _q,
+    ...shared
+  } = opts;
+  void _t;
+  void _c;
+  void _a;
+  void _q;
+  let merged = [];
+  const seenDedupe = shared.dedupe ?? "website";
+  for (let i = 0; i < queries.length; i++) {
+    if (signal?.aborted) break;
+    const q = queries[i];
+    const label = (() => {
+      try {
+        return composeQuery(q);
+      } catch {
+        return q.query ?? q.type ?? "search";
+      }
+    })();
+    onProgress?.(`[${i + 1}/${queries.length}] ${label}`);
+    const subOpts = {
+      ...shared,
+      type: q.type ?? "",
+      // Defer sorting to the merged pass; keep per-search dedupe/filter local.
+      ...q.query ? { query: q.query } : {},
+      ...q.city ? { city: q.city } : {},
+      ...q.area ? { area: q.area } : {}
+    };
+    delete subOpts.sort;
+    if (signal) subOpts.signal = signal;
+    if (onProgress) subOpts.onProgress = (m) => onProgress(`  ${m}`);
+    try {
+      const found = await scrapeLeads(subOpts);
+      merged = merged.concat(found);
+    } catch (err) {
+      onProgress?.(`  ! skipped "${label}": ${err.message}`);
     }
-    return row;
-  });
-}
-function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
-  if (format === "json") {
-    const picked = leads.map((lead) => {
-      const o = {};
-      for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
-      return o;
-    });
-    return { data: JSON.stringify(picked, null, 2), binary: false };
   }
-  const rows = toRows(leads, fields);
-  if (format === "csv") {
-    const csvRows = rows;
-    return { data: csvStringify(csvRows, { escapeFormulas: true }), binary: false };
-  }
-  const columns = fields.map((f) => {
-    const header = HEADERS[f];
-    const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
-    return { header, width: Math.min(60, Math.max(8, widest + 2)) };
-  });
-  return {
-    data: jsonToXlsx(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
-    binary: true
-  };
+  merged = dedupeLeads(merged, seenDedupe);
+  if (opts.sort) merged = sortLeads(merged, opts.sort, opts.sortDir);
+  if (total !== void 0 && total > 0) merged = merged.slice(0, total);
+  onProgress?.(`merged ${merged.length} unique lead${merged.length === 1 ? "" : "s"}.`);
+  return merged;
 }
 
 // src/convert.ts
@@ -585,6 +1013,8 @@ async function convertFile(input, opts = {}) {
 }
 
 // src/cli.ts
+var SOCIAL_FIELDS = ENRICHED_FIELDS.filter((f) => f !== "email");
+var OUTPUT_FORMATS = ["json", "ndjson", "csv", "xlsx"];
 var C = {
   reset: "\x1B[0m",
   bold: "\x1B[1m",
@@ -600,29 +1030,37 @@ var log = (s = "") => void stderr.write(s + "\n");
 function parseArgs(list) {
   const a = {
     format: "json",
+    append: false,
     limit: 60,
     headless: false,
     details: true,
     delay: 700,
     emails: false,
     socials: false,
+    verifyEmails: false,
     hasPhone: false,
     hasWebsite: false,
     hasEmail: false,
+    hasValidEmail: false,
+    cleanUrls: true,
+    jitter: false,
     yes: false,
     help: false
   };
   for (let i = 0; i < list.length; i++) {
     const arg = list[i];
     const next = () => list[++i] ?? "";
-    if (arg === "--city") a.city = next();
-    else if (arg === "--area") a.area = next();
-    else if (arg === "-t" || arg === "--type") a.type = next();
+    if (arg === "--city" || arg === "--cities") a.city = next();
+    else if (arg === "--area" || arg === "--areas") a.area = next();
+    else if (arg === "-t" || arg === "--type" || arg === "--types") a.type = next();
     else if (arg === "-q" || arg === "--query") a.query = next();
     else if (arg === "--fields") a.fields = next();
+    else if (arg === "--preset") a.preset = next();
     else if (arg === "-f" || arg === "--format") a.format = next();
     else if (arg === "-o" || arg === "--out") a.out = next();
+    else if (arg === "--append") a.append = true;
     else if (arg === "-n" || arg === "--limit") a.limit = parseInt(next(), 10) || a.limit;
+    else if (arg === "--total") a.total = parseInt(next(), 10) || a.total;
     else if (arg === "--headless") a.headless = true;
     else if (arg === "--no-details") a.details = false;
     else if (arg === "--delay") a.delay = parseInt(next(), 10) || a.delay;
@@ -631,14 +1069,32 @@ function parseArgs(list) {
     else if (arg === "--enrich") {
       a.emails = true;
       a.socials = true;
-    } else if (arg === "--min-rating") a.minRating = parseFloat(next());
+    } else if (arg === "--verify-emails" || arg === "--verify") {
+      a.verifyEmails = true;
+      a.emails = true;
+    } else if (arg === "--proxy") a.proxy = next();
+    else if (arg === "--retries") a.retries = parseInt(next(), 10);
+    else if (arg === "--jitter") a.jitter = true;
+    else if (arg === "--min-rating") a.minRating = parseFloat(next());
     else if (arg === "--min-reviews") a.minReviews = parseInt(next(), 10);
     else if (arg === "--has-phone") a.hasPhone = true;
     else if (arg === "--has-website") a.hasWebsite = true;
     else if (arg === "--has-email") {
       a.hasEmail = true;
       a.emails = true;
+    } else if (arg === "--has-valid-email") {
+      a.hasValidEmail = true;
+      a.verifyEmails = true;
+      a.emails = true;
     } else if (arg === "--dedupe") a.dedupe = next();
+    else if (arg === "--sort") a.sort = next();
+    else if (arg === "--desc") a.desc = true;
+    else if (arg === "--asc") a.desc = false;
+    else if (arg === "--country") a.country = next();
+    else if (arg === "--lang" || arg === "--locale") a.locale = next();
+    else if (arg === "--region" || arg === "--gl") a.region = next();
+    else if (arg === "--concurrency") a.concurrency = parseInt(next(), 10) || a.concurrency;
+    else if (arg === "--no-clean-urls") a.cleanUrls = false;
     else if (arg === "--sheet") a.sheet = next();
     else if (arg === "--max-time") a.maxTime = parseInt(next(), 10);
     else if (arg === "-y" || arg === "--yes") a.yes = true;
@@ -656,35 +1112,57 @@ ${c("bold", "Usage")}
   npx lacspace-leads convert <file> [-f json|csv|xlsx] [-o out]
 
 ${c("bold", "Search options")}
-  -t, --type <text>     Business type / keyword, e.g. "restaurants"
-      --city <text>     City, e.g. "Kathmandu"
-      --area <text>     Area / neighbourhood, e.g. "Baneshwor"
+  -t, --type <text>     Business type/keyword. Comma-separate for several,
+                        e.g. "restaurants,cafes"
+      --city <text>     City. Comma-separate for several.
+      --area <text>     Area/neighbourhood. Comma-separate to sweep a whole
+                        city, e.g. --area "Baneshwor,Thamel,Patan"
   -q, --query <text>    Raw query verbatim (overrides city/area/type)
       --fields <list>   Columns: ${ALL_FIELDS.join(",")}
-  -n, --limit <n>       Max listings to collect   (default 60)
+      --preset <name>   Field bundle: ${Object.keys(FIELD_PRESETS).join(" | ")}
+  -n, --limit <n>       Max listings per search   (default 60)
+      --total <n>       Cap the merged result (batch searches)
       --no-details      Names + Maps URLs only (fast, no per-listing open)
 
 ${c("bold", "Enrichment (visits each website)")}
       --emails          Also find an email from each website
-      --socials         Also find Facebook / Instagram / WhatsApp
+      --socials         Also find Facebook/Instagram/WhatsApp/LinkedIn/X/
+                        YouTube/TikTok/Telegram
       --enrich          Both of the above
+      --verify-emails   Check each email's domain has MX records (implies --emails)
+      --concurrency <n> Websites to enrich in parallel   (default 3)
+
+${c("bold", "Clean-up")}
+      --country <c>     Normalise phones to E.164 for this country
+                        (ISO-2 like NP/US, or a calling code like 977)
+      --no-clean-urls   Don't tidy website URLs (redirects/tracking params)
 
-${c("bold", "Filters")}
+${c("bold", "Filters & order")}
       --min-rating <n>  Keep only ratings \u2265 n
       --min-reviews <n> Keep only \u2265 n reviews
       --has-phone       Keep only leads with a phone
       --has-website     Keep only leads with a website
       --has-email       Keep only leads with an email (implies --emails)
-      --dedupe <key>    website | phone | name | none   (default website)
+      --has-valid-email Keep only leads with an MX-verified email (implies --verify-emails)
+      --dedupe <key>    website | phone | name | smart | none   (default website;
+                        --append uses smart: website\u2192phone\u2192name)
+      --sort <key>      rating | reviews | name | priceLevel
+      --desc / --asc    Sort direction
 
 ${c("bold", "Output")}
-  -f, --format <fmt>    json | csv | xlsx        (default json)
-  -o, --out <file>      Output file (default: a slug + date)
+  -f, --format <fmt>    json | ndjson | csv | xlsx      (default json)
+  -o, --out <file>      Output file, or "-" for stdout (default: slug + date)
+      --append          Merge into an existing output file (accumulate + dedupe)
       --sheet <name>    Excel sheet name         (default "Leads")
 
 ${c("bold", "Runtime")}
       --delay <ms>      Pause between listings    (default 700)
+      --jitter          Randomise the delay \xB140% (more human)
+      --retries <n>     Retry a listing that fails to open   (default 1)
       --max-time <s>    Stop collecting after n seconds
+      --proxy <url>     Route the browser via a proxy (http://user:pass@host:port)
+      --lang <locale>   Browser locale, e.g. en-US, ne-NP   (default en-US)
+      --region <cc>     Region bias for results, e.g. np, us
       --headless        Run the browser without a window
   -y, --yes             Skip prompts + the browser-open confirmation
   -h, --help            Show this help
@@ -692,7 +1170,11 @@ ${c("bold", "Runtime")}
 ${c("bold", "Examples")}
   npx lacspace-leads restaurants --city Kathmandu --area Baneshwor -f xlsx
   npx lacspace-leads "dental clinic" --city Pokhara --emails --has-email -f csv -n 40
-  npx lacspace-leads gyms --city Lalitpur --min-rating 4 --no-details -n 100
+  npx lacspace-leads gyms --city Lalitpur --min-rating 4 --sort reviews --desc
+  npx lacspace-leads cafes --city Kathmandu --area "Thamel,Baneshwor,Patan" --country NP
+  npx lacspace-leads salons --city Pokhara --preset outreach --country NP -f csv -o -
+  npx lacspace-leads dentists --city Pokhara --verify-emails --has-valid-email -f csv
+  npx lacspace-leads cafes --city Kathmandu -o master.csv --append   # accumulate daily
   npx lacspace-leads convert leads.json -f xlsx
 
 ${c("dim", "Please scrape responsibly: keep volumes small, respect Google's Terms of")}
@@ -767,29 +1249,62 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
     exit(1);
     return;
   }
-  const base = args.fields ? normalizeFields(args.fields) : [...DEFAULT_FIELDS];
+  if (!OUTPUT_FORMATS.includes(args.format)) {
+    log(c("red", `
+\u2717 Unknown format "${args.format}". Use: ${OUTPUT_FORMATS.join(", ")}.`));
+    exit(1);
+    return;
+  }
+  if (args.preset && !resolvePreset(args.preset)) {
+    log(c("red", `
+\u2717 Unknown preset "${args.preset}". Use: ${Object.keys(FIELD_PRESETS).join(", ")}.`));
+    exit(1);
+    return;
+  }
+  const base = args.fields ? normalizeFields(args.fields) : resolvePreset(args.preset) ?? [...DEFAULT_FIELDS];
   const wanted = new Set(base);
   if (args.emails) wanted.add("email");
-  if (args.socials) {
-    wanted.add("facebook");
-    wanted.add("instagram");
-    wanted.add("whatsapp");
+  if (args.socials) for (const f of SOCIAL_FIELDS) wanted.add(f);
+  if (args.verifyEmails) {
+    wanted.add("email");
+    wanted.add("emailStatus");
   }
   const fields = ALL_FIELDS.filter((f) => wanted.has(f));
-  const out = resolve2(args.out ?? defaultFilename(query, args.format));
+  const toStdout = args.out === "-";
+  const out = toStdout ? "-" : resolve2(args.out ?? defaultFilename(query, args.format));
   const filters = {};
   if (args.minRating !== void 0) filters.minRating = args.minRating;
   if (args.minReviews !== void 0) filters.minReviews = args.minReviews;
   if (args.hasPhone) filters.hasPhone = true;
   if (args.hasWebsite) filters.hasWebsite = true;
   if (args.hasEmail) filters.hasEmail = true;
+  if (args.hasValidEmail) filters.hasValidEmail = true;
   const hasFilters = Object.keys(filters).length > 0;
-  log(`
+  const queries = expandQueries({
+    type: args.type ?? "",
+    city: args.city ?? "",
+    area: args.area ?? "",
+    query: args.query ?? ""
+  });
+  const isBatch = queries.length > 1;
+  if (isBatch) log(`
+  ${c("dim", "searches")} ${c("bold", String(queries.length))} ${c("dim", "\u2192")} ${queries.map((q) => {
+    try {
+      return composeQuery(q);
+    } catch {
+      return "?";
+    }
+  }).join("  \xB7  ")}`);
+  else log(`
   ${c("dim", "search")}  ${c("bold", query)}`);
   log(`  ${c("dim", "fields")}  ${fields.join(", ")}`);
-  if (args.emails || args.socials) log(`  ${c("dim", "enrich")}  ${[args.emails && "emails", args.socials && "socials"].filter(Boolean).join(" + ")} ${c("dim", "(visits each website)")}`);
+  if (args.emails || args.socials) log(`  ${c("dim", "enrich")}  ${[args.emails && "emails", args.socials && "socials"].filter(Boolean).join(" + ")} ${c("dim", `(${args.concurrency ?? 3}\xD7 parallel, visits each website)`)}`);
+  if (args.verifyEmails) log(`  ${c("dim", "verify")}  email domains (MX lookup)`);
   if (hasFilters) log(`  ${c("dim", "filters")} ${Object.entries(filters).map(([k, v]) => `${k}=${v}`).join(", ")}`);
-  log(`  ${c("dim", "limit")}   ${args.limit}   ${c("dim", "format")} ${args.format}   ${c("dim", "\u2192")} ${out}`);
+  if (args.sort) log(`  ${c("dim", "sort")}    ${args.sort} ${args.desc === false ? "asc" : "desc"}`);
+  if (args.country) log(`  ${c("dim", "phones")}  E.164 for ${args.country}`);
+  if (args.proxy) log(`  ${c("dim", "proxy")}   ${args.proxy.replace(/\/\/[^@]+@/, "//***@")}`);
+  log(`  ${c("dim", "limit")}   ${args.limit}${isBatch ? "/search" : ""}${args.total ? ` (cap ${args.total})` : ""}   ${c("dim", "format")} ${args.format}   ${c("dim", "\u2192")} ${toStdout ? "stdout" : out}${args.append ? c("dim", " (append)") : ""}`);
   if (!args.yes) {
     const ok = await prompt(`
 ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c("dim", "[y/N]")} `);
@@ -814,16 +1329,33 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
     headless: args.headless,
     details: args.details,
     delayMs: args.delay,
+    cleanUrls: args.cleanUrls,
     enrich: args.emails || args.socials,
+    verifyEmails: args.verifyEmails,
+    jitter: args.jitter,
     signal: controller.signal,
     onProgress: (m) => log(`  ${c("cyan", "\u25F7")} ${c("dim", m)}`)
   };
   if (hasFilters) opts.filters = filters;
   if (args.dedupe) opts.dedupe = args.dedupe;
+  if (args.sort) opts.sort = args.sort;
+  if (args.desc !== void 0) opts.sortDir = args.desc ? "desc" : "asc";
+  if (args.country) opts.country = args.country;
+  if (args.locale) opts.locale = args.locale;
+  if (args.region) opts.region = args.region;
+  if (args.concurrency !== void 0) opts.concurrency = args.concurrency;
+  if (args.proxy) opts.proxy = args.proxy;
+  if (args.retries !== void 0) opts.retries = args.retries;
   if (args.maxTime !== void 0) opts.maxMs = args.maxTime * 1e3;
   let leads;
   try {
-    leads = await scrapeLeads(opts);
+    if (isBatch) {
+      const batchOpts = { ...opts };
+      if (args.total !== void 0) batchOpts.total = args.total;
+      leads = await searchLeadsBatch(queries, batchOpts);
+    } else {
+      leads = await scrapeLeads(opts);
+    }
   } catch (err) {
     log(c("red", `
 \u2717 ${err.message}`));
@@ -836,18 +1368,40 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
     log(c("yellow", "\n  No leads collected. Try a broader area, looser filters, or a smaller --limit.\n"));
     return;
   }
+  let fresh = leads.length;
+  if (args.append && !toStdout && existsSync(out)) {
+    try {
+      const existing = rowsToLeads(await readRows(out));
+      const before = existing.length;
+      const merged = dedupeLeads([...existing, ...leads], args.dedupe ?? "smart");
+      fresh = merged.length - before;
+      log(`  ${c("cyan", "\u25F7")} ${c("dim", `merged with ${before} existing \u2192 ${merged.length} total (${fresh} new)`)}`);
+      leads = merged;
+    } catch (err) {
+      log(c("yellow", `  ! couldn't read "${out}" to append (${err.message}); overwriting instead`));
+    }
+  }
   const serOpts = {};
   if (args.sheet) serOpts.sheetName = args.sheet;
   const { data, binary } = serialize(leads, args.format, fields, serOpts);
+  if (toStdout) {
+    stdout.write(binary ? Buffer.from(data) : data);
+    if (!binary) stdout.write("\n");
+    log(`
+  ${c("green", "\u2714")} ${c("bold", String(leads.length))} leads written to stdout ${c("dim", `(${args.format})`)}
+`);
+    return;
+  }
   writeFileSync(out, binary ? Buffer.from(data) : data);
-  const withPhone = leads.filter((l) => l.phone).length;
-  const withSite = leads.filter((l) => l.website).length;
-  const withEmail = leads.filter((l) => l.email).length;
+  const s = computeStats(leads);
   log(`
-  ${c("green", "\u2714")} Saved ${c("bold", String(leads.length))} leads \u2192 ${c("cyan", out)}`);
-  const stats = [`${withPhone} with a phone`, `${withSite} with a website`];
-  if (args.emails) stats.push(`${withEmail} with an email`);
-  log(`    ${c("dim", stats.join(" \xB7 "))}
+  ${c("green", "\u2714")} Saved ${c("bold", String(leads.length))} leads \u2192 ${c("cyan", out)}${args.append && fresh !== leads.length ? c("dim", ` (${fresh} new)`) : ""}`);
+  const parts = [`${s.withPhone} with a phone`, `${s.withWebsite} with a website`];
+  if (args.emails) parts.push(`${s.withEmail} with an email`);
+  if (args.verifyEmails) parts.push(`${s.withValidEmail} MX-valid`);
+  if (args.socials) parts.push(`${s.withSocial} with a social link`);
+  if (s.avgRating !== void 0) parts.push(`avg \u2605 ${s.avgRating}`);
+  log(`    ${c("dim", parts.join(" \xB7 "))}
 `);
 }
 main().catch((err) => {