lacspace-leads 0.2.0 → 0.3.0

package/dist/lib.js

@@ -15,13 +15,42 @@ var ALL_FIELDS = [
   "facebook",
   "instagram",
   "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram",
   "plusCode",
   "latitude",
   "longitude",
   "hours",
   "mapsUrl"
 ];
-var ENRICHED_FIELDS = ["email", "facebook", "instagram", "whatsapp"];
+var ENRICHED_FIELDS = [
+  "email",
+  "facebook",
+  "instagram",
+  "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram"
+];
+var FIELD_PRESETS = {
+  /** Just enough to place a business: name, phone, website. */
+  minimal: ["name", "phone", "website"],
+  /** Cold-outreach columns: who, how to reach them, where. */
+  outreach: ["name", "phone", "email", "website", "address"],
+  /** Contact-book columns incl. socials (implies enrichment). */
+  contact: ["name", "phone", "email", "website", "facebook", "instagram", "whatsapp"],
+  /** Map/geo columns for plotting. */
+  geo: ["name", "address", "latitude", "longitude", "plusCode", "mapsUrl"],
+  /** Everything Maps shows, no website enrichment. */
+  full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f)),
+  /** Every field, including enriched ones. */
+  everything: [...ALL_FIELDS]
+};
 var DEFAULT_FIELDS = ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f));
 
 // src/query.ts
@@ -32,8 +61,38 @@ function composeQuery(opts) {
   if (!type) throw new Error("A business `type` (or an explicit `query`) is required.");
   return where ? `${type} in ${where}` : type;
 }
-function mapsSearchUrl(query) {
-  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?hl=en`;
+function mapsSearchUrl(query, opts = {}) {
+  const hl = (opts.hl ?? "en").split("-")[0] || "en";
+  const params = new URLSearchParams({ hl });
+  if (opts.gl) params.set("gl", opts.gl.toLowerCase());
+  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?${params.toString()}`;
+}
+function expandQueries(opts) {
+  if (opts.query && opts.query.trim()) return [{ query: opts.query.trim() }];
+  const split = (s) => {
+    const parts = (s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+    return parts.length ? parts : [void 0];
+  };
+  const types = split(opts.type);
+  const cities = split(opts.city);
+  const areas = split(opts.area);
+  const out = [];
+  for (const type of types) {
+    for (const city of cities) {
+      for (const area of areas) {
+        const entry = {};
+        if (type) entry.type = type;
+        if (city) entry.city = city;
+        if (area) entry.area = area;
+        out.push(entry);
+      }
+    }
+  }
+  return out;
+}
+function resolvePreset(name) {
+  if (!name) return void 0;
+  return FIELD_PRESETS[name.trim().toLowerCase()];
 }
 function normalizeFields(input) {
   if (!input) return [...ALL_FIELDS];
@@ -74,12 +133,23 @@ function extractSocials(html) {
   const hrefs = [...html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1] ?? "");
   for (const href of hrefs) {
     const u = href.trim();
-    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?/i.test(u)) {
+    if (!u) continue;
+    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?|dialog\//i.test(u)) {
       out.facebook = absolutize(u);
-    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u)) {
+    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u) && !/\/(share|p)\//i.test(u)) {
       out.instagram = absolutize(u);
     } else if (!out.whatsapp && /(wa\.me\/|api\.whatsapp\.com\/|whatsapp:\/\/)/i.test(u)) {
       out.whatsapp = u.startsWith("http") || u.startsWith("whatsapp") ? u : `https:${u}`;
+    } else if (!out.linkedin && /(^|\/\/|\.)linkedin\.com\/(company|in|school)\//i.test(u) && !/sharing|shareArticle/i.test(u)) {
+      out.linkedin = absolutize(u);
+    } else if (!out.twitter && /(^|\/\/|\.)(twitter|x)\.com\//i.test(u) && !/(intent|share|widgets|hashtag)\b/i.test(u)) {
+      out.twitter = absolutize(u);
+    } else if (!out.youtube && /(youtube\.com\/(channel|c|user|@)|youtu\.be\/)/i.test(u)) {
+      out.youtube = absolutize(u);
+    } else if (!out.tiktok && /(^|\/\/|\.)tiktok\.com\/@/i.test(u)) {
+      out.tiktok = absolutize(u);
+    } else if (!out.telegram && /(t\.me\/|telegram\.me\/)/i.test(u) && !/\/share\b/i.test(u)) {
+      out.telegram = absolutize(u);
     }
   }
   return out;
@@ -130,11 +200,9 @@ async function enrichContacts(website, opts = {}) {
         const email = extractEmails(page);
         if (email) {
           contacts.email = email;
-          if (!contacts.facebook || !contacts.instagram || !contacts.whatsapp) {
-            const s = extractSocials(page);
-            if (!contacts.facebook && s.facebook) contacts.facebook = s.facebook;
-            if (!contacts.instagram && s.instagram) contacts.instagram = s.instagram;
-            if (!contacts.whatsapp && s.whatsapp) contacts.whatsapp = s.whatsapp;
+          const s = extractSocials(page);
+          for (const k of Object.keys(s)) {
+            if (!contacts[k] && s[k]) contacts[k] = s[k];
           }
           break;
         }
@@ -189,7 +257,148 @@ function dedupeLeads(leads, by = "website") {
   return out;
 }
 
+// src/normalize.ts
+var TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|gbraid$|wbraid$|mc_|_hs|ref$|ref_src$|source$|igshid$|yclid$)/i;
+function cleanWebsite(url) {
+  if (!url) return void 0;
+  let raw = url.trim();
+  if (!raw) return void 0;
+  const g = raw.match(/[?&](?:q|url)=([^&]+)/i);
+  if (g && /google\.[a-z.]+\/url/i.test(raw)) {
+    try {
+      raw = decodeURIComponent(g[1]);
+    } catch {
+    }
+  }
+  let u;
+  try {
+    u = new URL(raw.startsWith("http") ? raw : `https://${raw}`);
+  } catch {
+    return raw;
+  }
+  u.hash = "";
+  u.host = u.host.toLowerCase();
+  for (const key of [...u.searchParams.keys()]) {
+    if (TRACKING_PARAMS.test(key)) u.searchParams.delete(key);
+  }
+  let href = u.toString();
+  if (u.pathname === "/" && !u.search) href = href.replace(/\/$/, "");
+  return href;
+}
+var CALLING_CODES = {
+  NP: "977",
+  IN: "91",
+  US: "1",
+  CA: "1",
+  GB: "44",
+  AU: "61",
+  NZ: "64",
+  SG: "65",
+  MY: "60",
+  ID: "62",
+  TH: "66",
+  PH: "63",
+  VN: "84",
+  BD: "880",
+  PK: "92",
+  LK: "94",
+  AE: "971",
+  SA: "966",
+  QA: "974",
+  KW: "965",
+  BH: "973",
+  OM: "968",
+  CN: "86",
+  HK: "852",
+  JP: "81",
+  KR: "82",
+  DE: "49",
+  FR: "33",
+  ES: "34",
+  IT: "39",
+  NL: "31",
+  BE: "32",
+  CH: "41",
+  AT: "43",
+  SE: "46",
+  NO: "47",
+  DK: "45",
+  FI: "358",
+  IE: "353",
+  PT: "351",
+  PL: "48",
+  ZA: "27",
+  NG: "234",
+  KE: "254",
+  EG: "20",
+  BR: "55",
+  MX: "52",
+  AR: "54",
+  TR: "90"
+};
+function callingCode(country) {
+  if (!country) return void 0;
+  const c = country.trim().replace(/^\+/, "");
+  if (/^\d{1,4}$/.test(c)) return c;
+  return CALLING_CODES[c.toUpperCase()];
+}
+function normalizePhone(raw, country) {
+  if (!raw) return void 0;
+  const trimmed = raw.trim();
+  if (!trimmed) return void 0;
+  const isIntl = /^\+/.test(trimmed) || /^00\d/.test(trimmed);
+  let digits = trimmed.replace(/[^0-9]/g, "");
+  if (/^00\d/.test(trimmed)) digits = digits.replace(/^00/, "");
+  const cc = callingCode(country);
+  if (isIntl) {
+    return plausible(digits) ? `+${digits}` : trimmed;
+  }
+  if (!cc) return trimmed;
+  if (digits.startsWith(cc)) return plausible(digits) ? `+${digits}` : trimmed;
+  const local = digits.replace(/^0+/, "");
+  const e164 = `${cc}${local}`;
+  return plausible(e164) ? `+${e164}` : trimmed;
+}
+function plausible(digits) {
+  return digits.length >= 8 && digits.length <= 15;
+}
+var priceRank = (p) => p ? p.replace(/[^$€£₹¥₩]/g, "").length : 0;
+function sortLeads(leads, by, dir) {
+  if (!by) return [...leads];
+  const desc = dir ? dir === "desc" : by !== "name";
+  const val = (l) => {
+    if (by === "name") return l.name?.toLowerCase();
+    if (by === "priceLevel") return l.priceLevel ? priceRank(l.priceLevel) : void 0;
+    return l[by];
+  };
+  return [...leads].map((lead, i) => ({ lead, i })).sort((a, b) => {
+    const av = val(a.lead);
+    const bv = val(b.lead);
+    const aMissing = av === void 0 || av === "";
+    const bMissing = bv === void 0 || bv === "";
+    if (aMissing && bMissing) return a.i - b.i;
+    if (aMissing) return 1;
+    if (bMissing) return -1;
+    let cmp = 0;
+    if (typeof av === "number" && typeof bv === "number") cmp = av - bv;
+    else cmp = String(av).localeCompare(String(bv));
+    if (desc) cmp = -cmp;
+    return cmp !== 0 ? cmp : a.i - b.i;
+  }).map((x) => x.lead);
+}
+
 // src/scrape.ts
+async function pool(items, n, fn, signal) {
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => {
+    for (; ; ) {
+      const i = next++;
+      if (i >= items.length || signal?.aborted) return;
+      await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+}
 var LeadsError = class extends Error {
   code;
   cause;
@@ -314,7 +523,13 @@ async function extractDetail(page, fields, fallbackName) {
       }
       if (fields.has("reviews")) {
         const rv = await box.locator('span[aria-label*="review"]').first().getAttribute("aria-label").catch(() => null);
-        lead.reviews = parseReviewCount(rv);
+        let count = parseReviewCount(rv);
+        if (count === void 0) {
+          const boxText = await box.innerText().catch(() => "");
+          const m = boxText.match(/\(([\d.,\s]+)\)/);
+          if (m) count = parseReviewCount(m[1]);
+        }
+        lead.reviews = count;
       }
     }
   }
@@ -366,12 +581,15 @@ async function scrapeLeads(opts) {
   onProgress?.(`searching Google Maps for "${query}"\u2026`);
   const browser = await launchBrowser(headless);
   try {
+    const locale = opts.locale ?? "en-US";
     const context = await browser.newContext({
       viewport: { width: 1280, height: 900 },
-      locale: "en-US"
+      locale
     });
     const page = await context.newPage();
-    await page.goto(mapsSearchUrl(query), { waitUntil: "domcontentloaded", timeout: 45e3 });
+    const urlOpts = { hl: locale };
+    if (opts.region) urlOpts.gl = opts.region;
+    await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
     await dismissConsent(page);
     await loadResults(page, limit, delayMs, onProgress, signal);
     if (signal?.aborted) throw new LeadsError("Search aborted.", "ABORTED");
@@ -421,22 +639,42 @@ async function scrapeLeads(opts) {
       }
     }
     onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
+    if (opts.cleanUrls ?? true) {
+      for (const lead of leads) {
+        if (lead.website) lead.website = cleanWebsite(lead.website);
+      }
+    }
     leads = dedupeLeads(leads, opts.dedupe ?? "website");
     if (wantEnrich) {
-      for (let i = 0; i < leads.length; i++) {
-        if (signal?.aborted || overBudget()) break;
-        const site = leads[i].website;
-        if (!site) continue;
-        onProgress?.(`enriching ${i + 1}/${leads.length}: ${leads[i].name ?? site}\u2026`);
-        const c = await enrichContacts(site);
-        if (fields.has("email") && c.email) leads[i].email = c.email;
-        if (fields.has("facebook") && c.facebook) leads[i].facebook = c.facebook;
-        if (fields.has("instagram") && c.instagram) leads[i].instagram = c.instagram;
-        if (fields.has("whatsapp") && c.whatsapp) leads[i].whatsapp = c.whatsapp;
-        if (!fields.has("website")) delete leads[i].website;
+      const socialFields = ENRICHED_FIELDS.filter((f) => f !== "email");
+      let done = 0;
+      await pool(
+        leads,
+        Math.max(1, Math.trunc(opts.concurrency ?? 3)),
+        async (lead) => {
+          if (overBudget()) return;
+          const site = lead.website;
+          if (site) {
+            onProgress?.(`enriching ${++done}/${leads.length}: ${lead.name ?? site}\u2026`);
+            const c = await enrichContacts(site);
+            if (fields.has("email") && c.email) lead.email = c.email;
+            for (const f of socialFields) {
+              const v = c[f];
+              if (fields.has(f) && v) lead[f] = v;
+            }
+          }
+          if (!fields.has("website")) delete lead.website;
+        },
+        signal
+      );
+    }
+    if (opts.country && fields.has("phone")) {
+      for (const lead of leads) {
+        if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
       }
     }
     if (opts.filters) leads = filterLeads(leads, opts.filters);
+    if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
     onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
     return leads;
   } finally {
@@ -445,6 +683,69 @@ async function scrapeLeads(opts) {
   }
 }
 
+// src/batch.ts
+async function searchLeadsBatch(queries, opts = {}) {
+  const {
+    total,
+    onProgress,
+    signal,
+    type: _t,
+    city: _c,
+    area: _a,
+    query: _q,
+    ...shared
+  } = opts;
+  void _t;
+  void _c;
+  void _a;
+  void _q;
+  let merged = [];
+  const seenDedupe = shared.dedupe ?? "website";
+  for (let i = 0; i < queries.length; i++) {
+    if (signal?.aborted) break;
+    const q = queries[i];
+    const label = (() => {
+      try {
+        return composeQuery(q);
+      } catch {
+        return q.query ?? q.type ?? "search";
+      }
+    })();
+    onProgress?.(`[${i + 1}/${queries.length}] ${label}`);
+    const subOpts = {
+      ...shared,
+      type: q.type ?? "",
+      // Defer sorting to the merged pass; keep per-search dedupe/filter local.
+      ...q.query ? { query: q.query } : {},
+      ...q.city ? { city: q.city } : {},
+      ...q.area ? { area: q.area } : {}
+    };
+    delete subOpts.sort;
+    if (signal) subOpts.signal = signal;
+    if (onProgress) subOpts.onProgress = (m) => onProgress(`  ${m}`);
+    try {
+      const found = await scrapeLeads(subOpts);
+      merged = merged.concat(found);
+    } catch (err) {
+      onProgress?.(`  ! skipped "${label}": ${err.message}`);
+    }
+  }
+  merged = dedupeLeads(merged, seenDedupe);
+  if (opts.sort) merged = sortLeads(merged, opts.sort, opts.sortDir);
+  if (total !== void 0 && total > 0) merged = merged.slice(0, total);
+  onProgress?.(`merged ${merged.length} unique lead${merged.length === 1 ? "" : "s"}.`);
+  return merged;
+}
+async function searchLeadsMulti(opts) {
+  const queries = expandQueries({
+    ...opts.type ? { type: opts.type } : {},
+    ...opts.city ? { city: opts.city } : {},
+    ...opts.area ? { area: opts.area } : {},
+    ...opts.query ? { query: opts.query } : {}
+  });
+  return searchLeadsBatch(queries, opts);
+}
+
 // src/export.ts
 import { stringify as csvStringify } from "@lacspace/csv";
 import { jsonToXlsx } from "@lacspace/xlsx";
@@ -461,6 +762,11 @@ var HEADERS = {
   facebook: "Facebook",
   instagram: "Instagram",
   whatsapp: "WhatsApp",
+  linkedin: "LinkedIn",
+  twitter: "Twitter/X",
+  youtube: "YouTube",
+  tiktok: "TikTok",
+  telegram: "Telegram",
   plusCode: "Plus Code",
   latitude: "Latitude",
   longitude: "Longitude",
@@ -478,13 +784,14 @@ function toRows(leads, fields = ALL_FIELDS) {
   });
 }
 function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
-  if (format === "json") {
+  if (format === "json" || format === "ndjson") {
     const picked = leads.map((lead) => {
       const o = {};
       for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
       return o;
     });
-    return { data: JSON.stringify(picked, null, 2), binary: false };
+    const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
+    return { data, binary: false };
   }
   const rows = toRows(leads, fields);
   if (format === "csv") {
@@ -577,8 +884,13 @@ async function convertFile(input, opts = {}) {
 }
 export {
   ALL_FIELDS,
+  CALLING_CODES,
+  DEFAULT_FIELDS,
   ENRICHED_FIELDS,
+  FIELD_PRESETS,
   LeadsError,
+  callingCode,
+  cleanWebsite,
   columnsOf,
   composeQuery,
   convertFile,
@@ -586,18 +898,24 @@ export {
   defaultFilename,
   detectFormat,
   enrichContacts,
+  expandQueries,
   extractEmails,
   extractSocials,
   filterLeads,
   mapsSearchUrl,
   normalizeFields,
+  normalizePhone,
   parseLatLng,
   parseRating,
   parseReviewCount,
   readRows,
+  resolvePreset,
   scrapeLeads,
   scrapeLeads as searchLeads,
+  searchLeadsBatch,
+  searchLeadsMulti,
   serialize,
   serializeRows,
+  sortLeads,
   toRows
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lacspace-leads",
-  "version": "0.2.0",
-  "description": "Free, open-source local-business lead finder — name a city, area and business type, and it drives a real browser over Google Maps to collect names, phones, websites, ratings and addresses, then exports to JSON, CSV or Excel. No API keys.",
+  "version": "0.3.0",
+  "description": "Free, open-source local-business lead finder — name a city, area and business type, and it drives a real browser over Google Maps to collect names, phones, websites, ratings, addresses, emails and social links, then exports to JSON, NDJSON, CSV or Excel. Sweep multiple areas at once, normalise phones to E.164, and enrich in parallel. No API keys.",
   "type": "module",
   "bin": {
     "lacspace-leads": "dist/cli.js"
@@ -35,11 +35,14 @@
     "local-business",
     "prospecting",
     "b2b",
+    "email-finder",
     "csv",
     "excel",
+    "ndjson",
     "playwright",
     "cli",
     "osint",
+    "e164",
     "nepal",
     "typescript"
   ],
@@ -48,10 +51,11 @@
   "homepage": "https://developer.lacspace.com",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/lacspace/lacspace-leads.git"
+    "url": "git+https://github.com/lacspace/npm-packages.git",
+    "directory": "lacspace-leads"
   },
   "bugs": {
-    "url": "https://github.com/lacspace/lacspace-leads/issues"
+    "url": "https://github.com/lacspace/npm-packages/issues"
   },
   "engines": {
     "node": ">=20"