lacspace-leads 0.2.0 → 0.3.0

package/dist/lib.cjs

@@ -21,8 +21,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var lib_exports = {};
 __export(lib_exports, {
   ALL_FIELDS: () => ALL_FIELDS,
+  CALLING_CODES: () => CALLING_CODES,
+  DEFAULT_FIELDS: () => DEFAULT_FIELDS,
   ENRICHED_FIELDS: () => ENRICHED_FIELDS,
+  FIELD_PRESETS: () => FIELD_PRESETS,
   LeadsError: () => LeadsError,
+  callingCode: () => callingCode,
+  cleanWebsite: () => cleanWebsite,
   columnsOf: () => columnsOf,
   composeQuery: () => composeQuery,
   convertFile: () => convertFile,
@@ -30,19 +35,25 @@ __export(lib_exports, {
   defaultFilename: () => defaultFilename,
   detectFormat: () => detectFormat,
   enrichContacts: () => enrichContacts,
+  expandQueries: () => expandQueries,
   extractEmails: () => extractEmails,
   extractSocials: () => extractSocials,
   filterLeads: () => filterLeads,
   mapsSearchUrl: () => mapsSearchUrl,
   normalizeFields: () => normalizeFields,
+  normalizePhone: () => normalizePhone,
   parseLatLng: () => parseLatLng,
   parseRating: () => parseRating,
   parseReviewCount: () => parseReviewCount,
   readRows: () => readRows,
+  resolvePreset: () => resolvePreset,
   scrapeLeads: () => scrapeLeads,
   searchLeads: () => scrapeLeads,
+  searchLeadsBatch: () => searchLeadsBatch,
+  searchLeadsMulti: () => searchLeadsMulti,
   serialize: () => serialize,
   serializeRows: () => serializeRows,
+  sortLeads: () => sortLeads,
   toRows: () => toRows
 });
 module.exports = __toCommonJS(lib_exports);
@@ -64,13 +75,42 @@ var ALL_FIELDS = [
   "facebook",
   "instagram",
   "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram",
   "plusCode",
   "latitude",
   "longitude",
   "hours",
   "mapsUrl"
 ];
-var ENRICHED_FIELDS = ["email", "facebook", "instagram", "whatsapp"];
+var ENRICHED_FIELDS = [
+  "email",
+  "facebook",
+  "instagram",
+  "whatsapp",
+  "linkedin",
+  "twitter",
+  "youtube",
+  "tiktok",
+  "telegram"
+];
+var FIELD_PRESETS = {
+  /** Just enough to place a business: name, phone, website. */
+  minimal: ["name", "phone", "website"],
+  /** Cold-outreach columns: who, how to reach them, where. */
+  outreach: ["name", "phone", "email", "website", "address"],
+  /** Contact-book columns incl. socials (implies enrichment). */
+  contact: ["name", "phone", "email", "website", "facebook", "instagram", "whatsapp"],
+  /** Map/geo columns for plotting. */
+  geo: ["name", "address", "latitude", "longitude", "plusCode", "mapsUrl"],
+  /** Everything Maps shows, no website enrichment. */
+  full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f)),
+  /** Every field, including enriched ones. */
+  everything: [...ALL_FIELDS]
+};
 var DEFAULT_FIELDS = ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f));
 
 // src/query.ts
@@ -81,8 +121,38 @@ function composeQuery(opts) {
   if (!type) throw new Error("A business `type` (or an explicit `query`) is required.");
   return where ? `${type} in ${where}` : type;
 }
-function mapsSearchUrl(query) {
-  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?hl=en`;
+function mapsSearchUrl(query, opts = {}) {
+  const hl = (opts.hl ?? "en").split("-")[0] || "en";
+  const params = new URLSearchParams({ hl });
+  if (opts.gl) params.set("gl", opts.gl.toLowerCase());
+  return `https://www.google.com/maps/search/${encodeURIComponent(query)}?${params.toString()}`;
+}
+function expandQueries(opts) {
+  if (opts.query && opts.query.trim()) return [{ query: opts.query.trim() }];
+  const split = (s) => {
+    const parts = (s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+    return parts.length ? parts : [void 0];
+  };
+  const types = split(opts.type);
+  const cities = split(opts.city);
+  const areas = split(opts.area);
+  const out = [];
+  for (const type of types) {
+    for (const city of cities) {
+      for (const area of areas) {
+        const entry = {};
+        if (type) entry.type = type;
+        if (city) entry.city = city;
+        if (area) entry.area = area;
+        out.push(entry);
+      }
+    }
+  }
+  return out;
+}
+function resolvePreset(name) {
+  if (!name) return void 0;
+  return FIELD_PRESETS[name.trim().toLowerCase()];
 }
 function normalizeFields(input) {
   if (!input) return [...ALL_FIELDS];
@@ -123,12 +193,23 @@ function extractSocials(html) {
   const hrefs = [...html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1] ?? "");
   for (const href of hrefs) {
     const u = href.trim();
-    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?/i.test(u)) {
+    if (!u) continue;
+    if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?|dialog\//i.test(u)) {
       out.facebook = absolutize(u);
-    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u)) {
+    } else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u) && !/\/(share|p)\//i.test(u)) {
       out.instagram = absolutize(u);
     } else if (!out.whatsapp && /(wa\.me\/|api\.whatsapp\.com\/|whatsapp:\/\/)/i.test(u)) {
       out.whatsapp = u.startsWith("http") || u.startsWith("whatsapp") ? u : `https:${u}`;
+    } else if (!out.linkedin && /(^|\/\/|\.)linkedin\.com\/(company|in|school)\//i.test(u) && !/sharing|shareArticle/i.test(u)) {
+      out.linkedin = absolutize(u);
+    } else if (!out.twitter && /(^|\/\/|\.)(twitter|x)\.com\//i.test(u) && !/(intent|share|widgets|hashtag)\b/i.test(u)) {
+      out.twitter = absolutize(u);
+    } else if (!out.youtube && /(youtube\.com\/(channel|c|user|@)|youtu\.be\/)/i.test(u)) {
+      out.youtube = absolutize(u);
+    } else if (!out.tiktok && /(^|\/\/|\.)tiktok\.com\/@/i.test(u)) {
+      out.tiktok = absolutize(u);
+    } else if (!out.telegram && /(t\.me\/|telegram\.me\/)/i.test(u) && !/\/share\b/i.test(u)) {
+      out.telegram = absolutize(u);
     }
   }
   return out;
@@ -179,11 +260,9 @@ async function enrichContacts(website, opts = {}) {
         const email = extractEmails(page);
         if (email) {
           contacts.email = email;
-          if (!contacts.facebook || !contacts.instagram || !contacts.whatsapp) {
-            const s = extractSocials(page);
-            if (!contacts.facebook && s.facebook) contacts.facebook = s.facebook;
-            if (!contacts.instagram && s.instagram) contacts.instagram = s.instagram;
-            if (!contacts.whatsapp && s.whatsapp) contacts.whatsapp = s.whatsapp;
+          const s = extractSocials(page);
+          for (const k of Object.keys(s)) {
+            if (!contacts[k] && s[k]) contacts[k] = s[k];
           }
           break;
         }
@@ -238,7 +317,148 @@ function dedupeLeads(leads, by = "website") {
   return out;
 }
 
+// src/normalize.ts
+var TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|gbraid$|wbraid$|mc_|_hs|ref$|ref_src$|source$|igshid$|yclid$)/i;
+function cleanWebsite(url) {
+  if (!url) return void 0;
+  let raw = url.trim();
+  if (!raw) return void 0;
+  const g = raw.match(/[?&](?:q|url)=([^&]+)/i);
+  if (g && /google\.[a-z.]+\/url/i.test(raw)) {
+    try {
+      raw = decodeURIComponent(g[1]);
+    } catch {
+    }
+  }
+  let u;
+  try {
+    u = new URL(raw.startsWith("http") ? raw : `https://${raw}`);
+  } catch {
+    return raw;
+  }
+  u.hash = "";
+  u.host = u.host.toLowerCase();
+  for (const key of [...u.searchParams.keys()]) {
+    if (TRACKING_PARAMS.test(key)) u.searchParams.delete(key);
+  }
+  let href = u.toString();
+  if (u.pathname === "/" && !u.search) href = href.replace(/\/$/, "");
+  return href;
+}
+var CALLING_CODES = {
+  NP: "977",
+  IN: "91",
+  US: "1",
+  CA: "1",
+  GB: "44",
+  AU: "61",
+  NZ: "64",
+  SG: "65",
+  MY: "60",
+  ID: "62",
+  TH: "66",
+  PH: "63",
+  VN: "84",
+  BD: "880",
+  PK: "92",
+  LK: "94",
+  AE: "971",
+  SA: "966",
+  QA: "974",
+  KW: "965",
+  BH: "973",
+  OM: "968",
+  CN: "86",
+  HK: "852",
+  JP: "81",
+  KR: "82",
+  DE: "49",
+  FR: "33",
+  ES: "34",
+  IT: "39",
+  NL: "31",
+  BE: "32",
+  CH: "41",
+  AT: "43",
+  SE: "46",
+  NO: "47",
+  DK: "45",
+  FI: "358",
+  IE: "353",
+  PT: "351",
+  PL: "48",
+  ZA: "27",
+  NG: "234",
+  KE: "254",
+  EG: "20",
+  BR: "55",
+  MX: "52",
+  AR: "54",
+  TR: "90"
+};
+function callingCode(country) {
+  if (!country) return void 0;
+  const c = country.trim().replace(/^\+/, "");
+  if (/^\d{1,4}$/.test(c)) return c;
+  return CALLING_CODES[c.toUpperCase()];
+}
+function normalizePhone(raw, country) {
+  if (!raw) return void 0;
+  const trimmed = raw.trim();
+  if (!trimmed) return void 0;
+  const isIntl = /^\+/.test(trimmed) || /^00\d/.test(trimmed);
+  let digits = trimmed.replace(/[^0-9]/g, "");
+  if (/^00\d/.test(trimmed)) digits = digits.replace(/^00/, "");
+  const cc = callingCode(country);
+  if (isIntl) {
+    return plausible(digits) ? `+${digits}` : trimmed;
+  }
+  if (!cc) return trimmed;
+  if (digits.startsWith(cc)) return plausible(digits) ? `+${digits}` : trimmed;
+  const local = digits.replace(/^0+/, "");
+  const e164 = `${cc}${local}`;
+  return plausible(e164) ? `+${e164}` : trimmed;
+}
+function plausible(digits) {
+  return digits.length >= 8 && digits.length <= 15;
+}
+var priceRank = (p) => p ? p.replace(/[^$€£₹¥₩]/g, "").length : 0;
+function sortLeads(leads, by, dir) {
+  if (!by) return [...leads];
+  const desc = dir ? dir === "desc" : by !== "name";
+  const val = (l) => {
+    if (by === "name") return l.name?.toLowerCase();
+    if (by === "priceLevel") return l.priceLevel ? priceRank(l.priceLevel) : void 0;
+    return l[by];
+  };
+  return [...leads].map((lead, i) => ({ lead, i })).sort((a, b) => {
+    const av = val(a.lead);
+    const bv = val(b.lead);
+    const aMissing = av === void 0 || av === "";
+    const bMissing = bv === void 0 || bv === "";
+    if (aMissing && bMissing) return a.i - b.i;
+    if (aMissing) return 1;
+    if (bMissing) return -1;
+    let cmp = 0;
+    if (typeof av === "number" && typeof bv === "number") cmp = av - bv;
+    else cmp = String(av).localeCompare(String(bv));
+    if (desc) cmp = -cmp;
+    return cmp !== 0 ? cmp : a.i - b.i;
+  }).map((x) => x.lead);
+}
+
 // src/scrape.ts
+async function pool(items, n, fn, signal) {
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => {
+    for (; ; ) {
+      const i = next++;
+      if (i >= items.length || signal?.aborted) return;
+      await fn(items[i], i);
+    }
+  });
+  await Promise.all(workers);
+}
 var LeadsError = class extends Error {
   code;
   cause;
@@ -363,7 +583,13 @@ async function extractDetail(page, fields, fallbackName) {
       }
       if (fields.has("reviews")) {
         const rv = await box.locator('span[aria-label*="review"]').first().getAttribute("aria-label").catch(() => null);
-        lead.reviews = parseReviewCount(rv);
+        let count = parseReviewCount(rv);
+        if (count === void 0) {
+          const boxText = await box.innerText().catch(() => "");
+          const m = boxText.match(/\(([\d.,\s]+)\)/);
+          if (m) count = parseReviewCount(m[1]);
+        }
+        lead.reviews = count;
       }
     }
   }
@@ -415,12 +641,15 @@ async function scrapeLeads(opts) {
   onProgress?.(`searching Google Maps for "${query}"\u2026`);
   const browser = await launchBrowser(headless);
   try {
+    const locale = opts.locale ?? "en-US";
     const context = await browser.newContext({
       viewport: { width: 1280, height: 900 },
-      locale: "en-US"
+      locale
     });
     const page = await context.newPage();
-    await page.goto(mapsSearchUrl(query), { waitUntil: "domcontentloaded", timeout: 45e3 });
+    const urlOpts = { hl: locale };
+    if (opts.region) urlOpts.gl = opts.region;
+    await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
     await dismissConsent(page);
     await loadResults(page, limit, delayMs, onProgress, signal);
     if (signal?.aborted) throw new LeadsError("Search aborted.", "ABORTED");
@@ -470,22 +699,42 @@ async function scrapeLeads(opts) {
       }
     }
     onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
+    if (opts.cleanUrls ?? true) {
+      for (const lead of leads) {
+        if (lead.website) lead.website = cleanWebsite(lead.website);
+      }
+    }
     leads = dedupeLeads(leads, opts.dedupe ?? "website");
     if (wantEnrich) {
-      for (let i = 0; i < leads.length; i++) {
-        if (signal?.aborted || overBudget()) break;
-        const site = leads[i].website;
-        if (!site) continue;
-        onProgress?.(`enriching ${i + 1}/${leads.length}: ${leads[i].name ?? site}\u2026`);
-        const c = await enrichContacts(site);
-        if (fields.has("email") && c.email) leads[i].email = c.email;
-        if (fields.has("facebook") && c.facebook) leads[i].facebook = c.facebook;
-        if (fields.has("instagram") && c.instagram) leads[i].instagram = c.instagram;
-        if (fields.has("whatsapp") && c.whatsapp) leads[i].whatsapp = c.whatsapp;
-        if (!fields.has("website")) delete leads[i].website;
+      const socialFields = ENRICHED_FIELDS.filter((f) => f !== "email");
+      let done = 0;
+      await pool(
+        leads,
+        Math.max(1, Math.trunc(opts.concurrency ?? 3)),
+        async (lead) => {
+          if (overBudget()) return;
+          const site = lead.website;
+          if (site) {
+            onProgress?.(`enriching ${++done}/${leads.length}: ${lead.name ?? site}\u2026`);
+            const c = await enrichContacts(site);
+            if (fields.has("email") && c.email) lead.email = c.email;
+            for (const f of socialFields) {
+              const v = c[f];
+              if (fields.has(f) && v) lead[f] = v;
+            }
+          }
+          if (!fields.has("website")) delete lead.website;
+        },
+        signal
+      );
+    }
+    if (opts.country && fields.has("phone")) {
+      for (const lead of leads) {
+        if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
       }
     }
     if (opts.filters) leads = filterLeads(leads, opts.filters);
+    if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
     onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
     return leads;
   } finally {
@@ -494,6 +743,69 @@ async function scrapeLeads(opts) {
   }
 }
 
+// src/batch.ts
+async function searchLeadsBatch(queries, opts = {}) {
+  const {
+    total,
+    onProgress,
+    signal,
+    type: _t,
+    city: _c,
+    area: _a,
+    query: _q,
+    ...shared
+  } = opts;
+  void _t;
+  void _c;
+  void _a;
+  void _q;
+  let merged = [];
+  const seenDedupe = shared.dedupe ?? "website";
+  for (let i = 0; i < queries.length; i++) {
+    if (signal?.aborted) break;
+    const q = queries[i];
+    const label = (() => {
+      try {
+        return composeQuery(q);
+      } catch {
+        return q.query ?? q.type ?? "search";
+      }
+    })();
+    onProgress?.(`[${i + 1}/${queries.length}] ${label}`);
+    const subOpts = {
+      ...shared,
+      type: q.type ?? "",
+      // Defer sorting to the merged pass; keep per-search dedupe/filter local.
+      ...q.query ? { query: q.query } : {},
+      ...q.city ? { city: q.city } : {},
+      ...q.area ? { area: q.area } : {}
+    };
+    delete subOpts.sort;
+    if (signal) subOpts.signal = signal;
+    if (onProgress) subOpts.onProgress = (m) => onProgress(`  ${m}`);
+    try {
+      const found = await scrapeLeads(subOpts);
+      merged = merged.concat(found);
+    } catch (err) {
+      onProgress?.(`  ! skipped "${label}": ${err.message}`);
+    }
+  }
+  merged = dedupeLeads(merged, seenDedupe);
+  if (opts.sort) merged = sortLeads(merged, opts.sort, opts.sortDir);
+  if (total !== void 0 && total > 0) merged = merged.slice(0, total);
+  onProgress?.(`merged ${merged.length} unique lead${merged.length === 1 ? "" : "s"}.`);
+  return merged;
+}
+async function searchLeadsMulti(opts) {
+  const queries = expandQueries({
+    ...opts.type ? { type: opts.type } : {},
+    ...opts.city ? { city: opts.city } : {},
+    ...opts.area ? { area: opts.area } : {},
+    ...opts.query ? { query: opts.query } : {}
+  });
+  return searchLeadsBatch(queries, opts);
+}
+
 // src/export.ts
 var import_csv = require("@lacspace/csv");
 var import_xlsx = require("@lacspace/xlsx");
@@ -510,6 +822,11 @@ var HEADERS = {
   facebook: "Facebook",
   instagram: "Instagram",
   whatsapp: "WhatsApp",
+  linkedin: "LinkedIn",
+  twitter: "Twitter/X",
+  youtube: "YouTube",
+  tiktok: "TikTok",
+  telegram: "Telegram",
   plusCode: "Plus Code",
   latitude: "Latitude",
   longitude: "Longitude",
@@ -527,13 +844,14 @@ function toRows(leads, fields = ALL_FIELDS) {
   });
 }
 function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
-  if (format === "json") {
+  if (format === "json" || format === "ndjson") {
     const picked = leads.map((lead) => {
       const o = {};
       for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
       return o;
     });
-    return { data: JSON.stringify(picked, null, 2), binary: false };
+    const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
+    return { data, binary: false };
   }
   const rows = toRows(leads, fields);
   if (format === "csv") {
@@ -627,8 +945,13 @@ async function convertFile(input, opts = {}) {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   ALL_FIELDS,
+  CALLING_CODES,
+  DEFAULT_FIELDS,
   ENRICHED_FIELDS,
+  FIELD_PRESETS,
   LeadsError,
+  callingCode,
+  cleanWebsite,
   columnsOf,
   composeQuery,
   convertFile,
@@ -636,18 +959,24 @@ async function convertFile(input, opts = {}) {
   defaultFilename,
   detectFormat,
   enrichContacts,
+  expandQueries,
   extractEmails,
   extractSocials,
   filterLeads,
   mapsSearchUrl,
   normalizeFields,
+  normalizePhone,
   parseLatLng,
   parseRating,
   parseReviewCount,
   readRows,
+  resolvePreset,
   scrapeLeads,
   searchLeads,
+  searchLeadsBatch,
+  searchLeadsMulti,
   serialize,
   serializeRows,
+  sortLeads,
   toRows
 });