lacspace-leads 0.3.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -8
- package/dist/cli.js +383 -100
- package/dist/lib.cjs +323 -91
- package/dist/lib.d.cts +134 -8
- package/dist/lib.d.ts +134 -8
- package/dist/lib.js +307 -86
- package/package.json +6 -2
package/dist/lib.cjs
CHANGED
|
@@ -30,31 +30,42 @@ __export(lib_exports, {
|
|
|
30
30
|
cleanWebsite: () => cleanWebsite,
|
|
31
31
|
columnsOf: () => columnsOf,
|
|
32
32
|
composeQuery: () => composeQuery,
|
|
33
|
+
computeStats: () => computeStats,
|
|
33
34
|
convertFile: () => convertFile,
|
|
34
35
|
dedupeLeads: () => dedupeLeads,
|
|
35
36
|
defaultFilename: () => defaultFilename,
|
|
36
37
|
detectFormat: () => detectFormat,
|
|
38
|
+
emailDomain: () => emailDomain,
|
|
39
|
+
emailFormatValid: () => emailFormatValid,
|
|
37
40
|
enrichContacts: () => enrichContacts,
|
|
38
41
|
expandQueries: () => expandQueries,
|
|
39
42
|
extractEmails: () => extractEmails,
|
|
40
43
|
extractSocials: () => extractSocials,
|
|
41
44
|
filterLeads: () => filterLeads,
|
|
45
|
+
haversineMeters: () => haversineMeters,
|
|
42
46
|
mapsSearchUrl: () => mapsSearchUrl,
|
|
43
47
|
normalizeFields: () => normalizeFields,
|
|
44
48
|
normalizePhone: () => normalizePhone,
|
|
49
|
+
parseDistance: () => parseDistance,
|
|
45
50
|
parseLatLng: () => parseLatLng,
|
|
51
|
+
parseLatLngPair: () => parseLatLngPair,
|
|
46
52
|
parseRating: () => parseRating,
|
|
47
53
|
parseReviewCount: () => parseReviewCount,
|
|
48
54
|
readRows: () => readRows,
|
|
49
55
|
resolvePreset: () => resolvePreset,
|
|
56
|
+
rowsToLeads: () => rowsToLeads,
|
|
50
57
|
scrapeLeads: () => scrapeLeads,
|
|
51
58
|
searchLeads: () => scrapeLeads,
|
|
52
59
|
searchLeadsBatch: () => searchLeadsBatch,
|
|
60
|
+
searchLeadsDetailed: () => searchLeadsDetailed,
|
|
53
61
|
searchLeadsMulti: () => searchLeadsMulti,
|
|
54
62
|
serialize: () => serialize,
|
|
55
63
|
serializeRows: () => serializeRows,
|
|
56
64
|
sortLeads: () => sortLeads,
|
|
57
|
-
toRows: () => toRows
|
|
65
|
+
toRows: () => toRows,
|
|
66
|
+
verifyEmail: () => verifyEmail,
|
|
67
|
+
verifyEmails: () => verifyEmails,
|
|
68
|
+
zoomForRadius: () => zoomForRadius
|
|
58
69
|
});
|
|
59
70
|
module.exports = __toCommonJS(lib_exports);
|
|
60
71
|
|
|
@@ -80,12 +91,15 @@ var ALL_FIELDS = [
|
|
|
80
91
|
"youtube",
|
|
81
92
|
"tiktok",
|
|
82
93
|
"telegram",
|
|
94
|
+
"emailStatus",
|
|
83
95
|
"plusCode",
|
|
84
96
|
"latitude",
|
|
85
97
|
"longitude",
|
|
98
|
+
"distanceKm",
|
|
86
99
|
"hours",
|
|
87
100
|
"mapsUrl"
|
|
88
101
|
];
|
|
102
|
+
var DERIVED_FIELDS = ["distanceKm"];
|
|
89
103
|
var ENRICHED_FIELDS = [
|
|
90
104
|
"email",
|
|
91
105
|
"facebook",
|
|
@@ -95,7 +109,8 @@ var ENRICHED_FIELDS = [
|
|
|
95
109
|
"twitter",
|
|
96
110
|
"youtube",
|
|
97
111
|
"tiktok",
|
|
98
|
-
"telegram"
|
|
112
|
+
"telegram",
|
|
113
|
+
"emailStatus"
|
|
99
114
|
];
|
|
100
115
|
var FIELD_PRESETS = {
|
|
101
116
|
/** Just enough to place a business: name, phone, website. */
|
|
@@ -107,11 +122,13 @@ var FIELD_PRESETS = {
|
|
|
107
122
|
/** Map/geo columns for plotting. */
|
|
108
123
|
geo: ["name", "address", "latitude", "longitude", "plusCode", "mapsUrl"],
|
|
109
124
|
/** Everything Maps shows, no website enrichment. */
|
|
110
|
-
full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f)),
|
|
125
|
+
full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f) && !DERIVED_FIELDS.includes(f)),
|
|
111
126
|
/** Every field, including enriched ones. */
|
|
112
127
|
everything: [...ALL_FIELDS]
|
|
113
128
|
};
|
|
114
|
-
var DEFAULT_FIELDS = ALL_FIELDS.filter(
|
|
129
|
+
var DEFAULT_FIELDS = ALL_FIELDS.filter(
|
|
130
|
+
(f) => !ENRICHED_FIELDS.includes(f) && !DERIVED_FIELDS.includes(f)
|
|
131
|
+
);
|
|
115
132
|
|
|
116
133
|
// src/query.ts
|
|
117
134
|
function composeQuery(opts) {
|
|
@@ -125,7 +142,9 @@ function mapsSearchUrl(query, opts = {}) {
|
|
|
125
142
|
const hl = (opts.hl ?? "en").split("-")[0] || "en";
|
|
126
143
|
const params = new URLSearchParams({ hl });
|
|
127
144
|
if (opts.gl) params.set("gl", opts.gl.toLowerCase());
|
|
128
|
-
|
|
145
|
+
const base = `https://www.google.com/maps/search/${encodeURIComponent(query)}`;
|
|
146
|
+
const at = opts.center ? `/@${opts.center.lat},${opts.center.lng},${Math.round(opts.center.zoom ?? 14)}z` : "";
|
|
147
|
+
return `${base}${at}?${params.toString()}`;
|
|
129
148
|
}
|
|
130
149
|
function expandQueries(opts) {
|
|
131
150
|
if (opts.query && opts.query.trim()) return [{ query: opts.query.trim() }];
|
|
@@ -283,6 +302,7 @@ function filterLeads(leads, filters = {}) {
|
|
|
283
302
|
if (filters.hasPhone && !l.phone) return false;
|
|
284
303
|
if (filters.hasWebsite && !l.website) return false;
|
|
285
304
|
if (filters.hasEmail && !l.email) return false;
|
|
305
|
+
if (filters.hasValidEmail && l.emailStatus !== "valid") return false;
|
|
286
306
|
return true;
|
|
287
307
|
});
|
|
288
308
|
}
|
|
@@ -290,7 +310,8 @@ function dedupeKey(lead, by) {
|
|
|
290
310
|
if (by === "none") return void 0;
|
|
291
311
|
if (by === "website") return normalizeHost(lead.website);
|
|
292
312
|
if (by === "phone") return lead.phone?.replace(/[^0-9]/g, "") || void 0;
|
|
293
|
-
return lead.name?.trim().toLowerCase() || void 0;
|
|
313
|
+
if (by === "name") return lead.name?.trim().toLowerCase() || void 0;
|
|
314
|
+
return normalizeHost(lead.website) ?? (lead.phone?.replace(/[^0-9]/g, "") || void 0) ?? (lead.name?.trim().toLowerCase() || void 0);
|
|
294
315
|
}
|
|
295
316
|
function normalizeHost(url) {
|
|
296
317
|
if (!url) return void 0;
|
|
@@ -423,12 +444,14 @@ function plausible(digits) {
|
|
|
423
444
|
return digits.length >= 8 && digits.length <= 15;
|
|
424
445
|
}
|
|
425
446
|
var priceRank = (p) => p ? p.replace(/[^$€£₹¥₩]/g, "").length : 0;
|
|
447
|
+
var ASC_BY_DEFAULT = /* @__PURE__ */ new Set(["name", "distance"]);
|
|
426
448
|
function sortLeads(leads, by, dir) {
|
|
427
449
|
if (!by) return [...leads];
|
|
428
|
-
const desc = dir ? dir === "desc" : by
|
|
450
|
+
const desc = dir ? dir === "desc" : !ASC_BY_DEFAULT.has(by);
|
|
429
451
|
const val = (l) => {
|
|
430
452
|
if (by === "name") return l.name?.toLowerCase();
|
|
431
453
|
if (by === "priceLevel") return l.priceLevel ? priceRank(l.priceLevel) : void 0;
|
|
454
|
+
if (by === "distance") return l.distanceKm;
|
|
432
455
|
return l[by];
|
|
433
456
|
};
|
|
434
457
|
return [...leads].map((lead, i) => ({ lead, i })).sort((a, b) => {
|
|
@@ -447,6 +470,196 @@ function sortLeads(leads, by, dir) {
|
|
|
447
470
|
}).map((x) => x.lead);
|
|
448
471
|
}
|
|
449
472
|
|
|
473
|
+
// src/verify.ts
|
|
474
|
+
var import_promises = require("dns/promises");
|
|
475
|
+
var EMAIL_SHAPE = /^[^\s@]+@([^\s@.]+\.)+[^\s@.]{2,}$/;
|
|
476
|
+
function emailFormatValid(email) {
|
|
477
|
+
return !!email && EMAIL_SHAPE.test(email.trim());
|
|
478
|
+
}
|
|
479
|
+
function emailDomain(email) {
|
|
480
|
+
if (!email) return void 0;
|
|
481
|
+
const at = email.lastIndexOf("@");
|
|
482
|
+
return at >= 0 ? email.slice(at + 1).trim().toLowerCase() || void 0 : void 0;
|
|
483
|
+
}
|
|
484
|
+
async function domainHasMail(domain, cache, timeoutMs) {
|
|
485
|
+
const cached = cache.get(domain);
|
|
486
|
+
if (cached !== void 0) return cached;
|
|
487
|
+
const timer = new Promise((res) => setTimeout(() => res(false), timeoutMs));
|
|
488
|
+
const lookup = (0, import_promises.resolveMx)(domain).then((records) => Array.isArray(records) && records.length > 0).catch(() => false);
|
|
489
|
+
const ok = await Promise.race([lookup, timer]);
|
|
490
|
+
cache.set(domain, ok);
|
|
491
|
+
return ok;
|
|
492
|
+
}
|
|
493
|
+
async function verifyEmail(email, opts = {}) {
|
|
494
|
+
if (!email) return "unknown";
|
|
495
|
+
if (!emailFormatValid(email)) return "invalid-format";
|
|
496
|
+
const domain = emailDomain(email);
|
|
497
|
+
if (!domain) return "invalid-format";
|
|
498
|
+
const has = await domainHasMail(domain, opts.cache ?? /* @__PURE__ */ new Map(), opts.timeoutMs ?? 5e3);
|
|
499
|
+
return has ? "valid" : "no-mx";
|
|
500
|
+
}
|
|
501
|
+
async function verifyEmails(leads, opts = {}) {
|
|
502
|
+
const cache = /* @__PURE__ */ new Map();
|
|
503
|
+
const targets = leads.filter((l) => l.email);
|
|
504
|
+
const n = Math.max(1, Math.min(opts.concurrency ?? 5, targets.length || 1));
|
|
505
|
+
let next = 0;
|
|
506
|
+
let done = 0;
|
|
507
|
+
const verifyOpts = { cache };
|
|
508
|
+
if (opts.timeoutMs !== void 0) verifyOpts.timeoutMs = opts.timeoutMs;
|
|
509
|
+
const workers = Array.from({ length: n }, async () => {
|
|
510
|
+
for (; ; ) {
|
|
511
|
+
const i = next++;
|
|
512
|
+
if (i >= targets.length) return;
|
|
513
|
+
const lead = targets[i];
|
|
514
|
+
lead.emailStatus = await verifyEmail(lead.email, verifyOpts);
|
|
515
|
+
opts.onProgress?.(`verifying email ${++done}/${targets.length}: ${lead.email}\u2026`);
|
|
516
|
+
}
|
|
517
|
+
});
|
|
518
|
+
await Promise.all(workers);
|
|
519
|
+
return leads;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// src/geo.ts
|
|
523
|
+
var R_EARTH_M = 63710088e-1;
|
|
524
|
+
var toRad = (deg) => deg * Math.PI / 180;
|
|
525
|
+
function haversineMeters(a, b) {
|
|
526
|
+
const dLat = toRad(b.lat - a.lat);
|
|
527
|
+
const dLng = toRad(b.lng - a.lng);
|
|
528
|
+
const lat1 = toRad(a.lat);
|
|
529
|
+
const lat2 = toRad(b.lat);
|
|
530
|
+
const h = Math.sin(dLat / 2) ** 2 + Math.cos(lat1) * Math.cos(lat2) * Math.sin(dLng / 2) ** 2;
|
|
531
|
+
return 2 * R_EARTH_M * Math.asin(Math.min(1, Math.sqrt(h)));
|
|
532
|
+
}
|
|
533
|
+
function parseLatLngPair(input) {
|
|
534
|
+
if (!input) return void 0;
|
|
535
|
+
const m = input.replace(/^@/, "").match(/(-?\d+(?:\.\d+)?)\s*[, ]\s*(-?\d+(?:\.\d+)?)/);
|
|
536
|
+
if (!m) return void 0;
|
|
537
|
+
const lat = parseFloat(m[1]);
|
|
538
|
+
const lng = parseFloat(m[2]);
|
|
539
|
+
if (!Number.isFinite(lat) || !Number.isFinite(lng)) return void 0;
|
|
540
|
+
if (lat < -90 || lat > 90 || lng < -180 || lng > 180) return void 0;
|
|
541
|
+
return { lat, lng };
|
|
542
|
+
}
|
|
543
|
+
function parseDistance(input) {
|
|
544
|
+
if (!input) return void 0;
|
|
545
|
+
const m = input.trim().toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(km|m|mi|mile|miles|meters?|metres?)?$/);
|
|
546
|
+
if (!m) return void 0;
|
|
547
|
+
const value = parseFloat(m[1]);
|
|
548
|
+
if (!Number.isFinite(value) || value <= 0) return void 0;
|
|
549
|
+
const unit = m[2] ?? "m";
|
|
550
|
+
const metres = unit === "km" ? value * 1e3 : unit.startsWith("mi") ? value * 1609.344 : value;
|
|
551
|
+
return metres;
|
|
552
|
+
}
|
|
553
|
+
function zoomForRadius(radiusM, lat = 0) {
|
|
554
|
+
const mpp = 2 * Math.max(50, radiusM) / 600;
|
|
555
|
+
const zoom = Math.log2(156543.03392 * Math.cos(toRad(lat)) / mpp);
|
|
556
|
+
return Math.max(3, Math.min(19, Math.round(zoom)));
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// src/export.ts
|
|
560
|
+
var import_csv = require("@lacspace/csv");
|
|
561
|
+
var import_xlsx = require("@lacspace/xlsx");
|
|
562
|
+
var HEADERS = {
|
|
563
|
+
name: "Name",
|
|
564
|
+
category: "Category",
|
|
565
|
+
rating: "Rating",
|
|
566
|
+
reviews: "Reviews",
|
|
567
|
+
priceLevel: "Price",
|
|
568
|
+
address: "Address",
|
|
569
|
+
phone: "Phone",
|
|
570
|
+
website: "Website",
|
|
571
|
+
email: "Email",
|
|
572
|
+
facebook: "Facebook",
|
|
573
|
+
instagram: "Instagram",
|
|
574
|
+
whatsapp: "WhatsApp",
|
|
575
|
+
linkedin: "LinkedIn",
|
|
576
|
+
twitter: "Twitter/X",
|
|
577
|
+
youtube: "YouTube",
|
|
578
|
+
tiktok: "TikTok",
|
|
579
|
+
telegram: "Telegram",
|
|
580
|
+
emailStatus: "Email Status",
|
|
581
|
+
plusCode: "Plus Code",
|
|
582
|
+
latitude: "Latitude",
|
|
583
|
+
longitude: "Longitude",
|
|
584
|
+
distanceKm: "Distance (km)",
|
|
585
|
+
hours: "Hours",
|
|
586
|
+
mapsUrl: "Maps URL"
|
|
587
|
+
};
|
|
588
|
+
function toRows(leads, fields = ALL_FIELDS) {
|
|
589
|
+
return leads.map((lead) => {
|
|
590
|
+
const row = {};
|
|
591
|
+
for (const f of fields) {
|
|
592
|
+
const v = lead[f];
|
|
593
|
+
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
594
|
+
}
|
|
595
|
+
return row;
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
599
|
+
if (format === "json" || format === "ndjson") {
|
|
600
|
+
const picked = leads.map((lead) => {
|
|
601
|
+
const o = {};
|
|
602
|
+
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
603
|
+
return o;
|
|
604
|
+
});
|
|
605
|
+
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
606
|
+
return { data, binary: false };
|
|
607
|
+
}
|
|
608
|
+
const rows = toRows(leads, fields);
|
|
609
|
+
if (format === "csv") {
|
|
610
|
+
const csvRows = rows;
|
|
611
|
+
return { data: (0, import_csv.stringify)(csvRows, { escapeFormulas: true }), binary: false };
|
|
612
|
+
}
|
|
613
|
+
const columns = fields.map((f) => {
|
|
614
|
+
const header = HEADERS[f];
|
|
615
|
+
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
616
|
+
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
617
|
+
});
|
|
618
|
+
return {
|
|
619
|
+
data: (0, import_xlsx.jsonToXlsx)(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
620
|
+
binary: true
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
function computeStats(leads) {
|
|
624
|
+
const socialKeys = ["facebook", "instagram", "whatsapp", "linkedin", "twitter", "youtube", "tiktok", "telegram"];
|
|
625
|
+
const ratings = leads.map((l) => l.rating).filter((r) => typeof r === "number");
|
|
626
|
+
const stats = {
|
|
627
|
+
total: leads.length,
|
|
628
|
+
withPhone: leads.filter((l) => l.phone).length,
|
|
629
|
+
withWebsite: leads.filter((l) => l.website).length,
|
|
630
|
+
withEmail: leads.filter((l) => l.email).length,
|
|
631
|
+
withValidEmail: leads.filter((l) => l.emailStatus === "valid").length,
|
|
632
|
+
withSocial: leads.filter((l) => socialKeys.some((k) => l[k])).length
|
|
633
|
+
};
|
|
634
|
+
if (ratings.length) stats.avgRating = Math.round(ratings.reduce((a, b) => a + b, 0) / ratings.length * 10) / 10;
|
|
635
|
+
return stats;
|
|
636
|
+
}
|
|
637
|
+
var FIELD_BY_KEY = (() => {
|
|
638
|
+
const m = /* @__PURE__ */ new Map();
|
|
639
|
+
for (const f of ALL_FIELDS) {
|
|
640
|
+
m.set(f.toLowerCase(), f);
|
|
641
|
+
m.set(HEADERS[f].toLowerCase(), f);
|
|
642
|
+
}
|
|
643
|
+
return m;
|
|
644
|
+
})();
|
|
645
|
+
var NUMERIC_FIELDS = /* @__PURE__ */ new Set(["rating", "reviews", "latitude", "longitude", "distanceKm"]);
|
|
646
|
+
function rowsToLeads(rows) {
|
|
647
|
+
return rows.map((row) => {
|
|
648
|
+
const lead = {};
|
|
649
|
+
for (const [key, raw] of Object.entries(row)) {
|
|
650
|
+
const field = FIELD_BY_KEY.get(key.trim().toLowerCase());
|
|
651
|
+
if (!field || raw === null || raw === void 0 || raw === "") continue;
|
|
652
|
+
if (NUMERIC_FIELDS.has(field)) {
|
|
653
|
+
const n = typeof raw === "number" ? raw : parseFloat(String(raw));
|
|
654
|
+
if (Number.isFinite(n)) lead[field] = n;
|
|
655
|
+
} else {
|
|
656
|
+
lead[field] = String(raw);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
return lead;
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
|
|
450
663
|
// src/scrape.ts
|
|
451
664
|
async function pool(items, n, fn, signal) {
|
|
452
665
|
let next = 0;
|
|
@@ -496,17 +709,19 @@ function parseLatLng(url) {
|
|
|
496
709
|
if (at) return { latitude: parseFloat(at[1]), longitude: parseFloat(at[2]) };
|
|
497
710
|
return {};
|
|
498
711
|
}
|
|
499
|
-
async function launchBrowser(headless) {
|
|
712
|
+
async function launchBrowser(headless, proxy) {
|
|
500
713
|
let lastErr;
|
|
714
|
+
const base = { headless };
|
|
715
|
+
if (proxy) base.proxy = { server: proxy };
|
|
501
716
|
for (const channel of ["chrome", "msedge"]) {
|
|
502
717
|
try {
|
|
503
|
-
return await import_playwright_core.chromium.launch({
|
|
718
|
+
return await import_playwright_core.chromium.launch({ ...base, channel });
|
|
504
719
|
} catch (e) {
|
|
505
720
|
lastErr = e;
|
|
506
721
|
}
|
|
507
722
|
}
|
|
508
723
|
try {
|
|
509
|
-
return await import_playwright_core.chromium.launch(
|
|
724
|
+
return await import_playwright_core.chromium.launch(base);
|
|
510
725
|
} catch (e) {
|
|
511
726
|
lastErr = e;
|
|
512
727
|
}
|
|
@@ -629,17 +844,26 @@ async function scrapeLeads(opts) {
|
|
|
629
844
|
const fields = new Set(normalizeFields(opts.fields ?? DEFAULT_FIELDS));
|
|
630
845
|
const delayMs = Math.max(0, Math.trunc(opts.delayMs ?? 700));
|
|
631
846
|
const headless = opts.headless ?? false;
|
|
847
|
+
const retries = Math.max(0, Math.trunc(opts.retries ?? 1));
|
|
632
848
|
const onProgress = opts.onProgress;
|
|
849
|
+
const onLead = opts.onLead;
|
|
633
850
|
const signal = opts.signal;
|
|
634
851
|
const startedAt = Date.now();
|
|
635
852
|
const overBudget = () => opts.maxMs !== void 0 && Date.now() - startedAt > opts.maxMs;
|
|
636
|
-
const
|
|
853
|
+
const pauseMs = () => opts.jitter ? Math.round(delayMs * (0.6 + Math.random() * 0.8)) : delayMs;
|
|
854
|
+
const wantVerify = Boolean(opts.verifyEmails) || Boolean(opts.filters?.hasValidEmail) || fields.has("emailStatus");
|
|
855
|
+
const wantEnrich = Boolean(opts.enrich) || ENRICHED_FIELDS.some((f) => fields.has(f)) || Boolean(opts.filters?.hasEmail) || wantVerify;
|
|
856
|
+
const near = opts.near;
|
|
637
857
|
const collect = new Set(fields);
|
|
638
858
|
if (wantEnrich) collect.add("website");
|
|
859
|
+
if (near) {
|
|
860
|
+
collect.add("latitude");
|
|
861
|
+
collect.add("longitude");
|
|
862
|
+
}
|
|
639
863
|
const detailOnly = ["name", "mapsUrl"];
|
|
640
864
|
const wantDetails = (opts.details ?? true) && ([...collect].some((f) => !detailOnly.includes(f)) || wantEnrich);
|
|
641
865
|
onProgress?.(`searching Google Maps for "${query}"\u2026`);
|
|
642
|
-
const browser = await launchBrowser(headless);
|
|
866
|
+
const browser = await launchBrowser(headless, opts.proxy);
|
|
643
867
|
try {
|
|
644
868
|
const locale = opts.locale ?? "en-US";
|
|
645
869
|
const context = await browser.newContext({
|
|
@@ -649,6 +873,7 @@ async function scrapeLeads(opts) {
|
|
|
649
873
|
const page = await context.newPage();
|
|
650
874
|
const urlOpts = { hl: locale };
|
|
651
875
|
if (opts.region) urlOpts.gl = opts.region;
|
|
876
|
+
if (near) urlOpts.center = { lat: near.lat, lng: near.lng, zoom: zoomForRadius(opts.radiusM ?? 2e3, near.lat) };
|
|
652
877
|
await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
653
878
|
await dismissConsent(page);
|
|
654
879
|
await loadResults(page, limit, delayMs, onProgress, signal);
|
|
@@ -677,25 +902,47 @@ async function scrapeLeads(opts) {
|
|
|
677
902
|
}
|
|
678
903
|
return lead;
|
|
679
904
|
});
|
|
905
|
+
for (const lead of leads) {
|
|
906
|
+
try {
|
|
907
|
+
onLead?.(lead);
|
|
908
|
+
} catch {
|
|
909
|
+
}
|
|
910
|
+
}
|
|
680
911
|
} else {
|
|
681
912
|
for (let i = 0; i < cards.length; i++) {
|
|
682
913
|
if (signal?.aborted || overBudget()) break;
|
|
683
914
|
const card = cards[i];
|
|
684
915
|
onProgress?.(`reading ${i + 1}/${cards.length}: ${card.name ?? "listing"}\u2026`);
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
916
|
+
let lead;
|
|
917
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
918
|
+
try {
|
|
919
|
+
await page.goto(card.href, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
920
|
+
await page.locator("h1.DUwDvf").first().waitFor({ timeout: 8e3 }).catch(() => {
|
|
921
|
+
});
|
|
922
|
+
lead = await extractDetail(page, collect, card.name);
|
|
923
|
+
break;
|
|
924
|
+
} catch {
|
|
925
|
+
if (attempt < retries && !signal?.aborted) {
|
|
926
|
+
onProgress?.(` retrying ${i + 1}/${cards.length} (attempt ${attempt + 2})\u2026`);
|
|
927
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
if (!lead) {
|
|
691
932
|
if (card.name && (collect.has("name") || collect.has("mapsUrl"))) {
|
|
692
|
-
|
|
693
|
-
if (collect.has("name"))
|
|
694
|
-
if (collect.has("mapsUrl"))
|
|
695
|
-
|
|
933
|
+
lead = {};
|
|
934
|
+
if (collect.has("name")) lead.name = card.name;
|
|
935
|
+
if (collect.has("mapsUrl")) lead.mapsUrl = card.href;
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
if (lead) {
|
|
939
|
+
leads.push(lead);
|
|
940
|
+
try {
|
|
941
|
+
onLead?.(lead);
|
|
942
|
+
} catch {
|
|
696
943
|
}
|
|
697
944
|
}
|
|
698
|
-
if (delayMs) await page.waitForTimeout(
|
|
945
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
699
946
|
}
|
|
700
947
|
}
|
|
701
948
|
onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
|
|
@@ -704,6 +951,19 @@ async function scrapeLeads(opts) {
|
|
|
704
951
|
if (lead.website) lead.website = cleanWebsite(lead.website);
|
|
705
952
|
}
|
|
706
953
|
}
|
|
954
|
+
if (near) {
|
|
955
|
+
for (const lead of leads) {
|
|
956
|
+
if (typeof lead.latitude === "number" && typeof lead.longitude === "number") {
|
|
957
|
+
const m = haversineMeters(near, { lat: lead.latitude, lng: lead.longitude });
|
|
958
|
+
lead.distanceKm = Math.round(m / 1e3 * 100) / 100;
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
if (opts.radiusM !== void 0) {
|
|
962
|
+
const r = opts.radiusM;
|
|
963
|
+
leads = leads.filter((l) => l.distanceKm !== void 0 && l.distanceKm * 1e3 <= r);
|
|
964
|
+
onProgress?.(`within ${Math.round(r)} m: ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
707
967
|
leads = dedupeLeads(leads, opts.dedupe ?? "website");
|
|
708
968
|
if (wantEnrich) {
|
|
709
969
|
const socialFields = ENRICHED_FIELDS.filter((f) => f !== "email");
|
|
@@ -728,6 +988,14 @@ async function scrapeLeads(opts) {
|
|
|
728
988
|
signal
|
|
729
989
|
);
|
|
730
990
|
}
|
|
991
|
+
if (wantVerify && !signal?.aborted) {
|
|
992
|
+
onProgress?.("verifying email domains\u2026");
|
|
993
|
+
const vOpts = {
|
|
994
|
+
concurrency: Math.max(1, Math.trunc(opts.concurrency ?? 5))
|
|
995
|
+
};
|
|
996
|
+
if (onProgress) vOpts.onProgress = (m) => onProgress(m);
|
|
997
|
+
await verifyEmails(leads, vOpts);
|
|
998
|
+
}
|
|
731
999
|
if (opts.country && fields.has("phone")) {
|
|
732
1000
|
for (const lead of leads) {
|
|
733
1001
|
if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
|
|
@@ -735,6 +1003,17 @@ async function scrapeLeads(opts) {
|
|
|
735
1003
|
}
|
|
736
1004
|
if (opts.filters) leads = filterLeads(leads, opts.filters);
|
|
737
1005
|
if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
|
|
1006
|
+
if (wantVerify && !fields.has("emailStatus")) for (const l of leads) delete l.emailStatus;
|
|
1007
|
+
if (near) {
|
|
1008
|
+
const keepLat = fields.has("latitude");
|
|
1009
|
+
const keepLng = fields.has("longitude");
|
|
1010
|
+
const keepDist = fields.has("distanceKm");
|
|
1011
|
+
for (const l of leads) {
|
|
1012
|
+
if (!keepLat) delete l.latitude;
|
|
1013
|
+
if (!keepLng) delete l.longitude;
|
|
1014
|
+
if (!keepDist) delete l.distanceKm;
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
738
1017
|
onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
|
|
739
1018
|
return leads;
|
|
740
1019
|
} finally {
|
|
@@ -742,6 +1021,11 @@ async function scrapeLeads(opts) {
|
|
|
742
1021
|
});
|
|
743
1022
|
}
|
|
744
1023
|
}
|
|
1024
|
+
async function searchLeadsDetailed(opts) {
|
|
1025
|
+
const started = Date.now();
|
|
1026
|
+
const leads = await scrapeLeads(opts);
|
|
1027
|
+
return { leads, stats: computeStats(leads), elapsedMs: Date.now() - started };
|
|
1028
|
+
}
|
|
745
1029
|
|
|
746
1030
|
// src/batch.ts
|
|
747
1031
|
async function searchLeadsBatch(queries, opts = {}) {
|
|
@@ -806,71 +1090,8 @@ async function searchLeadsMulti(opts) {
|
|
|
806
1090
|
return searchLeadsBatch(queries, opts);
|
|
807
1091
|
}
|
|
808
1092
|
|
|
809
|
-
// src/export.ts
|
|
810
|
-
var import_csv = require("@lacspace/csv");
|
|
811
|
-
var import_xlsx = require("@lacspace/xlsx");
|
|
812
|
-
var HEADERS = {
|
|
813
|
-
name: "Name",
|
|
814
|
-
category: "Category",
|
|
815
|
-
rating: "Rating",
|
|
816
|
-
reviews: "Reviews",
|
|
817
|
-
priceLevel: "Price",
|
|
818
|
-
address: "Address",
|
|
819
|
-
phone: "Phone",
|
|
820
|
-
website: "Website",
|
|
821
|
-
email: "Email",
|
|
822
|
-
facebook: "Facebook",
|
|
823
|
-
instagram: "Instagram",
|
|
824
|
-
whatsapp: "WhatsApp",
|
|
825
|
-
linkedin: "LinkedIn",
|
|
826
|
-
twitter: "Twitter/X",
|
|
827
|
-
youtube: "YouTube",
|
|
828
|
-
tiktok: "TikTok",
|
|
829
|
-
telegram: "Telegram",
|
|
830
|
-
plusCode: "Plus Code",
|
|
831
|
-
latitude: "Latitude",
|
|
832
|
-
longitude: "Longitude",
|
|
833
|
-
hours: "Hours",
|
|
834
|
-
mapsUrl: "Maps URL"
|
|
835
|
-
};
|
|
836
|
-
function toRows(leads, fields = ALL_FIELDS) {
|
|
837
|
-
return leads.map((lead) => {
|
|
838
|
-
const row = {};
|
|
839
|
-
for (const f of fields) {
|
|
840
|
-
const v = lead[f];
|
|
841
|
-
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
842
|
-
}
|
|
843
|
-
return row;
|
|
844
|
-
});
|
|
845
|
-
}
|
|
846
|
-
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
847
|
-
if (format === "json" || format === "ndjson") {
|
|
848
|
-
const picked = leads.map((lead) => {
|
|
849
|
-
const o = {};
|
|
850
|
-
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
851
|
-
return o;
|
|
852
|
-
});
|
|
853
|
-
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
854
|
-
return { data, binary: false };
|
|
855
|
-
}
|
|
856
|
-
const rows = toRows(leads, fields);
|
|
857
|
-
if (format === "csv") {
|
|
858
|
-
const csvRows = rows;
|
|
859
|
-
return { data: (0, import_csv.stringify)(csvRows, { escapeFormulas: true }), binary: false };
|
|
860
|
-
}
|
|
861
|
-
const columns = fields.map((f) => {
|
|
862
|
-
const header = HEADERS[f];
|
|
863
|
-
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
864
|
-
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
865
|
-
});
|
|
866
|
-
return {
|
|
867
|
-
data: (0, import_xlsx.jsonToXlsx)(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
868
|
-
binary: true
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
|
|
872
1093
|
// src/convert.ts
|
|
873
|
-
var
|
|
1094
|
+
var import_promises2 = require("fs/promises");
|
|
874
1095
|
var import_node_path = require("path");
|
|
875
1096
|
var import_csv2 = require("@lacspace/csv");
|
|
876
1097
|
var import_xlsx2 = require("@lacspace/xlsx");
|
|
@@ -884,10 +1105,10 @@ function detectFormat(file) {
|
|
|
884
1105
|
async function readRows(file, format) {
|
|
885
1106
|
const fmt = format ?? detectFormat(file);
|
|
886
1107
|
if (fmt === "xlsx") {
|
|
887
|
-
const bytes = await (0,
|
|
1108
|
+
const bytes = await (0, import_promises2.readFile)(file);
|
|
888
1109
|
return await (0, import_xlsx2.xlsxToJson)(bytes);
|
|
889
1110
|
}
|
|
890
|
-
const text = await (0,
|
|
1111
|
+
const text = await (0, import_promises2.readFile)(file, "utf8");
|
|
891
1112
|
if (fmt === "csv") {
|
|
892
1113
|
return (0, import_csv2.parse)(text, { header: true });
|
|
893
1114
|
}
|
|
@@ -939,7 +1160,7 @@ async function convertFile(input, opts = {}) {
|
|
|
939
1160
|
const format = opts.format ?? (opts.out ? detectFormat(opts.out) : "json");
|
|
940
1161
|
const out = (0, import_node_path.resolve)(opts.out ?? input.replace(/\.[^.]+$/, "") + "." + format);
|
|
941
1162
|
const { data, binary } = serializeRows(rows, format, { sheetName: opts.sheetName ?? "Sheet1" });
|
|
942
|
-
await (0,
|
|
1163
|
+
await (0, import_promises2.writeFile)(out, binary ? Buffer.from(data) : data);
|
|
943
1164
|
return { out, format, count: rows.length };
|
|
944
1165
|
}
|
|
945
1166
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -954,29 +1175,40 @@ async function convertFile(input, opts = {}) {
|
|
|
954
1175
|
cleanWebsite,
|
|
955
1176
|
columnsOf,
|
|
956
1177
|
composeQuery,
|
|
1178
|
+
computeStats,
|
|
957
1179
|
convertFile,
|
|
958
1180
|
dedupeLeads,
|
|
959
1181
|
defaultFilename,
|
|
960
1182
|
detectFormat,
|
|
1183
|
+
emailDomain,
|
|
1184
|
+
emailFormatValid,
|
|
961
1185
|
enrichContacts,
|
|
962
1186
|
expandQueries,
|
|
963
1187
|
extractEmails,
|
|
964
1188
|
extractSocials,
|
|
965
1189
|
filterLeads,
|
|
1190
|
+
haversineMeters,
|
|
966
1191
|
mapsSearchUrl,
|
|
967
1192
|
normalizeFields,
|
|
968
1193
|
normalizePhone,
|
|
1194
|
+
parseDistance,
|
|
969
1195
|
parseLatLng,
|
|
1196
|
+
parseLatLngPair,
|
|
970
1197
|
parseRating,
|
|
971
1198
|
parseReviewCount,
|
|
972
1199
|
readRows,
|
|
973
1200
|
resolvePreset,
|
|
1201
|
+
rowsToLeads,
|
|
974
1202
|
scrapeLeads,
|
|
975
1203
|
searchLeads,
|
|
976
1204
|
searchLeadsBatch,
|
|
1205
|
+
searchLeadsDetailed,
|
|
977
1206
|
searchLeadsMulti,
|
|
978
1207
|
serialize,
|
|
979
1208
|
serializeRows,
|
|
980
1209
|
sortLeads,
|
|
981
|
-
toRows
|
|
1210
|
+
toRows,
|
|
1211
|
+
verifyEmail,
|
|
1212
|
+
verifyEmails,
|
|
1213
|
+
zoomForRadius
|
|
982
1214
|
});
|