lacspace-leads 0.3.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -6
- package/dist/cli.js +271 -94
- package/dist/lib.cjs +236 -87
- package/dist/lib.d.cts +83 -4
- package/dist/lib.d.ts +83 -4
- package/dist/lib.js +224 -82
- package/package.json +4 -2
package/dist/lib.cjs
CHANGED
|
@@ -30,10 +30,13 @@ __export(lib_exports, {
|
|
|
30
30
|
cleanWebsite: () => cleanWebsite,
|
|
31
31
|
columnsOf: () => columnsOf,
|
|
32
32
|
composeQuery: () => composeQuery,
|
|
33
|
+
computeStats: () => computeStats,
|
|
33
34
|
convertFile: () => convertFile,
|
|
34
35
|
dedupeLeads: () => dedupeLeads,
|
|
35
36
|
defaultFilename: () => defaultFilename,
|
|
36
37
|
detectFormat: () => detectFormat,
|
|
38
|
+
emailDomain: () => emailDomain,
|
|
39
|
+
emailFormatValid: () => emailFormatValid,
|
|
37
40
|
enrichContacts: () => enrichContacts,
|
|
38
41
|
expandQueries: () => expandQueries,
|
|
39
42
|
extractEmails: () => extractEmails,
|
|
@@ -47,14 +50,18 @@ __export(lib_exports, {
|
|
|
47
50
|
parseReviewCount: () => parseReviewCount,
|
|
48
51
|
readRows: () => readRows,
|
|
49
52
|
resolvePreset: () => resolvePreset,
|
|
53
|
+
rowsToLeads: () => rowsToLeads,
|
|
50
54
|
scrapeLeads: () => scrapeLeads,
|
|
51
55
|
searchLeads: () => scrapeLeads,
|
|
52
56
|
searchLeadsBatch: () => searchLeadsBatch,
|
|
57
|
+
searchLeadsDetailed: () => searchLeadsDetailed,
|
|
53
58
|
searchLeadsMulti: () => searchLeadsMulti,
|
|
54
59
|
serialize: () => serialize,
|
|
55
60
|
serializeRows: () => serializeRows,
|
|
56
61
|
sortLeads: () => sortLeads,
|
|
57
|
-
toRows: () => toRows
|
|
62
|
+
toRows: () => toRows,
|
|
63
|
+
verifyEmail: () => verifyEmail,
|
|
64
|
+
verifyEmails: () => verifyEmails
|
|
58
65
|
});
|
|
59
66
|
module.exports = __toCommonJS(lib_exports);
|
|
60
67
|
|
|
@@ -80,6 +87,7 @@ var ALL_FIELDS = [
|
|
|
80
87
|
"youtube",
|
|
81
88
|
"tiktok",
|
|
82
89
|
"telegram",
|
|
90
|
+
"emailStatus",
|
|
83
91
|
"plusCode",
|
|
84
92
|
"latitude",
|
|
85
93
|
"longitude",
|
|
@@ -95,7 +103,8 @@ var ENRICHED_FIELDS = [
|
|
|
95
103
|
"twitter",
|
|
96
104
|
"youtube",
|
|
97
105
|
"tiktok",
|
|
98
|
-
"telegram"
|
|
106
|
+
"telegram",
|
|
107
|
+
"emailStatus"
|
|
99
108
|
];
|
|
100
109
|
var FIELD_PRESETS = {
|
|
101
110
|
/** Just enough to place a business: name, phone, website. */
|
|
@@ -283,6 +292,7 @@ function filterLeads(leads, filters = {}) {
|
|
|
283
292
|
if (filters.hasPhone && !l.phone) return false;
|
|
284
293
|
if (filters.hasWebsite && !l.website) return false;
|
|
285
294
|
if (filters.hasEmail && !l.email) return false;
|
|
295
|
+
if (filters.hasValidEmail && l.emailStatus !== "valid") return false;
|
|
286
296
|
return true;
|
|
287
297
|
});
|
|
288
298
|
}
|
|
@@ -290,7 +300,8 @@ function dedupeKey(lead, by) {
|
|
|
290
300
|
if (by === "none") return void 0;
|
|
291
301
|
if (by === "website") return normalizeHost(lead.website);
|
|
292
302
|
if (by === "phone") return lead.phone?.replace(/[^0-9]/g, "") || void 0;
|
|
293
|
-
return lead.name?.trim().toLowerCase() || void 0;
|
|
303
|
+
if (by === "name") return lead.name?.trim().toLowerCase() || void 0;
|
|
304
|
+
return normalizeHost(lead.website) ?? (lead.phone?.replace(/[^0-9]/g, "") || void 0) ?? (lead.name?.trim().toLowerCase() || void 0);
|
|
294
305
|
}
|
|
295
306
|
function normalizeHost(url) {
|
|
296
307
|
if (!url) return void 0;
|
|
@@ -447,6 +458,158 @@ function sortLeads(leads, by, dir) {
|
|
|
447
458
|
}).map((x) => x.lead);
|
|
448
459
|
}
|
|
449
460
|
|
|
461
|
+
// src/verify.ts
|
|
462
|
+
var import_promises = require("dns/promises");
|
|
463
|
+
var EMAIL_SHAPE = /^[^\s@]+@([^\s@.]+\.)+[^\s@.]{2,}$/;
|
|
464
|
+
function emailFormatValid(email) {
|
|
465
|
+
return !!email && EMAIL_SHAPE.test(email.trim());
|
|
466
|
+
}
|
|
467
|
+
function emailDomain(email) {
|
|
468
|
+
if (!email) return void 0;
|
|
469
|
+
const at = email.lastIndexOf("@");
|
|
470
|
+
return at >= 0 ? email.slice(at + 1).trim().toLowerCase() || void 0 : void 0;
|
|
471
|
+
}
|
|
472
|
+
async function domainHasMail(domain, cache, timeoutMs) {
|
|
473
|
+
const cached = cache.get(domain);
|
|
474
|
+
if (cached !== void 0) return cached;
|
|
475
|
+
const timer = new Promise((res) => setTimeout(() => res(false), timeoutMs));
|
|
476
|
+
const lookup = (0, import_promises.resolveMx)(domain).then((records) => Array.isArray(records) && records.length > 0).catch(() => false);
|
|
477
|
+
const ok = await Promise.race([lookup, timer]);
|
|
478
|
+
cache.set(domain, ok);
|
|
479
|
+
return ok;
|
|
480
|
+
}
|
|
481
|
+
async function verifyEmail(email, opts = {}) {
|
|
482
|
+
if (!email) return "unknown";
|
|
483
|
+
if (!emailFormatValid(email)) return "invalid-format";
|
|
484
|
+
const domain = emailDomain(email);
|
|
485
|
+
if (!domain) return "invalid-format";
|
|
486
|
+
const has = await domainHasMail(domain, opts.cache ?? /* @__PURE__ */ new Map(), opts.timeoutMs ?? 5e3);
|
|
487
|
+
return has ? "valid" : "no-mx";
|
|
488
|
+
}
|
|
489
|
+
async function verifyEmails(leads, opts = {}) {
|
|
490
|
+
const cache = /* @__PURE__ */ new Map();
|
|
491
|
+
const targets = leads.filter((l) => l.email);
|
|
492
|
+
const n = Math.max(1, Math.min(opts.concurrency ?? 5, targets.length || 1));
|
|
493
|
+
let next = 0;
|
|
494
|
+
let done = 0;
|
|
495
|
+
const verifyOpts = { cache };
|
|
496
|
+
if (opts.timeoutMs !== void 0) verifyOpts.timeoutMs = opts.timeoutMs;
|
|
497
|
+
const workers = Array.from({ length: n }, async () => {
|
|
498
|
+
for (; ; ) {
|
|
499
|
+
const i = next++;
|
|
500
|
+
if (i >= targets.length) return;
|
|
501
|
+
const lead = targets[i];
|
|
502
|
+
lead.emailStatus = await verifyEmail(lead.email, verifyOpts);
|
|
503
|
+
opts.onProgress?.(`verifying email ${++done}/${targets.length}: ${lead.email}\u2026`);
|
|
504
|
+
}
|
|
505
|
+
});
|
|
506
|
+
await Promise.all(workers);
|
|
507
|
+
return leads;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// src/export.ts
|
|
511
|
+
var import_csv = require("@lacspace/csv");
|
|
512
|
+
var import_xlsx = require("@lacspace/xlsx");
|
|
513
|
+
var HEADERS = {
|
|
514
|
+
name: "Name",
|
|
515
|
+
category: "Category",
|
|
516
|
+
rating: "Rating",
|
|
517
|
+
reviews: "Reviews",
|
|
518
|
+
priceLevel: "Price",
|
|
519
|
+
address: "Address",
|
|
520
|
+
phone: "Phone",
|
|
521
|
+
website: "Website",
|
|
522
|
+
email: "Email",
|
|
523
|
+
facebook: "Facebook",
|
|
524
|
+
instagram: "Instagram",
|
|
525
|
+
whatsapp: "WhatsApp",
|
|
526
|
+
linkedin: "LinkedIn",
|
|
527
|
+
twitter: "Twitter/X",
|
|
528
|
+
youtube: "YouTube",
|
|
529
|
+
tiktok: "TikTok",
|
|
530
|
+
telegram: "Telegram",
|
|
531
|
+
emailStatus: "Email Status",
|
|
532
|
+
plusCode: "Plus Code",
|
|
533
|
+
latitude: "Latitude",
|
|
534
|
+
longitude: "Longitude",
|
|
535
|
+
hours: "Hours",
|
|
536
|
+
mapsUrl: "Maps URL"
|
|
537
|
+
};
|
|
538
|
+
function toRows(leads, fields = ALL_FIELDS) {
|
|
539
|
+
return leads.map((lead) => {
|
|
540
|
+
const row = {};
|
|
541
|
+
for (const f of fields) {
|
|
542
|
+
const v = lead[f];
|
|
543
|
+
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
544
|
+
}
|
|
545
|
+
return row;
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
549
|
+
if (format === "json" || format === "ndjson") {
|
|
550
|
+
const picked = leads.map((lead) => {
|
|
551
|
+
const o = {};
|
|
552
|
+
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
553
|
+
return o;
|
|
554
|
+
});
|
|
555
|
+
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
556
|
+
return { data, binary: false };
|
|
557
|
+
}
|
|
558
|
+
const rows = toRows(leads, fields);
|
|
559
|
+
if (format === "csv") {
|
|
560
|
+
const csvRows = rows;
|
|
561
|
+
return { data: (0, import_csv.stringify)(csvRows, { escapeFormulas: true }), binary: false };
|
|
562
|
+
}
|
|
563
|
+
const columns = fields.map((f) => {
|
|
564
|
+
const header = HEADERS[f];
|
|
565
|
+
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
566
|
+
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
567
|
+
});
|
|
568
|
+
return {
|
|
569
|
+
data: (0, import_xlsx.jsonToXlsx)(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
570
|
+
binary: true
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
function computeStats(leads) {
|
|
574
|
+
const socialKeys = ["facebook", "instagram", "whatsapp", "linkedin", "twitter", "youtube", "tiktok", "telegram"];
|
|
575
|
+
const ratings = leads.map((l) => l.rating).filter((r) => typeof r === "number");
|
|
576
|
+
const stats = {
|
|
577
|
+
total: leads.length,
|
|
578
|
+
withPhone: leads.filter((l) => l.phone).length,
|
|
579
|
+
withWebsite: leads.filter((l) => l.website).length,
|
|
580
|
+
withEmail: leads.filter((l) => l.email).length,
|
|
581
|
+
withValidEmail: leads.filter((l) => l.emailStatus === "valid").length,
|
|
582
|
+
withSocial: leads.filter((l) => socialKeys.some((k) => l[k])).length
|
|
583
|
+
};
|
|
584
|
+
if (ratings.length) stats.avgRating = Math.round(ratings.reduce((a, b) => a + b, 0) / ratings.length * 10) / 10;
|
|
585
|
+
return stats;
|
|
586
|
+
}
|
|
587
|
+
var FIELD_BY_KEY = (() => {
|
|
588
|
+
const m = /* @__PURE__ */ new Map();
|
|
589
|
+
for (const f of ALL_FIELDS) {
|
|
590
|
+
m.set(f.toLowerCase(), f);
|
|
591
|
+
m.set(HEADERS[f].toLowerCase(), f);
|
|
592
|
+
}
|
|
593
|
+
return m;
|
|
594
|
+
})();
|
|
595
|
+
var NUMERIC_FIELDS = /* @__PURE__ */ new Set(["rating", "reviews", "latitude", "longitude"]);
|
|
596
|
+
function rowsToLeads(rows) {
|
|
597
|
+
return rows.map((row) => {
|
|
598
|
+
const lead = {};
|
|
599
|
+
for (const [key, raw] of Object.entries(row)) {
|
|
600
|
+
const field = FIELD_BY_KEY.get(key.trim().toLowerCase());
|
|
601
|
+
if (!field || raw === null || raw === void 0 || raw === "") continue;
|
|
602
|
+
if (NUMERIC_FIELDS.has(field)) {
|
|
603
|
+
const n = typeof raw === "number" ? raw : parseFloat(String(raw));
|
|
604
|
+
if (Number.isFinite(n)) lead[field] = n;
|
|
605
|
+
} else {
|
|
606
|
+
lead[field] = String(raw);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return lead;
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
|
|
450
613
|
// src/scrape.ts
|
|
451
614
|
async function pool(items, n, fn, signal) {
|
|
452
615
|
let next = 0;
|
|
@@ -496,17 +659,19 @@ function parseLatLng(url) {
|
|
|
496
659
|
if (at) return { latitude: parseFloat(at[1]), longitude: parseFloat(at[2]) };
|
|
497
660
|
return {};
|
|
498
661
|
}
|
|
499
|
-
async function launchBrowser(headless) {
|
|
662
|
+
async function launchBrowser(headless, proxy) {
|
|
500
663
|
let lastErr;
|
|
664
|
+
const base = { headless };
|
|
665
|
+
if (proxy) base.proxy = { server: proxy };
|
|
501
666
|
for (const channel of ["chrome", "msedge"]) {
|
|
502
667
|
try {
|
|
503
|
-
return await import_playwright_core.chromium.launch({
|
|
668
|
+
return await import_playwright_core.chromium.launch({ ...base, channel });
|
|
504
669
|
} catch (e) {
|
|
505
670
|
lastErr = e;
|
|
506
671
|
}
|
|
507
672
|
}
|
|
508
673
|
try {
|
|
509
|
-
return await import_playwright_core.chromium.launch(
|
|
674
|
+
return await import_playwright_core.chromium.launch(base);
|
|
510
675
|
} catch (e) {
|
|
511
676
|
lastErr = e;
|
|
512
677
|
}
|
|
@@ -629,17 +794,21 @@ async function scrapeLeads(opts) {
|
|
|
629
794
|
const fields = new Set(normalizeFields(opts.fields ?? DEFAULT_FIELDS));
|
|
630
795
|
const delayMs = Math.max(0, Math.trunc(opts.delayMs ?? 700));
|
|
631
796
|
const headless = opts.headless ?? false;
|
|
797
|
+
const retries = Math.max(0, Math.trunc(opts.retries ?? 1));
|
|
632
798
|
const onProgress = opts.onProgress;
|
|
799
|
+
const onLead = opts.onLead;
|
|
633
800
|
const signal = opts.signal;
|
|
634
801
|
const startedAt = Date.now();
|
|
635
802
|
const overBudget = () => opts.maxMs !== void 0 && Date.now() - startedAt > opts.maxMs;
|
|
636
|
-
const
|
|
803
|
+
const pauseMs = () => opts.jitter ? Math.round(delayMs * (0.6 + Math.random() * 0.8)) : delayMs;
|
|
804
|
+
const wantVerify = Boolean(opts.verifyEmails) || Boolean(opts.filters?.hasValidEmail) || fields.has("emailStatus");
|
|
805
|
+
const wantEnrich = Boolean(opts.enrich) || ENRICHED_FIELDS.some((f) => fields.has(f)) || Boolean(opts.filters?.hasEmail) || wantVerify;
|
|
637
806
|
const collect = new Set(fields);
|
|
638
807
|
if (wantEnrich) collect.add("website");
|
|
639
808
|
const detailOnly = ["name", "mapsUrl"];
|
|
640
809
|
const wantDetails = (opts.details ?? true) && ([...collect].some((f) => !detailOnly.includes(f)) || wantEnrich);
|
|
641
810
|
onProgress?.(`searching Google Maps for "${query}"\u2026`);
|
|
642
|
-
const browser = await launchBrowser(headless);
|
|
811
|
+
const browser = await launchBrowser(headless, opts.proxy);
|
|
643
812
|
try {
|
|
644
813
|
const locale = opts.locale ?? "en-US";
|
|
645
814
|
const context = await browser.newContext({
|
|
@@ -677,25 +846,47 @@ async function scrapeLeads(opts) {
|
|
|
677
846
|
}
|
|
678
847
|
return lead;
|
|
679
848
|
});
|
|
849
|
+
for (const lead of leads) {
|
|
850
|
+
try {
|
|
851
|
+
onLead?.(lead);
|
|
852
|
+
} catch {
|
|
853
|
+
}
|
|
854
|
+
}
|
|
680
855
|
} else {
|
|
681
856
|
for (let i = 0; i < cards.length; i++) {
|
|
682
857
|
if (signal?.aborted || overBudget()) break;
|
|
683
858
|
const card = cards[i];
|
|
684
859
|
onProgress?.(`reading ${i + 1}/${cards.length}: ${card.name ?? "listing"}\u2026`);
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
860
|
+
let lead;
|
|
861
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
862
|
+
try {
|
|
863
|
+
await page.goto(card.href, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
864
|
+
await page.locator("h1.DUwDvf").first().waitFor({ timeout: 8e3 }).catch(() => {
|
|
865
|
+
});
|
|
866
|
+
lead = await extractDetail(page, collect, card.name);
|
|
867
|
+
break;
|
|
868
|
+
} catch {
|
|
869
|
+
if (attempt < retries && !signal?.aborted) {
|
|
870
|
+
onProgress?.(` retrying ${i + 1}/${cards.length} (attempt ${attempt + 2})\u2026`);
|
|
871
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
if (!lead) {
|
|
691
876
|
if (card.name && (collect.has("name") || collect.has("mapsUrl"))) {
|
|
692
|
-
|
|
693
|
-
if (collect.has("name"))
|
|
694
|
-
if (collect.has("mapsUrl"))
|
|
695
|
-
|
|
877
|
+
lead = {};
|
|
878
|
+
if (collect.has("name")) lead.name = card.name;
|
|
879
|
+
if (collect.has("mapsUrl")) lead.mapsUrl = card.href;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
if (lead) {
|
|
883
|
+
leads.push(lead);
|
|
884
|
+
try {
|
|
885
|
+
onLead?.(lead);
|
|
886
|
+
} catch {
|
|
696
887
|
}
|
|
697
888
|
}
|
|
698
|
-
if (delayMs) await page.waitForTimeout(
|
|
889
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
699
890
|
}
|
|
700
891
|
}
|
|
701
892
|
onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
|
|
@@ -728,6 +919,14 @@ async function scrapeLeads(opts) {
|
|
|
728
919
|
signal
|
|
729
920
|
);
|
|
730
921
|
}
|
|
922
|
+
if (wantVerify && !signal?.aborted) {
|
|
923
|
+
onProgress?.("verifying email domains\u2026");
|
|
924
|
+
const vOpts = {
|
|
925
|
+
concurrency: Math.max(1, Math.trunc(opts.concurrency ?? 5))
|
|
926
|
+
};
|
|
927
|
+
if (onProgress) vOpts.onProgress = (m) => onProgress(m);
|
|
928
|
+
await verifyEmails(leads, vOpts);
|
|
929
|
+
}
|
|
731
930
|
if (opts.country && fields.has("phone")) {
|
|
732
931
|
for (const lead of leads) {
|
|
733
932
|
if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
|
|
@@ -735,6 +934,7 @@ async function scrapeLeads(opts) {
|
|
|
735
934
|
}
|
|
736
935
|
if (opts.filters) leads = filterLeads(leads, opts.filters);
|
|
737
936
|
if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
|
|
937
|
+
if (wantVerify && !fields.has("emailStatus")) for (const l of leads) delete l.emailStatus;
|
|
738
938
|
onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
|
|
739
939
|
return leads;
|
|
740
940
|
} finally {
|
|
@@ -742,6 +942,11 @@ async function scrapeLeads(opts) {
|
|
|
742
942
|
});
|
|
743
943
|
}
|
|
744
944
|
}
|
|
945
|
+
async function searchLeadsDetailed(opts) {
|
|
946
|
+
const started = Date.now();
|
|
947
|
+
const leads = await scrapeLeads(opts);
|
|
948
|
+
return { leads, stats: computeStats(leads), elapsedMs: Date.now() - started };
|
|
949
|
+
}
|
|
745
950
|
|
|
746
951
|
// src/batch.ts
|
|
747
952
|
async function searchLeadsBatch(queries, opts = {}) {
|
|
@@ -806,71 +1011,8 @@ async function searchLeadsMulti(opts) {
|
|
|
806
1011
|
return searchLeadsBatch(queries, opts);
|
|
807
1012
|
}
|
|
808
1013
|
|
|
809
|
-
// src/export.ts
|
|
810
|
-
var import_csv = require("@lacspace/csv");
|
|
811
|
-
var import_xlsx = require("@lacspace/xlsx");
|
|
812
|
-
var HEADERS = {
|
|
813
|
-
name: "Name",
|
|
814
|
-
category: "Category",
|
|
815
|
-
rating: "Rating",
|
|
816
|
-
reviews: "Reviews",
|
|
817
|
-
priceLevel: "Price",
|
|
818
|
-
address: "Address",
|
|
819
|
-
phone: "Phone",
|
|
820
|
-
website: "Website",
|
|
821
|
-
email: "Email",
|
|
822
|
-
facebook: "Facebook",
|
|
823
|
-
instagram: "Instagram",
|
|
824
|
-
whatsapp: "WhatsApp",
|
|
825
|
-
linkedin: "LinkedIn",
|
|
826
|
-
twitter: "Twitter/X",
|
|
827
|
-
youtube: "YouTube",
|
|
828
|
-
tiktok: "TikTok",
|
|
829
|
-
telegram: "Telegram",
|
|
830
|
-
plusCode: "Plus Code",
|
|
831
|
-
latitude: "Latitude",
|
|
832
|
-
longitude: "Longitude",
|
|
833
|
-
hours: "Hours",
|
|
834
|
-
mapsUrl: "Maps URL"
|
|
835
|
-
};
|
|
836
|
-
function toRows(leads, fields = ALL_FIELDS) {
|
|
837
|
-
return leads.map((lead) => {
|
|
838
|
-
const row = {};
|
|
839
|
-
for (const f of fields) {
|
|
840
|
-
const v = lead[f];
|
|
841
|
-
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
842
|
-
}
|
|
843
|
-
return row;
|
|
844
|
-
});
|
|
845
|
-
}
|
|
846
|
-
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
847
|
-
if (format === "json" || format === "ndjson") {
|
|
848
|
-
const picked = leads.map((lead) => {
|
|
849
|
-
const o = {};
|
|
850
|
-
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
851
|
-
return o;
|
|
852
|
-
});
|
|
853
|
-
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
854
|
-
return { data, binary: false };
|
|
855
|
-
}
|
|
856
|
-
const rows = toRows(leads, fields);
|
|
857
|
-
if (format === "csv") {
|
|
858
|
-
const csvRows = rows;
|
|
859
|
-
return { data: (0, import_csv.stringify)(csvRows, { escapeFormulas: true }), binary: false };
|
|
860
|
-
}
|
|
861
|
-
const columns = fields.map((f) => {
|
|
862
|
-
const header = HEADERS[f];
|
|
863
|
-
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
864
|
-
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
865
|
-
});
|
|
866
|
-
return {
|
|
867
|
-
data: (0, import_xlsx.jsonToXlsx)(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
868
|
-
binary: true
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
|
|
872
1014
|
// src/convert.ts
|
|
873
|
-
var
|
|
1015
|
+
var import_promises2 = require("fs/promises");
|
|
874
1016
|
var import_node_path = require("path");
|
|
875
1017
|
var import_csv2 = require("@lacspace/csv");
|
|
876
1018
|
var import_xlsx2 = require("@lacspace/xlsx");
|
|
@@ -884,10 +1026,10 @@ function detectFormat(file) {
|
|
|
884
1026
|
async function readRows(file, format) {
|
|
885
1027
|
const fmt = format ?? detectFormat(file);
|
|
886
1028
|
if (fmt === "xlsx") {
|
|
887
|
-
const bytes = await (0,
|
|
1029
|
+
const bytes = await (0, import_promises2.readFile)(file);
|
|
888
1030
|
return await (0, import_xlsx2.xlsxToJson)(bytes);
|
|
889
1031
|
}
|
|
890
|
-
const text = await (0,
|
|
1032
|
+
const text = await (0, import_promises2.readFile)(file, "utf8");
|
|
891
1033
|
if (fmt === "csv") {
|
|
892
1034
|
return (0, import_csv2.parse)(text, { header: true });
|
|
893
1035
|
}
|
|
@@ -939,7 +1081,7 @@ async function convertFile(input, opts = {}) {
|
|
|
939
1081
|
const format = opts.format ?? (opts.out ? detectFormat(opts.out) : "json");
|
|
940
1082
|
const out = (0, import_node_path.resolve)(opts.out ?? input.replace(/\.[^.]+$/, "") + "." + format);
|
|
941
1083
|
const { data, binary } = serializeRows(rows, format, { sheetName: opts.sheetName ?? "Sheet1" });
|
|
942
|
-
await (0,
|
|
1084
|
+
await (0, import_promises2.writeFile)(out, binary ? Buffer.from(data) : data);
|
|
943
1085
|
return { out, format, count: rows.length };
|
|
944
1086
|
}
|
|
945
1087
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -954,10 +1096,13 @@ async function convertFile(input, opts = {}) {
|
|
|
954
1096
|
cleanWebsite,
|
|
955
1097
|
columnsOf,
|
|
956
1098
|
composeQuery,
|
|
1099
|
+
computeStats,
|
|
957
1100
|
convertFile,
|
|
958
1101
|
dedupeLeads,
|
|
959
1102
|
defaultFilename,
|
|
960
1103
|
detectFormat,
|
|
1104
|
+
emailDomain,
|
|
1105
|
+
emailFormatValid,
|
|
961
1106
|
enrichContacts,
|
|
962
1107
|
expandQueries,
|
|
963
1108
|
extractEmails,
|
|
@@ -971,12 +1116,16 @@ async function convertFile(input, opts = {}) {
|
|
|
971
1116
|
parseReviewCount,
|
|
972
1117
|
readRows,
|
|
973
1118
|
resolvePreset,
|
|
1119
|
+
rowsToLeads,
|
|
974
1120
|
scrapeLeads,
|
|
975
1121
|
searchLeads,
|
|
976
1122
|
searchLeadsBatch,
|
|
1123
|
+
searchLeadsDetailed,
|
|
977
1124
|
searchLeadsMulti,
|
|
978
1125
|
serialize,
|
|
979
1126
|
serializeRows,
|
|
980
1127
|
sortLeads,
|
|
981
|
-
toRows
|
|
1128
|
+
toRows,
|
|
1129
|
+
verifyEmail,
|
|
1130
|
+
verifyEmails
|
|
982
1131
|
});
|
package/dist/lib.d.cts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** The fields a {@link Lead} can carry — request any subset via `fields`. */
|
|
2
|
-
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
2
|
+
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "emailStatus" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
3
|
+
/** Deliverability verdict for a discovered email (see `verifyEmails`). */
|
|
4
|
+
type EmailStatus = "valid" | "no-mx" | "invalid-format" | "unknown";
|
|
3
5
|
/** Every field, in a sensible column order for exports. */
|
|
4
6
|
declare const ALL_FIELDS: LeadField[];
|
|
5
7
|
/** Fields that require visiting the business website (via `enrich`). */
|
|
@@ -43,6 +45,8 @@ interface Lead {
|
|
|
43
45
|
tiktok?: string;
|
|
44
46
|
/** Telegram link, from the website (needs enrichment). */
|
|
45
47
|
telegram?: string;
|
|
48
|
+
/** Email deliverability verdict (set by `verifyEmails`). */
|
|
49
|
+
emailStatus?: EmailStatus;
|
|
46
50
|
/** Google Plus Code, when shown. */
|
|
47
51
|
plusCode?: string;
|
|
48
52
|
/** Latitude, parsed from the listing's Maps URL. */
|
|
@@ -66,6 +70,8 @@ interface LeadFilters {
|
|
|
66
70
|
hasWebsite?: boolean;
|
|
67
71
|
/** Keep only leads that have an email (implies enrichment). */
|
|
68
72
|
hasEmail?: boolean;
|
|
73
|
+
/** Keep only leads whose email passed MX verification (implies `verifyEmails`). */
|
|
74
|
+
hasValidEmail?: boolean;
|
|
69
75
|
}
|
|
70
76
|
/** Output formats the tool can write. */
|
|
71
77
|
type OutputFormat = "json" | "ndjson" | "csv" | "xlsx";
|
|
@@ -103,8 +109,12 @@ interface SearchOptions {
|
|
|
103
109
|
enrich?: boolean;
|
|
104
110
|
/** Drop leads that don't pass these filters. */
|
|
105
111
|
filters?: LeadFilters;
|
|
106
|
-
/**
|
|
107
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Drop duplicate leads by this key. `"smart"` uses the strongest identity
|
|
114
|
+
* available per lead (website → phone → name) and is best for accumulating
|
|
115
|
+
* across runs. Default "website".
|
|
116
|
+
*/
|
|
117
|
+
dedupe?: "website" | "phone" | "name" | "smart" | "none";
|
|
108
118
|
/** Sort the results by this key before returning. */
|
|
109
119
|
sort?: SortKey$1;
|
|
110
120
|
/** Sort direction. Defaults to "desc" for numbers, "asc" for name. */
|
|
@@ -122,17 +132,43 @@ interface SearchOptions {
|
|
|
122
132
|
cleanUrls?: boolean;
|
|
123
133
|
/** How many websites to enrich in parallel. Default 3. */
|
|
124
134
|
concurrency?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Verify each discovered email by checking its domain has MX records (DNS,
|
|
137
|
+
* no message is sent). Populates `emailStatus`. Implies `enrich`. Default off.
|
|
138
|
+
*/
|
|
139
|
+
verifyEmails?: boolean;
|
|
125
140
|
/** Browser/UI locale, e.g. "en-US", "ne-NP". Default "en-US". */
|
|
126
141
|
locale?: string;
|
|
127
142
|
/** Google region bias (ccTLD-style), e.g. "np", "us". Sets Maps `gl`. */
|
|
128
143
|
region?: string;
|
|
144
|
+
/** Route the browser through this proxy, e.g. "http://user:pass@host:port". */
|
|
145
|
+
proxy?: string;
|
|
146
|
+
/** Retry a listing that fails to open, up to this many times. Default 1. */
|
|
147
|
+
retries?: number;
|
|
148
|
+
/** Randomise the delay between listings by ±40% to look more human. */
|
|
149
|
+
jitter?: boolean;
|
|
129
150
|
/** Stop collecting after this many milliseconds (best-effort). */
|
|
130
151
|
maxMs?: number;
|
|
131
152
|
/** Called with a short progress message as the search runs. */
|
|
132
153
|
onProgress?: (message: string) => void;
|
|
154
|
+
/**
|
|
155
|
+
* Called with each lead as it's collected (before enrichment), for live UIs
|
|
156
|
+
* or crash-safe incremental writing. Never throws into the run.
|
|
157
|
+
*/
|
|
158
|
+
onLead?: (lead: Lead) => void;
|
|
133
159
|
/** An AbortSignal to cancel a running search. */
|
|
134
160
|
signal?: AbortSignal;
|
|
135
161
|
}
|
|
162
|
+
/** Aggregate counts over a lead list — see `computeStats`. */
|
|
163
|
+
interface LeadStats {
|
|
164
|
+
total: number;
|
|
165
|
+
withPhone: number;
|
|
166
|
+
withWebsite: number;
|
|
167
|
+
withEmail: number;
|
|
168
|
+
withValidEmail: number;
|
|
169
|
+
withSocial: number;
|
|
170
|
+
avgRating?: number;
|
|
171
|
+
}
|
|
136
172
|
|
|
137
173
|
/** Error thrown by the scraper. Carries a machine `code` and optional cause. */
|
|
138
174
|
declare class LeadsError extends Error {
|
|
@@ -162,6 +198,18 @@ declare function parseLatLng(url: string): {
|
|
|
162
198
|
* @throws {LeadsError} when no browser can be launched, or the search is aborted.
|
|
163
199
|
*/
|
|
164
200
|
declare function scrapeLeads(opts: SearchOptions): Promise<Lead[]>;
|
|
201
|
+
/** A search result with aggregate stats and timing. */
|
|
202
|
+
interface DetailedResult {
|
|
203
|
+
leads: Lead[];
|
|
204
|
+
stats: LeadStats;
|
|
205
|
+
/** Wall-clock time of the search, in milliseconds. */
|
|
206
|
+
elapsedMs: number;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Like {@link scrapeLeads}, but also returns aggregate {@link LeadStats} and the
|
|
210
|
+
* elapsed time — handy for dashboards, reports and CI summaries.
|
|
211
|
+
*/
|
|
212
|
+
declare function searchLeadsDetailed(opts: SearchOptions): Promise<DetailedResult>;
|
|
165
213
|
|
|
166
214
|
/** One sub-search in a batch. */
|
|
167
215
|
interface BatchQuery {
|
|
@@ -198,6 +246,37 @@ declare function serialize(leads: Lead[], format: OutputFormat, fields?: LeadFie
|
|
|
198
246
|
data: string | Uint8Array;
|
|
199
247
|
binary: boolean;
|
|
200
248
|
};
|
|
249
|
+
/** Aggregate counts over a lead list — for summaries and reports. Pure. */
|
|
250
|
+
declare function computeStats(leads: Lead[]): LeadStats;
|
|
251
|
+
/**
|
|
252
|
+
* Map arbitrary rows (as read back from a JSON/CSV/Excel export, keyed by field
|
|
253
|
+
* name or header label) into {@link Lead}s. Unknown columns are ignored, blanks
|
|
254
|
+
* dropped, and numeric fields coerced. Powers `--append` / resume. Pure.
|
|
255
|
+
*/
|
|
256
|
+
declare function rowsToLeads(rows: Record<string, unknown>[]): Lead[];
|
|
257
|
+
|
|
258
|
+
/** True when `email` is a plausibly-valid address shape. Pure. */
|
|
259
|
+
declare function emailFormatValid(email: string | undefined): boolean;
|
|
260
|
+
/** The domain part of an email, lower-cased, or undefined. Pure. */
|
|
261
|
+
declare function emailDomain(email: string | undefined): string | undefined;
|
|
262
|
+
/**
|
|
263
|
+
* Classify a single email: `invalid-format` → `no-mx` → `valid`. `unknown` is
|
|
264
|
+
* reserved for a missing address. Never throws.
|
|
265
|
+
*/
|
|
266
|
+
declare function verifyEmail(email: string | undefined, opts?: {
|
|
267
|
+
cache?: Map<string, boolean>;
|
|
268
|
+
timeoutMs?: number;
|
|
269
|
+
}): Promise<EmailStatus>;
|
|
270
|
+
/**
|
|
271
|
+
* Verify the `email` on each lead in place, setting `emailStatus`. Domains are
|
|
272
|
+
* looked up once (shared cache) and checked a few in parallel. Leads without an
|
|
273
|
+
* email are left untouched. Node-only; never throws.
|
|
274
|
+
*/
|
|
275
|
+
declare function verifyEmails(leads: Lead[], opts?: {
|
|
276
|
+
concurrency?: number;
|
|
277
|
+
timeoutMs?: number;
|
|
278
|
+
onProgress?: (m: string) => void;
|
|
279
|
+
}): Promise<Lead[]>;
|
|
201
280
|
|
|
202
281
|
/**
|
|
203
282
|
* Compose the human search query from options: an explicit `query` wins,
|
|
@@ -346,4 +425,4 @@ declare function convertFile(input: string, opts?: {
|
|
|
346
425
|
count: number;
|
|
347
426
|
}>;
|
|
348
427
|
|
|
349
|
-
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, ENRICHED_FIELDS, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, convertFile, dedupeLeads, defaultFilename, detectFormat, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows };
|
|
428
|
+
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, type DetailedResult, ENRICHED_FIELDS, type EmailStatus, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, type LeadStats, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, computeStats, convertFile, dedupeLeads, defaultFilename, detectFormat, emailDomain, emailFormatValid, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, rowsToLeads, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsDetailed, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows, verifyEmail, verifyEmails };
|