lacspace-leads 0.3.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -6
- package/dist/cli.js +271 -94
- package/dist/lib.cjs +236 -87
- package/dist/lib.d.cts +83 -4
- package/dist/lib.d.ts +83 -4
- package/dist/lib.js +224 -82
- package/package.json +4 -2
package/dist/lib.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** The fields a {@link Lead} can carry — request any subset via `fields`. */
|
|
2
|
-
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
2
|
+
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "emailStatus" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
3
|
+
/** Deliverability verdict for a discovered email (see `verifyEmails`). */
|
|
4
|
+
type EmailStatus = "valid" | "no-mx" | "invalid-format" | "unknown";
|
|
3
5
|
/** Every field, in a sensible column order for exports. */
|
|
4
6
|
declare const ALL_FIELDS: LeadField[];
|
|
5
7
|
/** Fields that require visiting the business website (via `enrich`). */
|
|
@@ -43,6 +45,8 @@ interface Lead {
|
|
|
43
45
|
tiktok?: string;
|
|
44
46
|
/** Telegram link, from the website (needs enrichment). */
|
|
45
47
|
telegram?: string;
|
|
48
|
+
/** Email deliverability verdict (set by `verifyEmails`). */
|
|
49
|
+
emailStatus?: EmailStatus;
|
|
46
50
|
/** Google Plus Code, when shown. */
|
|
47
51
|
plusCode?: string;
|
|
48
52
|
/** Latitude, parsed from the listing's Maps URL. */
|
|
@@ -66,6 +70,8 @@ interface LeadFilters {
|
|
|
66
70
|
hasWebsite?: boolean;
|
|
67
71
|
/** Keep only leads that have an email (implies enrichment). */
|
|
68
72
|
hasEmail?: boolean;
|
|
73
|
+
/** Keep only leads whose email passed MX verification (implies `verifyEmails`). */
|
|
74
|
+
hasValidEmail?: boolean;
|
|
69
75
|
}
|
|
70
76
|
/** Output formats the tool can write. */
|
|
71
77
|
type OutputFormat = "json" | "ndjson" | "csv" | "xlsx";
|
|
@@ -103,8 +109,12 @@ interface SearchOptions {
|
|
|
103
109
|
enrich?: boolean;
|
|
104
110
|
/** Drop leads that don't pass these filters. */
|
|
105
111
|
filters?: LeadFilters;
|
|
106
|
-
/**
|
|
107
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Drop duplicate leads by this key. `"smart"` uses the strongest identity
|
|
114
|
+
* available per lead (website → phone → name) and is best for accumulating
|
|
115
|
+
* across runs. Default "website".
|
|
116
|
+
*/
|
|
117
|
+
dedupe?: "website" | "phone" | "name" | "smart" | "none";
|
|
108
118
|
/** Sort the results by this key before returning. */
|
|
109
119
|
sort?: SortKey$1;
|
|
110
120
|
/** Sort direction. Defaults to "desc" for numbers, "asc" for name. */
|
|
@@ -122,17 +132,43 @@ interface SearchOptions {
|
|
|
122
132
|
cleanUrls?: boolean;
|
|
123
133
|
/** How many websites to enrich in parallel. Default 3. */
|
|
124
134
|
concurrency?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Verify each discovered email by checking its domain has MX records (DNS,
|
|
137
|
+
* no message is sent). Populates `emailStatus`. Implies `enrich`. Default off.
|
|
138
|
+
*/
|
|
139
|
+
verifyEmails?: boolean;
|
|
125
140
|
/** Browser/UI locale, e.g. "en-US", "ne-NP". Default "en-US". */
|
|
126
141
|
locale?: string;
|
|
127
142
|
/** Google region bias (ccTLD-style), e.g. "np", "us". Sets Maps `gl`. */
|
|
128
143
|
region?: string;
|
|
144
|
+
/** Route the browser through this proxy, e.g. "http://user:pass@host:port". */
|
|
145
|
+
proxy?: string;
|
|
146
|
+
/** Retry a listing that fails to open, up to this many times. Default 1. */
|
|
147
|
+
retries?: number;
|
|
148
|
+
/** Randomise the delay between listings by ±40% to look more human. */
|
|
149
|
+
jitter?: boolean;
|
|
129
150
|
/** Stop collecting after this many milliseconds (best-effort). */
|
|
130
151
|
maxMs?: number;
|
|
131
152
|
/** Called with a short progress message as the search runs. */
|
|
132
153
|
onProgress?: (message: string) => void;
|
|
154
|
+
/**
|
|
155
|
+
* Called with each lead as it's collected (before enrichment), for live UIs
|
|
156
|
+
* or crash-safe incremental writing. Never throws into the run.
|
|
157
|
+
*/
|
|
158
|
+
onLead?: (lead: Lead) => void;
|
|
133
159
|
/** An AbortSignal to cancel a running search. */
|
|
134
160
|
signal?: AbortSignal;
|
|
135
161
|
}
|
|
162
|
+
/** Aggregate counts over a lead list — see `computeStats`. */
|
|
163
|
+
interface LeadStats {
|
|
164
|
+
total: number;
|
|
165
|
+
withPhone: number;
|
|
166
|
+
withWebsite: number;
|
|
167
|
+
withEmail: number;
|
|
168
|
+
withValidEmail: number;
|
|
169
|
+
withSocial: number;
|
|
170
|
+
avgRating?: number;
|
|
171
|
+
}
|
|
136
172
|
|
|
137
173
|
/** Error thrown by the scraper. Carries a machine `code` and optional cause. */
|
|
138
174
|
declare class LeadsError extends Error {
|
|
@@ -162,6 +198,18 @@ declare function parseLatLng(url: string): {
|
|
|
162
198
|
* @throws {LeadsError} when no browser can be launched, or the search is aborted.
|
|
163
199
|
*/
|
|
164
200
|
declare function scrapeLeads(opts: SearchOptions): Promise<Lead[]>;
|
|
201
|
+
/** A search result with aggregate stats and timing. */
|
|
202
|
+
interface DetailedResult {
|
|
203
|
+
leads: Lead[];
|
|
204
|
+
stats: LeadStats;
|
|
205
|
+
/** Wall-clock time of the search, in milliseconds. */
|
|
206
|
+
elapsedMs: number;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Like {@link scrapeLeads}, but also returns aggregate {@link LeadStats} and the
|
|
210
|
+
* elapsed time — handy for dashboards, reports and CI summaries.
|
|
211
|
+
*/
|
|
212
|
+
declare function searchLeadsDetailed(opts: SearchOptions): Promise<DetailedResult>;
|
|
165
213
|
|
|
166
214
|
/** One sub-search in a batch. */
|
|
167
215
|
interface BatchQuery {
|
|
@@ -198,6 +246,37 @@ declare function serialize(leads: Lead[], format: OutputFormat, fields?: LeadFie
|
|
|
198
246
|
data: string | Uint8Array;
|
|
199
247
|
binary: boolean;
|
|
200
248
|
};
|
|
249
|
+
/** Aggregate counts over a lead list — for summaries and reports. Pure. */
|
|
250
|
+
declare function computeStats(leads: Lead[]): LeadStats;
|
|
251
|
+
/**
|
|
252
|
+
* Map arbitrary rows (as read back from a JSON/CSV/Excel export, keyed by field
|
|
253
|
+
* name or header label) into {@link Lead}s. Unknown columns are ignored, blanks
|
|
254
|
+
* dropped, and numeric fields coerced. Powers `--append` / resume. Pure.
|
|
255
|
+
*/
|
|
256
|
+
declare function rowsToLeads(rows: Record<string, unknown>[]): Lead[];
|
|
257
|
+
|
|
258
|
+
/** True when `email` is a plausibly-valid address shape. Pure. */
|
|
259
|
+
declare function emailFormatValid(email: string | undefined): boolean;
|
|
260
|
+
/** The domain part of an email, lower-cased, or undefined. Pure. */
|
|
261
|
+
declare function emailDomain(email: string | undefined): string | undefined;
|
|
262
|
+
/**
|
|
263
|
+
* Classify a single email: `invalid-format` → `no-mx` → `valid`. `unknown` is
|
|
264
|
+
* reserved for a missing address. Never throws.
|
|
265
|
+
*/
|
|
266
|
+
declare function verifyEmail(email: string | undefined, opts?: {
|
|
267
|
+
cache?: Map<string, boolean>;
|
|
268
|
+
timeoutMs?: number;
|
|
269
|
+
}): Promise<EmailStatus>;
|
|
270
|
+
/**
|
|
271
|
+
* Verify the `email` on each lead in place, setting `emailStatus`. Domains are
|
|
272
|
+
* looked up once (shared cache) and checked a few in parallel. Leads without an
|
|
273
|
+
* email are left untouched. Node-only; never throws.
|
|
274
|
+
*/
|
|
275
|
+
declare function verifyEmails(leads: Lead[], opts?: {
|
|
276
|
+
concurrency?: number;
|
|
277
|
+
timeoutMs?: number;
|
|
278
|
+
onProgress?: (m: string) => void;
|
|
279
|
+
}): Promise<Lead[]>;
|
|
201
280
|
|
|
202
281
|
/**
|
|
203
282
|
* Compose the human search query from options: an explicit `query` wins,
|
|
@@ -346,4 +425,4 @@ declare function convertFile(input: string, opts?: {
|
|
|
346
425
|
count: number;
|
|
347
426
|
}>;
|
|
348
427
|
|
|
349
|
-
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, ENRICHED_FIELDS, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, convertFile, dedupeLeads, defaultFilename, detectFormat, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows };
|
|
428
|
+
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, type DetailedResult, ENRICHED_FIELDS, type EmailStatus, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, type LeadStats, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, computeStats, convertFile, dedupeLeads, defaultFilename, detectFormat, emailDomain, emailFormatValid, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, rowsToLeads, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsDetailed, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows, verifyEmail, verifyEmails };
|
package/dist/lib.js
CHANGED
|
@@ -20,6 +20,7 @@ var ALL_FIELDS = [
|
|
|
20
20
|
"youtube",
|
|
21
21
|
"tiktok",
|
|
22
22
|
"telegram",
|
|
23
|
+
"emailStatus",
|
|
23
24
|
"plusCode",
|
|
24
25
|
"latitude",
|
|
25
26
|
"longitude",
|
|
@@ -35,7 +36,8 @@ var ENRICHED_FIELDS = [
|
|
|
35
36
|
"twitter",
|
|
36
37
|
"youtube",
|
|
37
38
|
"tiktok",
|
|
38
|
-
"telegram"
|
|
39
|
+
"telegram",
|
|
40
|
+
"emailStatus"
|
|
39
41
|
];
|
|
40
42
|
var FIELD_PRESETS = {
|
|
41
43
|
/** Just enough to place a business: name, phone, website. */
|
|
@@ -223,6 +225,7 @@ function filterLeads(leads, filters = {}) {
|
|
|
223
225
|
if (filters.hasPhone && !l.phone) return false;
|
|
224
226
|
if (filters.hasWebsite && !l.website) return false;
|
|
225
227
|
if (filters.hasEmail && !l.email) return false;
|
|
228
|
+
if (filters.hasValidEmail && l.emailStatus !== "valid") return false;
|
|
226
229
|
return true;
|
|
227
230
|
});
|
|
228
231
|
}
|
|
@@ -230,7 +233,8 @@ function dedupeKey(lead, by) {
|
|
|
230
233
|
if (by === "none") return void 0;
|
|
231
234
|
if (by === "website") return normalizeHost(lead.website);
|
|
232
235
|
if (by === "phone") return lead.phone?.replace(/[^0-9]/g, "") || void 0;
|
|
233
|
-
return lead.name?.trim().toLowerCase() || void 0;
|
|
236
|
+
if (by === "name") return lead.name?.trim().toLowerCase() || void 0;
|
|
237
|
+
return normalizeHost(lead.website) ?? (lead.phone?.replace(/[^0-9]/g, "") || void 0) ?? (lead.name?.trim().toLowerCase() || void 0);
|
|
234
238
|
}
|
|
235
239
|
function normalizeHost(url) {
|
|
236
240
|
if (!url) return void 0;
|
|
@@ -387,6 +391,158 @@ function sortLeads(leads, by, dir) {
|
|
|
387
391
|
}).map((x) => x.lead);
|
|
388
392
|
}
|
|
389
393
|
|
|
394
|
+
// src/verify.ts
|
|
395
|
+
import { resolveMx } from "dns/promises";
|
|
396
|
+
var EMAIL_SHAPE = /^[^\s@]+@([^\s@.]+\.)+[^\s@.]{2,}$/;
|
|
397
|
+
function emailFormatValid(email) {
|
|
398
|
+
return !!email && EMAIL_SHAPE.test(email.trim());
|
|
399
|
+
}
|
|
400
|
+
function emailDomain(email) {
|
|
401
|
+
if (!email) return void 0;
|
|
402
|
+
const at = email.lastIndexOf("@");
|
|
403
|
+
return at >= 0 ? email.slice(at + 1).trim().toLowerCase() || void 0 : void 0;
|
|
404
|
+
}
|
|
405
|
+
async function domainHasMail(domain, cache, timeoutMs) {
|
|
406
|
+
const cached = cache.get(domain);
|
|
407
|
+
if (cached !== void 0) return cached;
|
|
408
|
+
const timer = new Promise((res) => setTimeout(() => res(false), timeoutMs));
|
|
409
|
+
const lookup = resolveMx(domain).then((records) => Array.isArray(records) && records.length > 0).catch(() => false);
|
|
410
|
+
const ok = await Promise.race([lookup, timer]);
|
|
411
|
+
cache.set(domain, ok);
|
|
412
|
+
return ok;
|
|
413
|
+
}
|
|
414
|
+
async function verifyEmail(email, opts = {}) {
|
|
415
|
+
if (!email) return "unknown";
|
|
416
|
+
if (!emailFormatValid(email)) return "invalid-format";
|
|
417
|
+
const domain = emailDomain(email);
|
|
418
|
+
if (!domain) return "invalid-format";
|
|
419
|
+
const has = await domainHasMail(domain, opts.cache ?? /* @__PURE__ */ new Map(), opts.timeoutMs ?? 5e3);
|
|
420
|
+
return has ? "valid" : "no-mx";
|
|
421
|
+
}
|
|
422
|
+
async function verifyEmails(leads, opts = {}) {
|
|
423
|
+
const cache = /* @__PURE__ */ new Map();
|
|
424
|
+
const targets = leads.filter((l) => l.email);
|
|
425
|
+
const n = Math.max(1, Math.min(opts.concurrency ?? 5, targets.length || 1));
|
|
426
|
+
let next = 0;
|
|
427
|
+
let done = 0;
|
|
428
|
+
const verifyOpts = { cache };
|
|
429
|
+
if (opts.timeoutMs !== void 0) verifyOpts.timeoutMs = opts.timeoutMs;
|
|
430
|
+
const workers = Array.from({ length: n }, async () => {
|
|
431
|
+
for (; ; ) {
|
|
432
|
+
const i = next++;
|
|
433
|
+
if (i >= targets.length) return;
|
|
434
|
+
const lead = targets[i];
|
|
435
|
+
lead.emailStatus = await verifyEmail(lead.email, verifyOpts);
|
|
436
|
+
opts.onProgress?.(`verifying email ${++done}/${targets.length}: ${lead.email}\u2026`);
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
await Promise.all(workers);
|
|
440
|
+
return leads;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// src/export.ts
|
|
444
|
+
import { stringify as csvStringify } from "@lacspace/csv";
|
|
445
|
+
import { jsonToXlsx } from "@lacspace/xlsx";
|
|
446
|
+
var HEADERS = {
|
|
447
|
+
name: "Name",
|
|
448
|
+
category: "Category",
|
|
449
|
+
rating: "Rating",
|
|
450
|
+
reviews: "Reviews",
|
|
451
|
+
priceLevel: "Price",
|
|
452
|
+
address: "Address",
|
|
453
|
+
phone: "Phone",
|
|
454
|
+
website: "Website",
|
|
455
|
+
email: "Email",
|
|
456
|
+
facebook: "Facebook",
|
|
457
|
+
instagram: "Instagram",
|
|
458
|
+
whatsapp: "WhatsApp",
|
|
459
|
+
linkedin: "LinkedIn",
|
|
460
|
+
twitter: "Twitter/X",
|
|
461
|
+
youtube: "YouTube",
|
|
462
|
+
tiktok: "TikTok",
|
|
463
|
+
telegram: "Telegram",
|
|
464
|
+
emailStatus: "Email Status",
|
|
465
|
+
plusCode: "Plus Code",
|
|
466
|
+
latitude: "Latitude",
|
|
467
|
+
longitude: "Longitude",
|
|
468
|
+
hours: "Hours",
|
|
469
|
+
mapsUrl: "Maps URL"
|
|
470
|
+
};
|
|
471
|
+
function toRows(leads, fields = ALL_FIELDS) {
|
|
472
|
+
return leads.map((lead) => {
|
|
473
|
+
const row = {};
|
|
474
|
+
for (const f of fields) {
|
|
475
|
+
const v = lead[f];
|
|
476
|
+
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
477
|
+
}
|
|
478
|
+
return row;
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
482
|
+
if (format === "json" || format === "ndjson") {
|
|
483
|
+
const picked = leads.map((lead) => {
|
|
484
|
+
const o = {};
|
|
485
|
+
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
486
|
+
return o;
|
|
487
|
+
});
|
|
488
|
+
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
489
|
+
return { data, binary: false };
|
|
490
|
+
}
|
|
491
|
+
const rows = toRows(leads, fields);
|
|
492
|
+
if (format === "csv") {
|
|
493
|
+
const csvRows = rows;
|
|
494
|
+
return { data: csvStringify(csvRows, { escapeFormulas: true }), binary: false };
|
|
495
|
+
}
|
|
496
|
+
const columns = fields.map((f) => {
|
|
497
|
+
const header = HEADERS[f];
|
|
498
|
+
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
499
|
+
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
500
|
+
});
|
|
501
|
+
return {
|
|
502
|
+
data: jsonToXlsx(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
503
|
+
binary: true
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
function computeStats(leads) {
|
|
507
|
+
const socialKeys = ["facebook", "instagram", "whatsapp", "linkedin", "twitter", "youtube", "tiktok", "telegram"];
|
|
508
|
+
const ratings = leads.map((l) => l.rating).filter((r) => typeof r === "number");
|
|
509
|
+
const stats = {
|
|
510
|
+
total: leads.length,
|
|
511
|
+
withPhone: leads.filter((l) => l.phone).length,
|
|
512
|
+
withWebsite: leads.filter((l) => l.website).length,
|
|
513
|
+
withEmail: leads.filter((l) => l.email).length,
|
|
514
|
+
withValidEmail: leads.filter((l) => l.emailStatus === "valid").length,
|
|
515
|
+
withSocial: leads.filter((l) => socialKeys.some((k) => l[k])).length
|
|
516
|
+
};
|
|
517
|
+
if (ratings.length) stats.avgRating = Math.round(ratings.reduce((a, b) => a + b, 0) / ratings.length * 10) / 10;
|
|
518
|
+
return stats;
|
|
519
|
+
}
|
|
520
|
+
var FIELD_BY_KEY = (() => {
|
|
521
|
+
const m = /* @__PURE__ */ new Map();
|
|
522
|
+
for (const f of ALL_FIELDS) {
|
|
523
|
+
m.set(f.toLowerCase(), f);
|
|
524
|
+
m.set(HEADERS[f].toLowerCase(), f);
|
|
525
|
+
}
|
|
526
|
+
return m;
|
|
527
|
+
})();
|
|
528
|
+
var NUMERIC_FIELDS = /* @__PURE__ */ new Set(["rating", "reviews", "latitude", "longitude"]);
|
|
529
|
+
function rowsToLeads(rows) {
|
|
530
|
+
return rows.map((row) => {
|
|
531
|
+
const lead = {};
|
|
532
|
+
for (const [key, raw] of Object.entries(row)) {
|
|
533
|
+
const field = FIELD_BY_KEY.get(key.trim().toLowerCase());
|
|
534
|
+
if (!field || raw === null || raw === void 0 || raw === "") continue;
|
|
535
|
+
if (NUMERIC_FIELDS.has(field)) {
|
|
536
|
+
const n = typeof raw === "number" ? raw : parseFloat(String(raw));
|
|
537
|
+
if (Number.isFinite(n)) lead[field] = n;
|
|
538
|
+
} else {
|
|
539
|
+
lead[field] = String(raw);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
return lead;
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
|
|
390
546
|
// src/scrape.ts
|
|
391
547
|
async function pool(items, n, fn, signal) {
|
|
392
548
|
let next = 0;
|
|
@@ -436,17 +592,19 @@ function parseLatLng(url) {
|
|
|
436
592
|
if (at) return { latitude: parseFloat(at[1]), longitude: parseFloat(at[2]) };
|
|
437
593
|
return {};
|
|
438
594
|
}
|
|
439
|
-
async function launchBrowser(headless) {
|
|
595
|
+
async function launchBrowser(headless, proxy) {
|
|
440
596
|
let lastErr;
|
|
597
|
+
const base = { headless };
|
|
598
|
+
if (proxy) base.proxy = { server: proxy };
|
|
441
599
|
for (const channel of ["chrome", "msedge"]) {
|
|
442
600
|
try {
|
|
443
|
-
return await chromium.launch({
|
|
601
|
+
return await chromium.launch({ ...base, channel });
|
|
444
602
|
} catch (e) {
|
|
445
603
|
lastErr = e;
|
|
446
604
|
}
|
|
447
605
|
}
|
|
448
606
|
try {
|
|
449
|
-
return await chromium.launch(
|
|
607
|
+
return await chromium.launch(base);
|
|
450
608
|
} catch (e) {
|
|
451
609
|
lastErr = e;
|
|
452
610
|
}
|
|
@@ -569,17 +727,21 @@ async function scrapeLeads(opts) {
|
|
|
569
727
|
const fields = new Set(normalizeFields(opts.fields ?? DEFAULT_FIELDS));
|
|
570
728
|
const delayMs = Math.max(0, Math.trunc(opts.delayMs ?? 700));
|
|
571
729
|
const headless = opts.headless ?? false;
|
|
730
|
+
const retries = Math.max(0, Math.trunc(opts.retries ?? 1));
|
|
572
731
|
const onProgress = opts.onProgress;
|
|
732
|
+
const onLead = opts.onLead;
|
|
573
733
|
const signal = opts.signal;
|
|
574
734
|
const startedAt = Date.now();
|
|
575
735
|
const overBudget = () => opts.maxMs !== void 0 && Date.now() - startedAt > opts.maxMs;
|
|
576
|
-
const
|
|
736
|
+
const pauseMs = () => opts.jitter ? Math.round(delayMs * (0.6 + Math.random() * 0.8)) : delayMs;
|
|
737
|
+
const wantVerify = Boolean(opts.verifyEmails) || Boolean(opts.filters?.hasValidEmail) || fields.has("emailStatus");
|
|
738
|
+
const wantEnrich = Boolean(opts.enrich) || ENRICHED_FIELDS.some((f) => fields.has(f)) || Boolean(opts.filters?.hasEmail) || wantVerify;
|
|
577
739
|
const collect = new Set(fields);
|
|
578
740
|
if (wantEnrich) collect.add("website");
|
|
579
741
|
const detailOnly = ["name", "mapsUrl"];
|
|
580
742
|
const wantDetails = (opts.details ?? true) && ([...collect].some((f) => !detailOnly.includes(f)) || wantEnrich);
|
|
581
743
|
onProgress?.(`searching Google Maps for "${query}"\u2026`);
|
|
582
|
-
const browser = await launchBrowser(headless);
|
|
744
|
+
const browser = await launchBrowser(headless, opts.proxy);
|
|
583
745
|
try {
|
|
584
746
|
const locale = opts.locale ?? "en-US";
|
|
585
747
|
const context = await browser.newContext({
|
|
@@ -617,25 +779,47 @@ async function scrapeLeads(opts) {
|
|
|
617
779
|
}
|
|
618
780
|
return lead;
|
|
619
781
|
});
|
|
782
|
+
for (const lead of leads) {
|
|
783
|
+
try {
|
|
784
|
+
onLead?.(lead);
|
|
785
|
+
} catch {
|
|
786
|
+
}
|
|
787
|
+
}
|
|
620
788
|
} else {
|
|
621
789
|
for (let i = 0; i < cards.length; i++) {
|
|
622
790
|
if (signal?.aborted || overBudget()) break;
|
|
623
791
|
const card = cards[i];
|
|
624
792
|
onProgress?.(`reading ${i + 1}/${cards.length}: ${card.name ?? "listing"}\u2026`);
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
793
|
+
let lead;
|
|
794
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
795
|
+
try {
|
|
796
|
+
await page.goto(card.href, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
797
|
+
await page.locator("h1.DUwDvf").first().waitFor({ timeout: 8e3 }).catch(() => {
|
|
798
|
+
});
|
|
799
|
+
lead = await extractDetail(page, collect, card.name);
|
|
800
|
+
break;
|
|
801
|
+
} catch {
|
|
802
|
+
if (attempt < retries && !signal?.aborted) {
|
|
803
|
+
onProgress?.(` retrying ${i + 1}/${cards.length} (attempt ${attempt + 2})\u2026`);
|
|
804
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
if (!lead) {
|
|
631
809
|
if (card.name && (collect.has("name") || collect.has("mapsUrl"))) {
|
|
632
|
-
|
|
633
|
-
if (collect.has("name"))
|
|
634
|
-
if (collect.has("mapsUrl"))
|
|
635
|
-
|
|
810
|
+
lead = {};
|
|
811
|
+
if (collect.has("name")) lead.name = card.name;
|
|
812
|
+
if (collect.has("mapsUrl")) lead.mapsUrl = card.href;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
if (lead) {
|
|
816
|
+
leads.push(lead);
|
|
817
|
+
try {
|
|
818
|
+
onLead?.(lead);
|
|
819
|
+
} catch {
|
|
636
820
|
}
|
|
637
821
|
}
|
|
638
|
-
if (delayMs) await page.waitForTimeout(
|
|
822
|
+
if (delayMs) await page.waitForTimeout(pauseMs());
|
|
639
823
|
}
|
|
640
824
|
}
|
|
641
825
|
onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
|
|
@@ -668,6 +852,14 @@ async function scrapeLeads(opts) {
|
|
|
668
852
|
signal
|
|
669
853
|
);
|
|
670
854
|
}
|
|
855
|
+
if (wantVerify && !signal?.aborted) {
|
|
856
|
+
onProgress?.("verifying email domains\u2026");
|
|
857
|
+
const vOpts = {
|
|
858
|
+
concurrency: Math.max(1, Math.trunc(opts.concurrency ?? 5))
|
|
859
|
+
};
|
|
860
|
+
if (onProgress) vOpts.onProgress = (m) => onProgress(m);
|
|
861
|
+
await verifyEmails(leads, vOpts);
|
|
862
|
+
}
|
|
671
863
|
if (opts.country && fields.has("phone")) {
|
|
672
864
|
for (const lead of leads) {
|
|
673
865
|
if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
|
|
@@ -675,6 +867,7 @@ async function scrapeLeads(opts) {
|
|
|
675
867
|
}
|
|
676
868
|
if (opts.filters) leads = filterLeads(leads, opts.filters);
|
|
677
869
|
if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
|
|
870
|
+
if (wantVerify && !fields.has("emailStatus")) for (const l of leads) delete l.emailStatus;
|
|
678
871
|
onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
|
|
679
872
|
return leads;
|
|
680
873
|
} finally {
|
|
@@ -682,6 +875,11 @@ async function scrapeLeads(opts) {
|
|
|
682
875
|
});
|
|
683
876
|
}
|
|
684
877
|
}
|
|
878
|
+
async function searchLeadsDetailed(opts) {
|
|
879
|
+
const started = Date.now();
|
|
880
|
+
const leads = await scrapeLeads(opts);
|
|
881
|
+
return { leads, stats: computeStats(leads), elapsedMs: Date.now() - started };
|
|
882
|
+
}
|
|
685
883
|
|
|
686
884
|
// src/batch.ts
|
|
687
885
|
async function searchLeadsBatch(queries, opts = {}) {
|
|
@@ -746,69 +944,6 @@ async function searchLeadsMulti(opts) {
|
|
|
746
944
|
return searchLeadsBatch(queries, opts);
|
|
747
945
|
}
|
|
748
946
|
|
|
749
|
-
// src/export.ts
|
|
750
|
-
import { stringify as csvStringify } from "@lacspace/csv";
|
|
751
|
-
import { jsonToXlsx } from "@lacspace/xlsx";
|
|
752
|
-
var HEADERS = {
|
|
753
|
-
name: "Name",
|
|
754
|
-
category: "Category",
|
|
755
|
-
rating: "Rating",
|
|
756
|
-
reviews: "Reviews",
|
|
757
|
-
priceLevel: "Price",
|
|
758
|
-
address: "Address",
|
|
759
|
-
phone: "Phone",
|
|
760
|
-
website: "Website",
|
|
761
|
-
email: "Email",
|
|
762
|
-
facebook: "Facebook",
|
|
763
|
-
instagram: "Instagram",
|
|
764
|
-
whatsapp: "WhatsApp",
|
|
765
|
-
linkedin: "LinkedIn",
|
|
766
|
-
twitter: "Twitter/X",
|
|
767
|
-
youtube: "YouTube",
|
|
768
|
-
tiktok: "TikTok",
|
|
769
|
-
telegram: "Telegram",
|
|
770
|
-
plusCode: "Plus Code",
|
|
771
|
-
latitude: "Latitude",
|
|
772
|
-
longitude: "Longitude",
|
|
773
|
-
hours: "Hours",
|
|
774
|
-
mapsUrl: "Maps URL"
|
|
775
|
-
};
|
|
776
|
-
function toRows(leads, fields = ALL_FIELDS) {
|
|
777
|
-
return leads.map((lead) => {
|
|
778
|
-
const row = {};
|
|
779
|
-
for (const f of fields) {
|
|
780
|
-
const v = lead[f];
|
|
781
|
-
row[HEADERS[f]] = v === void 0 || v === null ? "" : v;
|
|
782
|
-
}
|
|
783
|
-
return row;
|
|
784
|
-
});
|
|
785
|
-
}
|
|
786
|
-
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
787
|
-
if (format === "json" || format === "ndjson") {
|
|
788
|
-
const picked = leads.map((lead) => {
|
|
789
|
-
const o = {};
|
|
790
|
-
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
791
|
-
return o;
|
|
792
|
-
});
|
|
793
|
-
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
794
|
-
return { data, binary: false };
|
|
795
|
-
}
|
|
796
|
-
const rows = toRows(leads, fields);
|
|
797
|
-
if (format === "csv") {
|
|
798
|
-
const csvRows = rows;
|
|
799
|
-
return { data: csvStringify(csvRows, { escapeFormulas: true }), binary: false };
|
|
800
|
-
}
|
|
801
|
-
const columns = fields.map((f) => {
|
|
802
|
-
const header = HEADERS[f];
|
|
803
|
-
const widest = rows.reduce((m, r) => Math.max(m, String(r[header] ?? "").length), header.length);
|
|
804
|
-
return { header, width: Math.min(60, Math.max(8, widest + 2)) };
|
|
805
|
-
});
|
|
806
|
-
return {
|
|
807
|
-
data: jsonToXlsx(rows, { sheetName: opts.sheetName ?? "Leads", columns }),
|
|
808
|
-
binary: true
|
|
809
|
-
};
|
|
810
|
-
}
|
|
811
|
-
|
|
812
947
|
// src/convert.ts
|
|
813
948
|
import { readFile, writeFile } from "fs/promises";
|
|
814
949
|
import { extname, resolve } from "path";
|
|
@@ -893,10 +1028,13 @@ export {
|
|
|
893
1028
|
cleanWebsite,
|
|
894
1029
|
columnsOf,
|
|
895
1030
|
composeQuery,
|
|
1031
|
+
computeStats,
|
|
896
1032
|
convertFile,
|
|
897
1033
|
dedupeLeads,
|
|
898
1034
|
defaultFilename,
|
|
899
1035
|
detectFormat,
|
|
1036
|
+
emailDomain,
|
|
1037
|
+
emailFormatValid,
|
|
900
1038
|
enrichContacts,
|
|
901
1039
|
expandQueries,
|
|
902
1040
|
extractEmails,
|
|
@@ -910,12 +1048,16 @@ export {
|
|
|
910
1048
|
parseReviewCount,
|
|
911
1049
|
readRows,
|
|
912
1050
|
resolvePreset,
|
|
1051
|
+
rowsToLeads,
|
|
913
1052
|
scrapeLeads,
|
|
914
1053
|
scrapeLeads as searchLeads,
|
|
915
1054
|
searchLeadsBatch,
|
|
1055
|
+
searchLeadsDetailed,
|
|
916
1056
|
searchLeadsMulti,
|
|
917
1057
|
serialize,
|
|
918
1058
|
serializeRows,
|
|
919
1059
|
sortLeads,
|
|
920
|
-
toRows
|
|
1060
|
+
toRows,
|
|
1061
|
+
verifyEmail,
|
|
1062
|
+
verifyEmails
|
|
921
1063
|
};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lacspace-leads",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Free, open-source local-business lead finder — name a city, area and business type, and it drives a real browser over Google Maps to collect names, phones, websites, ratings, addresses, emails and social links, then exports to JSON, NDJSON, CSV or Excel. Sweep multiple areas at once, normalise phones to E.164, and enrich in parallel. No API keys.",
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"description": "Free, open-source local-business lead finder — name a city, area and business type, and it drives a real browser over Google Maps to collect names, phones, websites, ratings, addresses, emails and social links, then exports to JSON, NDJSON, CSV or Excel. Sweep multiple areas at once, verify emails by MX, normalise phones to E.164, accumulate into a master file, and enrich in parallel through an optional proxy. No API keys.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"lacspace-leads": "dist/cli.js"
|
|
@@ -36,6 +36,8 @@
|
|
|
36
36
|
"prospecting",
|
|
37
37
|
"b2b",
|
|
38
38
|
"email-finder",
|
|
39
|
+
"email-verification",
|
|
40
|
+
"mx-lookup",
|
|
39
41
|
"csv",
|
|
40
42
|
"excel",
|
|
41
43
|
"ndjson",
|