lacspace-leads 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -14
- package/dist/cli.js +426 -49
- package/dist/lib.cjs +355 -26
- package/dist/lib.d.cts +140 -6
- package/dist/lib.d.ts +140 -6
- package/dist/lib.js +344 -26
- package/package.json +5 -2
package/dist/cli.js
CHANGED
|
@@ -23,13 +23,42 @@ var ALL_FIELDS = [
|
|
|
23
23
|
"facebook",
|
|
24
24
|
"instagram",
|
|
25
25
|
"whatsapp",
|
|
26
|
+
"linkedin",
|
|
27
|
+
"twitter",
|
|
28
|
+
"youtube",
|
|
29
|
+
"tiktok",
|
|
30
|
+
"telegram",
|
|
26
31
|
"plusCode",
|
|
27
32
|
"latitude",
|
|
28
33
|
"longitude",
|
|
29
34
|
"hours",
|
|
30
35
|
"mapsUrl"
|
|
31
36
|
];
|
|
32
|
-
var ENRICHED_FIELDS = [
|
|
37
|
+
var ENRICHED_FIELDS = [
|
|
38
|
+
"email",
|
|
39
|
+
"facebook",
|
|
40
|
+
"instagram",
|
|
41
|
+
"whatsapp",
|
|
42
|
+
"linkedin",
|
|
43
|
+
"twitter",
|
|
44
|
+
"youtube",
|
|
45
|
+
"tiktok",
|
|
46
|
+
"telegram"
|
|
47
|
+
];
|
|
48
|
+
var FIELD_PRESETS = {
|
|
49
|
+
/** Just enough to place a business: name, phone, website. */
|
|
50
|
+
minimal: ["name", "phone", "website"],
|
|
51
|
+
/** Cold-outreach columns: who, how to reach them, where. */
|
|
52
|
+
outreach: ["name", "phone", "email", "website", "address"],
|
|
53
|
+
/** Contact-book columns incl. socials (implies enrichment). */
|
|
54
|
+
contact: ["name", "phone", "email", "website", "facebook", "instagram", "whatsapp"],
|
|
55
|
+
/** Map/geo columns for plotting. */
|
|
56
|
+
geo: ["name", "address", "latitude", "longitude", "plusCode", "mapsUrl"],
|
|
57
|
+
/** Everything Maps shows, no website enrichment. */
|
|
58
|
+
full: ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f)),
|
|
59
|
+
/** Every field, including enriched ones. */
|
|
60
|
+
everything: [...ALL_FIELDS]
|
|
61
|
+
};
|
|
33
62
|
var DEFAULT_FIELDS = ALL_FIELDS.filter((f) => !ENRICHED_FIELDS.includes(f));
|
|
34
63
|
|
|
35
64
|
// src/query.ts
|
|
@@ -40,8 +69,38 @@ function composeQuery(opts) {
|
|
|
40
69
|
if (!type) throw new Error("A business `type` (or an explicit `query`) is required.");
|
|
41
70
|
return where ? `${type} in ${where}` : type;
|
|
42
71
|
}
|
|
43
|
-
function mapsSearchUrl(query) {
|
|
44
|
-
|
|
72
|
+
function mapsSearchUrl(query, opts = {}) {
|
|
73
|
+
const hl = (opts.hl ?? "en").split("-")[0] || "en";
|
|
74
|
+
const params = new URLSearchParams({ hl });
|
|
75
|
+
if (opts.gl) params.set("gl", opts.gl.toLowerCase());
|
|
76
|
+
return `https://www.google.com/maps/search/${encodeURIComponent(query)}?${params.toString()}`;
|
|
77
|
+
}
|
|
78
|
+
function expandQueries(opts) {
|
|
79
|
+
if (opts.query && opts.query.trim()) return [{ query: opts.query.trim() }];
|
|
80
|
+
const split = (s) => {
|
|
81
|
+
const parts = (s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
|
|
82
|
+
return parts.length ? parts : [void 0];
|
|
83
|
+
};
|
|
84
|
+
const types = split(opts.type);
|
|
85
|
+
const cities = split(opts.city);
|
|
86
|
+
const areas = split(opts.area);
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const type of types) {
|
|
89
|
+
for (const city of cities) {
|
|
90
|
+
for (const area of areas) {
|
|
91
|
+
const entry = {};
|
|
92
|
+
if (type) entry.type = type;
|
|
93
|
+
if (city) entry.city = city;
|
|
94
|
+
if (area) entry.area = area;
|
|
95
|
+
out.push(entry);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
function resolvePreset(name) {
|
|
102
|
+
if (!name) return void 0;
|
|
103
|
+
return FIELD_PRESETS[name.trim().toLowerCase()];
|
|
45
104
|
}
|
|
46
105
|
function normalizeFields(input) {
|
|
47
106
|
if (!input) return [...ALL_FIELDS];
|
|
@@ -82,12 +141,23 @@ function extractSocials(html) {
|
|
|
82
141
|
const hrefs = [...html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1] ?? "");
|
|
83
142
|
for (const href of hrefs) {
|
|
84
143
|
const u = href.trim();
|
|
85
|
-
if (!
|
|
144
|
+
if (!u) continue;
|
|
145
|
+
if (!out.facebook && /(^|\/\/|\.)facebook\.com\//i.test(u) && !/sharer|plugins|\/tr\?|dialog\//i.test(u)) {
|
|
86
146
|
out.facebook = absolutize(u);
|
|
87
|
-
} else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u)) {
|
|
147
|
+
} else if (!out.instagram && /(^|\/\/|\.)instagram\.com\//i.test(u) && !/\/(share|p)\//i.test(u)) {
|
|
88
148
|
out.instagram = absolutize(u);
|
|
89
149
|
} else if (!out.whatsapp && /(wa\.me\/|api\.whatsapp\.com\/|whatsapp:\/\/)/i.test(u)) {
|
|
90
150
|
out.whatsapp = u.startsWith("http") || u.startsWith("whatsapp") ? u : `https:${u}`;
|
|
151
|
+
} else if (!out.linkedin && /(^|\/\/|\.)linkedin\.com\/(company|in|school)\//i.test(u) && !/sharing|shareArticle/i.test(u)) {
|
|
152
|
+
out.linkedin = absolutize(u);
|
|
153
|
+
} else if (!out.twitter && /(^|\/\/|\.)(twitter|x)\.com\//i.test(u) && !/(intent|share|widgets|hashtag)\b/i.test(u)) {
|
|
154
|
+
out.twitter = absolutize(u);
|
|
155
|
+
} else if (!out.youtube && /(youtube\.com\/(channel|c|user|@)|youtu\.be\/)/i.test(u)) {
|
|
156
|
+
out.youtube = absolutize(u);
|
|
157
|
+
} else if (!out.tiktok && /(^|\/\/|\.)tiktok\.com\/@/i.test(u)) {
|
|
158
|
+
out.tiktok = absolutize(u);
|
|
159
|
+
} else if (!out.telegram && /(t\.me\/|telegram\.me\/)/i.test(u) && !/\/share\b/i.test(u)) {
|
|
160
|
+
out.telegram = absolutize(u);
|
|
91
161
|
}
|
|
92
162
|
}
|
|
93
163
|
return out;
|
|
@@ -138,11 +208,9 @@ async function enrichContacts(website, opts = {}) {
|
|
|
138
208
|
const email = extractEmails(page);
|
|
139
209
|
if (email) {
|
|
140
210
|
contacts.email = email;
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if (!contacts
|
|
144
|
-
if (!contacts.instagram && s.instagram) contacts.instagram = s.instagram;
|
|
145
|
-
if (!contacts.whatsapp && s.whatsapp) contacts.whatsapp = s.whatsapp;
|
|
211
|
+
const s = extractSocials(page);
|
|
212
|
+
for (const k of Object.keys(s)) {
|
|
213
|
+
if (!contacts[k] && s[k]) contacts[k] = s[k];
|
|
146
214
|
}
|
|
147
215
|
break;
|
|
148
216
|
}
|
|
@@ -197,7 +265,148 @@ function dedupeLeads(leads, by = "website") {
|
|
|
197
265
|
return out;
|
|
198
266
|
}
|
|
199
267
|
|
|
268
|
+
// src/normalize.ts
|
|
269
|
+
var TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|gbraid$|wbraid$|mc_|_hs|ref$|ref_src$|source$|igshid$|yclid$)/i;
|
|
270
|
+
function cleanWebsite(url) {
|
|
271
|
+
if (!url) return void 0;
|
|
272
|
+
let raw = url.trim();
|
|
273
|
+
if (!raw) return void 0;
|
|
274
|
+
const g = raw.match(/[?&](?:q|url)=([^&]+)/i);
|
|
275
|
+
if (g && /google\.[a-z.]+\/url/i.test(raw)) {
|
|
276
|
+
try {
|
|
277
|
+
raw = decodeURIComponent(g[1]);
|
|
278
|
+
} catch {
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
let u;
|
|
282
|
+
try {
|
|
283
|
+
u = new URL(raw.startsWith("http") ? raw : `https://${raw}`);
|
|
284
|
+
} catch {
|
|
285
|
+
return raw;
|
|
286
|
+
}
|
|
287
|
+
u.hash = "";
|
|
288
|
+
u.host = u.host.toLowerCase();
|
|
289
|
+
for (const key of [...u.searchParams.keys()]) {
|
|
290
|
+
if (TRACKING_PARAMS.test(key)) u.searchParams.delete(key);
|
|
291
|
+
}
|
|
292
|
+
let href = u.toString();
|
|
293
|
+
if (u.pathname === "/" && !u.search) href = href.replace(/\/$/, "");
|
|
294
|
+
return href;
|
|
295
|
+
}
|
|
296
|
+
var CALLING_CODES = {
|
|
297
|
+
NP: "977",
|
|
298
|
+
IN: "91",
|
|
299
|
+
US: "1",
|
|
300
|
+
CA: "1",
|
|
301
|
+
GB: "44",
|
|
302
|
+
AU: "61",
|
|
303
|
+
NZ: "64",
|
|
304
|
+
SG: "65",
|
|
305
|
+
MY: "60",
|
|
306
|
+
ID: "62",
|
|
307
|
+
TH: "66",
|
|
308
|
+
PH: "63",
|
|
309
|
+
VN: "84",
|
|
310
|
+
BD: "880",
|
|
311
|
+
PK: "92",
|
|
312
|
+
LK: "94",
|
|
313
|
+
AE: "971",
|
|
314
|
+
SA: "966",
|
|
315
|
+
QA: "974",
|
|
316
|
+
KW: "965",
|
|
317
|
+
BH: "973",
|
|
318
|
+
OM: "968",
|
|
319
|
+
CN: "86",
|
|
320
|
+
HK: "852",
|
|
321
|
+
JP: "81",
|
|
322
|
+
KR: "82",
|
|
323
|
+
DE: "49",
|
|
324
|
+
FR: "33",
|
|
325
|
+
ES: "34",
|
|
326
|
+
IT: "39",
|
|
327
|
+
NL: "31",
|
|
328
|
+
BE: "32",
|
|
329
|
+
CH: "41",
|
|
330
|
+
AT: "43",
|
|
331
|
+
SE: "46",
|
|
332
|
+
NO: "47",
|
|
333
|
+
DK: "45",
|
|
334
|
+
FI: "358",
|
|
335
|
+
IE: "353",
|
|
336
|
+
PT: "351",
|
|
337
|
+
PL: "48",
|
|
338
|
+
ZA: "27",
|
|
339
|
+
NG: "234",
|
|
340
|
+
KE: "254",
|
|
341
|
+
EG: "20",
|
|
342
|
+
BR: "55",
|
|
343
|
+
MX: "52",
|
|
344
|
+
AR: "54",
|
|
345
|
+
TR: "90"
|
|
346
|
+
};
|
|
347
|
+
function callingCode(country) {
|
|
348
|
+
if (!country) return void 0;
|
|
349
|
+
const c2 = country.trim().replace(/^\+/, "");
|
|
350
|
+
if (/^\d{1,4}$/.test(c2)) return c2;
|
|
351
|
+
return CALLING_CODES[c2.toUpperCase()];
|
|
352
|
+
}
|
|
353
|
+
function normalizePhone(raw, country) {
|
|
354
|
+
if (!raw) return void 0;
|
|
355
|
+
const trimmed = raw.trim();
|
|
356
|
+
if (!trimmed) return void 0;
|
|
357
|
+
const isIntl = /^\+/.test(trimmed) || /^00\d/.test(trimmed);
|
|
358
|
+
let digits = trimmed.replace(/[^0-9]/g, "");
|
|
359
|
+
if (/^00\d/.test(trimmed)) digits = digits.replace(/^00/, "");
|
|
360
|
+
const cc = callingCode(country);
|
|
361
|
+
if (isIntl) {
|
|
362
|
+
return plausible(digits) ? `+${digits}` : trimmed;
|
|
363
|
+
}
|
|
364
|
+
if (!cc) return trimmed;
|
|
365
|
+
if (digits.startsWith(cc)) return plausible(digits) ? `+${digits}` : trimmed;
|
|
366
|
+
const local = digits.replace(/^0+/, "");
|
|
367
|
+
const e164 = `${cc}${local}`;
|
|
368
|
+
return plausible(e164) ? `+${e164}` : trimmed;
|
|
369
|
+
}
|
|
370
|
+
function plausible(digits) {
|
|
371
|
+
return digits.length >= 8 && digits.length <= 15;
|
|
372
|
+
}
|
|
373
|
+
var priceRank = (p) => p ? p.replace(/[^$€£₹¥₩]/g, "").length : 0;
|
|
374
|
+
function sortLeads(leads, by, dir) {
|
|
375
|
+
if (!by) return [...leads];
|
|
376
|
+
const desc = dir ? dir === "desc" : by !== "name";
|
|
377
|
+
const val = (l) => {
|
|
378
|
+
if (by === "name") return l.name?.toLowerCase();
|
|
379
|
+
if (by === "priceLevel") return l.priceLevel ? priceRank(l.priceLevel) : void 0;
|
|
380
|
+
return l[by];
|
|
381
|
+
};
|
|
382
|
+
return [...leads].map((lead, i) => ({ lead, i })).sort((a, b) => {
|
|
383
|
+
const av = val(a.lead);
|
|
384
|
+
const bv = val(b.lead);
|
|
385
|
+
const aMissing = av === void 0 || av === "";
|
|
386
|
+
const bMissing = bv === void 0 || bv === "";
|
|
387
|
+
if (aMissing && bMissing) return a.i - b.i;
|
|
388
|
+
if (aMissing) return 1;
|
|
389
|
+
if (bMissing) return -1;
|
|
390
|
+
let cmp = 0;
|
|
391
|
+
if (typeof av === "number" && typeof bv === "number") cmp = av - bv;
|
|
392
|
+
else cmp = String(av).localeCompare(String(bv));
|
|
393
|
+
if (desc) cmp = -cmp;
|
|
394
|
+
return cmp !== 0 ? cmp : a.i - b.i;
|
|
395
|
+
}).map((x) => x.lead);
|
|
396
|
+
}
|
|
397
|
+
|
|
200
398
|
// src/scrape.ts
|
|
399
|
+
async function pool(items, n, fn, signal) {
|
|
400
|
+
let next = 0;
|
|
401
|
+
const workers = Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => {
|
|
402
|
+
for (; ; ) {
|
|
403
|
+
const i = next++;
|
|
404
|
+
if (i >= items.length || signal?.aborted) return;
|
|
405
|
+
await fn(items[i], i);
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
await Promise.all(workers);
|
|
409
|
+
}
|
|
201
410
|
var LeadsError = class extends Error {
|
|
202
411
|
code;
|
|
203
412
|
cause;
|
|
@@ -322,7 +531,13 @@ async function extractDetail(page, fields, fallbackName) {
|
|
|
322
531
|
}
|
|
323
532
|
if (fields.has("reviews")) {
|
|
324
533
|
const rv = await box.locator('span[aria-label*="review"]').first().getAttribute("aria-label").catch(() => null);
|
|
325
|
-
|
|
534
|
+
let count = parseReviewCount(rv);
|
|
535
|
+
if (count === void 0) {
|
|
536
|
+
const boxText = await box.innerText().catch(() => "");
|
|
537
|
+
const m = boxText.match(/\(([\d.,\s]+)\)/);
|
|
538
|
+
if (m) count = parseReviewCount(m[1]);
|
|
539
|
+
}
|
|
540
|
+
lead.reviews = count;
|
|
326
541
|
}
|
|
327
542
|
}
|
|
328
543
|
}
|
|
@@ -374,12 +589,15 @@ async function scrapeLeads(opts) {
|
|
|
374
589
|
onProgress?.(`searching Google Maps for "${query}"\u2026`);
|
|
375
590
|
const browser = await launchBrowser(headless);
|
|
376
591
|
try {
|
|
592
|
+
const locale = opts.locale ?? "en-US";
|
|
377
593
|
const context = await browser.newContext({
|
|
378
594
|
viewport: { width: 1280, height: 900 },
|
|
379
|
-
locale
|
|
595
|
+
locale
|
|
380
596
|
});
|
|
381
597
|
const page = await context.newPage();
|
|
382
|
-
|
|
598
|
+
const urlOpts = { hl: locale };
|
|
599
|
+
if (opts.region) urlOpts.gl = opts.region;
|
|
600
|
+
await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
383
601
|
await dismissConsent(page);
|
|
384
602
|
await loadResults(page, limit, delayMs, onProgress, signal);
|
|
385
603
|
if (signal?.aborted) throw new LeadsError("Search aborted.", "ABORTED");
|
|
@@ -429,22 +647,42 @@ async function scrapeLeads(opts) {
|
|
|
429
647
|
}
|
|
430
648
|
}
|
|
431
649
|
onProgress?.(`collected ${leads.length} listing${leads.length === 1 ? "" : "s"}.`);
|
|
650
|
+
if (opts.cleanUrls ?? true) {
|
|
651
|
+
for (const lead of leads) {
|
|
652
|
+
if (lead.website) lead.website = cleanWebsite(lead.website);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
432
655
|
leads = dedupeLeads(leads, opts.dedupe ?? "website");
|
|
433
656
|
if (wantEnrich) {
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
657
|
+
const socialFields = ENRICHED_FIELDS.filter((f) => f !== "email");
|
|
658
|
+
let done = 0;
|
|
659
|
+
await pool(
|
|
660
|
+
leads,
|
|
661
|
+
Math.max(1, Math.trunc(opts.concurrency ?? 3)),
|
|
662
|
+
async (lead) => {
|
|
663
|
+
if (overBudget()) return;
|
|
664
|
+
const site = lead.website;
|
|
665
|
+
if (site) {
|
|
666
|
+
onProgress?.(`enriching ${++done}/${leads.length}: ${lead.name ?? site}\u2026`);
|
|
667
|
+
const c2 = await enrichContacts(site);
|
|
668
|
+
if (fields.has("email") && c2.email) lead.email = c2.email;
|
|
669
|
+
for (const f of socialFields) {
|
|
670
|
+
const v = c2[f];
|
|
671
|
+
if (fields.has(f) && v) lead[f] = v;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
if (!fields.has("website")) delete lead.website;
|
|
675
|
+
},
|
|
676
|
+
signal
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
if (opts.country && fields.has("phone")) {
|
|
680
|
+
for (const lead of leads) {
|
|
681
|
+
if (lead.phone) lead.phone = normalizePhone(lead.phone, opts.country);
|
|
445
682
|
}
|
|
446
683
|
}
|
|
447
684
|
if (opts.filters) leads = filterLeads(leads, opts.filters);
|
|
685
|
+
if (opts.sort) leads = sortLeads(leads, opts.sort, opts.sortDir);
|
|
448
686
|
onProgress?.(`collected ${leads.length} lead${leads.length === 1 ? "" : "s"}.`);
|
|
449
687
|
return leads;
|
|
450
688
|
} finally {
|
|
@@ -453,6 +691,60 @@ async function scrapeLeads(opts) {
|
|
|
453
691
|
}
|
|
454
692
|
}
|
|
455
693
|
|
|
694
|
+
// src/batch.ts
|
|
695
|
+
async function searchLeadsBatch(queries, opts = {}) {
|
|
696
|
+
const {
|
|
697
|
+
total,
|
|
698
|
+
onProgress,
|
|
699
|
+
signal,
|
|
700
|
+
type: _t,
|
|
701
|
+
city: _c,
|
|
702
|
+
area: _a,
|
|
703
|
+
query: _q,
|
|
704
|
+
...shared
|
|
705
|
+
} = opts;
|
|
706
|
+
void _t;
|
|
707
|
+
void _c;
|
|
708
|
+
void _a;
|
|
709
|
+
void _q;
|
|
710
|
+
let merged = [];
|
|
711
|
+
const seenDedupe = shared.dedupe ?? "website";
|
|
712
|
+
for (let i = 0; i < queries.length; i++) {
|
|
713
|
+
if (signal?.aborted) break;
|
|
714
|
+
const q = queries[i];
|
|
715
|
+
const label = (() => {
|
|
716
|
+
try {
|
|
717
|
+
return composeQuery(q);
|
|
718
|
+
} catch {
|
|
719
|
+
return q.query ?? q.type ?? "search";
|
|
720
|
+
}
|
|
721
|
+
})();
|
|
722
|
+
onProgress?.(`[${i + 1}/${queries.length}] ${label}`);
|
|
723
|
+
const subOpts = {
|
|
724
|
+
...shared,
|
|
725
|
+
type: q.type ?? "",
|
|
726
|
+
// Defer sorting to the merged pass; keep per-search dedupe/filter local.
|
|
727
|
+
...q.query ? { query: q.query } : {},
|
|
728
|
+
...q.city ? { city: q.city } : {},
|
|
729
|
+
...q.area ? { area: q.area } : {}
|
|
730
|
+
};
|
|
731
|
+
delete subOpts.sort;
|
|
732
|
+
if (signal) subOpts.signal = signal;
|
|
733
|
+
if (onProgress) subOpts.onProgress = (m) => onProgress(` ${m}`);
|
|
734
|
+
try {
|
|
735
|
+
const found = await scrapeLeads(subOpts);
|
|
736
|
+
merged = merged.concat(found);
|
|
737
|
+
} catch (err) {
|
|
738
|
+
onProgress?.(` ! skipped "${label}": ${err.message}`);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
merged = dedupeLeads(merged, seenDedupe);
|
|
742
|
+
if (opts.sort) merged = sortLeads(merged, opts.sort, opts.sortDir);
|
|
743
|
+
if (total !== void 0 && total > 0) merged = merged.slice(0, total);
|
|
744
|
+
onProgress?.(`merged ${merged.length} unique lead${merged.length === 1 ? "" : "s"}.`);
|
|
745
|
+
return merged;
|
|
746
|
+
}
|
|
747
|
+
|
|
456
748
|
// src/export.ts
|
|
457
749
|
import { stringify as csvStringify } from "@lacspace/csv";
|
|
458
750
|
import { jsonToXlsx } from "@lacspace/xlsx";
|
|
@@ -469,6 +761,11 @@ var HEADERS = {
|
|
|
469
761
|
facebook: "Facebook",
|
|
470
762
|
instagram: "Instagram",
|
|
471
763
|
whatsapp: "WhatsApp",
|
|
764
|
+
linkedin: "LinkedIn",
|
|
765
|
+
twitter: "Twitter/X",
|
|
766
|
+
youtube: "YouTube",
|
|
767
|
+
tiktok: "TikTok",
|
|
768
|
+
telegram: "Telegram",
|
|
472
769
|
plusCode: "Plus Code",
|
|
473
770
|
latitude: "Latitude",
|
|
474
771
|
longitude: "Longitude",
|
|
@@ -486,13 +783,14 @@ function toRows(leads, fields = ALL_FIELDS) {
|
|
|
486
783
|
});
|
|
487
784
|
}
|
|
488
785
|
function serialize(leads, format, fields = ALL_FIELDS, opts = {}) {
|
|
489
|
-
if (format === "json") {
|
|
786
|
+
if (format === "json" || format === "ndjson") {
|
|
490
787
|
const picked = leads.map((lead) => {
|
|
491
788
|
const o = {};
|
|
492
789
|
for (const f of fields) if (lead[f] !== void 0) o[f] = lead[f];
|
|
493
790
|
return o;
|
|
494
791
|
});
|
|
495
|
-
|
|
792
|
+
const data = format === "ndjson" ? picked.map((o) => JSON.stringify(o)).join("\n") + (picked.length ? "\n" : "") : JSON.stringify(picked, null, 2);
|
|
793
|
+
return { data, binary: false };
|
|
496
794
|
}
|
|
497
795
|
const rows = toRows(leads, fields);
|
|
498
796
|
if (format === "csv") {
|
|
@@ -585,6 +883,8 @@ async function convertFile(input, opts = {}) {
|
|
|
585
883
|
}
|
|
586
884
|
|
|
587
885
|
// src/cli.ts
|
|
886
|
+
var SOCIAL_FIELDS = ENRICHED_FIELDS.filter((f) => f !== "email");
|
|
887
|
+
var OUTPUT_FORMATS = ["json", "ndjson", "csv", "xlsx"];
|
|
588
888
|
var C = {
|
|
589
889
|
reset: "\x1B[0m",
|
|
590
890
|
bold: "\x1B[1m",
|
|
@@ -609,20 +909,23 @@ function parseArgs(list) {
|
|
|
609
909
|
hasPhone: false,
|
|
610
910
|
hasWebsite: false,
|
|
611
911
|
hasEmail: false,
|
|
912
|
+
cleanUrls: true,
|
|
612
913
|
yes: false,
|
|
613
914
|
help: false
|
|
614
915
|
};
|
|
615
916
|
for (let i = 0; i < list.length; i++) {
|
|
616
917
|
const arg = list[i];
|
|
617
918
|
const next = () => list[++i] ?? "";
|
|
618
|
-
if (arg === "--city") a.city = next();
|
|
619
|
-
else if (arg === "--area") a.area = next();
|
|
620
|
-
else if (arg === "-t" || arg === "--type") a.type = next();
|
|
919
|
+
if (arg === "--city" || arg === "--cities") a.city = next();
|
|
920
|
+
else if (arg === "--area" || arg === "--areas") a.area = next();
|
|
921
|
+
else if (arg === "-t" || arg === "--type" || arg === "--types") a.type = next();
|
|
621
922
|
else if (arg === "-q" || arg === "--query") a.query = next();
|
|
622
923
|
else if (arg === "--fields") a.fields = next();
|
|
924
|
+
else if (arg === "--preset") a.preset = next();
|
|
623
925
|
else if (arg === "-f" || arg === "--format") a.format = next();
|
|
624
926
|
else if (arg === "-o" || arg === "--out") a.out = next();
|
|
625
927
|
else if (arg === "-n" || arg === "--limit") a.limit = parseInt(next(), 10) || a.limit;
|
|
928
|
+
else if (arg === "--total") a.total = parseInt(next(), 10) || a.total;
|
|
626
929
|
else if (arg === "--headless") a.headless = true;
|
|
627
930
|
else if (arg === "--no-details") a.details = false;
|
|
628
931
|
else if (arg === "--delay") a.delay = parseInt(next(), 10) || a.delay;
|
|
@@ -639,6 +942,14 @@ function parseArgs(list) {
|
|
|
639
942
|
a.hasEmail = true;
|
|
640
943
|
a.emails = true;
|
|
641
944
|
} else if (arg === "--dedupe") a.dedupe = next();
|
|
945
|
+
else if (arg === "--sort") a.sort = next();
|
|
946
|
+
else if (arg === "--desc") a.desc = true;
|
|
947
|
+
else if (arg === "--asc") a.desc = false;
|
|
948
|
+
else if (arg === "--country") a.country = next();
|
|
949
|
+
else if (arg === "--lang" || arg === "--locale") a.locale = next();
|
|
950
|
+
else if (arg === "--region" || arg === "--gl") a.region = next();
|
|
951
|
+
else if (arg === "--concurrency") a.concurrency = parseInt(next(), 10) || a.concurrency;
|
|
952
|
+
else if (arg === "--no-clean-urls") a.cleanUrls = false;
|
|
642
953
|
else if (arg === "--sheet") a.sheet = next();
|
|
643
954
|
else if (arg === "--max-time") a.maxTime = parseInt(next(), 10);
|
|
644
955
|
else if (arg === "-y" || arg === "--yes") a.yes = true;
|
|
@@ -656,35 +967,50 @@ ${c("bold", "Usage")}
|
|
|
656
967
|
npx lacspace-leads convert <file> [-f json|csv|xlsx] [-o out]
|
|
657
968
|
|
|
658
969
|
${c("bold", "Search options")}
|
|
659
|
-
-t, --type <text> Business type
|
|
660
|
-
|
|
661
|
-
--
|
|
970
|
+
-t, --type <text> Business type/keyword. Comma-separate for several,
|
|
971
|
+
e.g. "restaurants,cafes"
|
|
972
|
+
--city <text> City. Comma-separate for several.
|
|
973
|
+
--area <text> Area/neighbourhood. Comma-separate to sweep a whole
|
|
974
|
+
city, e.g. --area "Baneshwor,Thamel,Patan"
|
|
662
975
|
-q, --query <text> Raw query verbatim (overrides city/area/type)
|
|
663
976
|
--fields <list> Columns: ${ALL_FIELDS.join(",")}
|
|
664
|
-
|
|
977
|
+
--preset <name> Field bundle: ${Object.keys(FIELD_PRESETS).join(" | ")}
|
|
978
|
+
-n, --limit <n> Max listings per search (default 60)
|
|
979
|
+
--total <n> Cap the merged result (batch searches)
|
|
665
980
|
--no-details Names + Maps URLs only (fast, no per-listing open)
|
|
666
981
|
|
|
667
982
|
${c("bold", "Enrichment (visits each website)")}
|
|
668
983
|
--emails Also find an email from each website
|
|
669
|
-
--socials Also find Facebook
|
|
984
|
+
--socials Also find Facebook/Instagram/WhatsApp/LinkedIn/X/
|
|
985
|
+
YouTube/TikTok/Telegram
|
|
670
986
|
--enrich Both of the above
|
|
987
|
+
--concurrency <n> Websites to enrich in parallel (default 3)
|
|
988
|
+
|
|
989
|
+
${c("bold", "Clean-up")}
|
|
990
|
+
--country <c> Normalise phones to E.164 for this country
|
|
991
|
+
(ISO-2 like NP/US, or a calling code like 977)
|
|
992
|
+
--no-clean-urls Don't tidy website URLs (redirects/tracking params)
|
|
671
993
|
|
|
672
|
-
${c("bold", "Filters")}
|
|
994
|
+
${c("bold", "Filters & order")}
|
|
673
995
|
--min-rating <n> Keep only ratings \u2265 n
|
|
674
996
|
--min-reviews <n> Keep only \u2265 n reviews
|
|
675
997
|
--has-phone Keep only leads with a phone
|
|
676
998
|
--has-website Keep only leads with a website
|
|
677
999
|
--has-email Keep only leads with an email (implies --emails)
|
|
678
1000
|
--dedupe <key> website | phone | name | none (default website)
|
|
1001
|
+
--sort <key> rating | reviews | name | priceLevel
|
|
1002
|
+
--desc / --asc Sort direction
|
|
679
1003
|
|
|
680
1004
|
${c("bold", "Output")}
|
|
681
|
-
-f, --format <fmt> json | csv | xlsx
|
|
682
|
-
-o, --out <file> Output file (default:
|
|
1005
|
+
-f, --format <fmt> json | ndjson | csv | xlsx (default json)
|
|
1006
|
+
-o, --out <file> Output file, or "-" for stdout (default: slug + date)
|
|
683
1007
|
--sheet <name> Excel sheet name (default "Leads")
|
|
684
1008
|
|
|
685
1009
|
${c("bold", "Runtime")}
|
|
686
1010
|
--delay <ms> Pause between listings (default 700)
|
|
687
1011
|
--max-time <s> Stop collecting after n seconds
|
|
1012
|
+
--lang <locale> Browser locale, e.g. en-US, ne-NP (default en-US)
|
|
1013
|
+
--region <cc> Region bias for results, e.g. np, us
|
|
688
1014
|
--headless Run the browser without a window
|
|
689
1015
|
-y, --yes Skip prompts + the browser-open confirmation
|
|
690
1016
|
-h, --help Show this help
|
|
@@ -692,7 +1018,9 @@ ${c("bold", "Runtime")}
|
|
|
692
1018
|
${c("bold", "Examples")}
|
|
693
1019
|
npx lacspace-leads restaurants --city Kathmandu --area Baneshwor -f xlsx
|
|
694
1020
|
npx lacspace-leads "dental clinic" --city Pokhara --emails --has-email -f csv -n 40
|
|
695
|
-
npx lacspace-leads gyms --city Lalitpur --min-rating 4 --
|
|
1021
|
+
npx lacspace-leads gyms --city Lalitpur --min-rating 4 --sort reviews --desc
|
|
1022
|
+
npx lacspace-leads cafes --city Kathmandu --area "Thamel,Baneshwor,Patan" --country NP
|
|
1023
|
+
npx lacspace-leads salons --city Pokhara --preset outreach --country NP -f csv -o -
|
|
696
1024
|
npx lacspace-leads convert leads.json -f xlsx
|
|
697
1025
|
|
|
698
1026
|
${c("dim", "Please scrape responsibly: keep volumes small, respect Google's Terms of")}
|
|
@@ -767,16 +1095,25 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
|
|
|
767
1095
|
exit(1);
|
|
768
1096
|
return;
|
|
769
1097
|
}
|
|
770
|
-
|
|
1098
|
+
if (!OUTPUT_FORMATS.includes(args.format)) {
|
|
1099
|
+
log(c("red", `
|
|
1100
|
+
\u2717 Unknown format "${args.format}". Use: ${OUTPUT_FORMATS.join(", ")}.`));
|
|
1101
|
+
exit(1);
|
|
1102
|
+
return;
|
|
1103
|
+
}
|
|
1104
|
+
if (args.preset && !resolvePreset(args.preset)) {
|
|
1105
|
+
log(c("red", `
|
|
1106
|
+
\u2717 Unknown preset "${args.preset}". Use: ${Object.keys(FIELD_PRESETS).join(", ")}.`));
|
|
1107
|
+
exit(1);
|
|
1108
|
+
return;
|
|
1109
|
+
}
|
|
1110
|
+
const base = args.fields ? normalizeFields(args.fields) : resolvePreset(args.preset) ?? [...DEFAULT_FIELDS];
|
|
771
1111
|
const wanted = new Set(base);
|
|
772
1112
|
if (args.emails) wanted.add("email");
|
|
773
|
-
if (args.socials)
|
|
774
|
-
wanted.add("facebook");
|
|
775
|
-
wanted.add("instagram");
|
|
776
|
-
wanted.add("whatsapp");
|
|
777
|
-
}
|
|
1113
|
+
if (args.socials) for (const f of SOCIAL_FIELDS) wanted.add(f);
|
|
778
1114
|
const fields = ALL_FIELDS.filter((f) => wanted.has(f));
|
|
779
|
-
const
|
|
1115
|
+
const toStdout = args.out === "-";
|
|
1116
|
+
const out = toStdout ? "-" : resolve2(args.out ?? defaultFilename(query, args.format));
|
|
780
1117
|
const filters = {};
|
|
781
1118
|
if (args.minRating !== void 0) filters.minRating = args.minRating;
|
|
782
1119
|
if (args.minReviews !== void 0) filters.minReviews = args.minReviews;
|
|
@@ -784,12 +1121,29 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
|
|
|
784
1121
|
if (args.hasWebsite) filters.hasWebsite = true;
|
|
785
1122
|
if (args.hasEmail) filters.hasEmail = true;
|
|
786
1123
|
const hasFilters = Object.keys(filters).length > 0;
|
|
787
|
-
|
|
1124
|
+
const queries = expandQueries({
|
|
1125
|
+
type: args.type ?? "",
|
|
1126
|
+
city: args.city ?? "",
|
|
1127
|
+
area: args.area ?? "",
|
|
1128
|
+
query: args.query ?? ""
|
|
1129
|
+
});
|
|
1130
|
+
const isBatch = queries.length > 1;
|
|
1131
|
+
if (isBatch) log(`
|
|
1132
|
+
${c("dim", "searches")} ${c("bold", String(queries.length))} ${c("dim", "\u2192")} ${queries.map((q) => {
|
|
1133
|
+
try {
|
|
1134
|
+
return composeQuery(q);
|
|
1135
|
+
} catch {
|
|
1136
|
+
return "?";
|
|
1137
|
+
}
|
|
1138
|
+
}).join(" \xB7 ")}`);
|
|
1139
|
+
else log(`
|
|
788
1140
|
${c("dim", "search")} ${c("bold", query)}`);
|
|
789
1141
|
log(` ${c("dim", "fields")} ${fields.join(", ")}`);
|
|
790
|
-
if (args.emails || args.socials) log(` ${c("dim", "enrich")} ${[args.emails && "emails", args.socials && "socials"].filter(Boolean).join(" + ")} ${c("dim",
|
|
1142
|
+
if (args.emails || args.socials) log(` ${c("dim", "enrich")} ${[args.emails && "emails", args.socials && "socials"].filter(Boolean).join(" + ")} ${c("dim", `(${args.concurrency ?? 3}\xD7 parallel, visits each website)`)}`);
|
|
791
1143
|
if (hasFilters) log(` ${c("dim", "filters")} ${Object.entries(filters).map(([k, v]) => `${k}=${v}`).join(", ")}`);
|
|
792
|
-
log(` ${c("dim", "
|
|
1144
|
+
if (args.sort) log(` ${c("dim", "sort")} ${args.sort} ${args.desc === false ? "asc" : "desc"}`);
|
|
1145
|
+
if (args.country) log(` ${c("dim", "phones")} E.164 for ${args.country}`);
|
|
1146
|
+
log(` ${c("dim", "limit")} ${args.limit}${isBatch ? "/search" : ""}${args.total ? ` (cap ${args.total})` : ""} ${c("dim", "format")} ${args.format} ${c("dim", "\u2192")} ${toStdout ? "stdout" : out}`);
|
|
793
1147
|
if (!args.yes) {
|
|
794
1148
|
const ok = await prompt(`
|
|
795
1149
|
${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c("dim", "[y/N]")} `);
|
|
@@ -814,16 +1168,29 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
|
|
|
814
1168
|
headless: args.headless,
|
|
815
1169
|
details: args.details,
|
|
816
1170
|
delayMs: args.delay,
|
|
1171
|
+
cleanUrls: args.cleanUrls,
|
|
817
1172
|
enrich: args.emails || args.socials,
|
|
818
1173
|
signal: controller.signal,
|
|
819
1174
|
onProgress: (m) => log(` ${c("cyan", "\u25F7")} ${c("dim", m)}`)
|
|
820
1175
|
};
|
|
821
1176
|
if (hasFilters) opts.filters = filters;
|
|
822
1177
|
if (args.dedupe) opts.dedupe = args.dedupe;
|
|
1178
|
+
if (args.sort) opts.sort = args.sort;
|
|
1179
|
+
if (args.desc !== void 0) opts.sortDir = args.desc ? "desc" : "asc";
|
|
1180
|
+
if (args.country) opts.country = args.country;
|
|
1181
|
+
if (args.locale) opts.locale = args.locale;
|
|
1182
|
+
if (args.region) opts.region = args.region;
|
|
1183
|
+
if (args.concurrency !== void 0) opts.concurrency = args.concurrency;
|
|
823
1184
|
if (args.maxTime !== void 0) opts.maxMs = args.maxTime * 1e3;
|
|
824
1185
|
let leads;
|
|
825
1186
|
try {
|
|
826
|
-
|
|
1187
|
+
if (isBatch) {
|
|
1188
|
+
const batchOpts = { ...opts };
|
|
1189
|
+
if (args.total !== void 0) batchOpts.total = args.total;
|
|
1190
|
+
leads = await searchLeadsBatch(queries, batchOpts);
|
|
1191
|
+
} else {
|
|
1192
|
+
leads = await scrapeLeads(opts);
|
|
1193
|
+
}
|
|
827
1194
|
} catch (err) {
|
|
828
1195
|
log(c("red", `
|
|
829
1196
|
\u2717 ${err.message}`));
|
|
@@ -839,14 +1206,24 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
|
|
|
839
1206
|
const serOpts = {};
|
|
840
1207
|
if (args.sheet) serOpts.sheetName = args.sheet;
|
|
841
1208
|
const { data, binary } = serialize(leads, args.format, fields, serOpts);
|
|
1209
|
+
if (toStdout) {
|
|
1210
|
+
stdout.write(binary ? Buffer.from(data) : data);
|
|
1211
|
+
if (!binary) stdout.write("\n");
|
|
1212
|
+
log(`
|
|
1213
|
+
${c("green", "\u2714")} ${c("bold", String(leads.length))} leads written to stdout ${c("dim", `(${args.format})`)}
|
|
1214
|
+
`);
|
|
1215
|
+
return;
|
|
1216
|
+
}
|
|
842
1217
|
writeFileSync(out, binary ? Buffer.from(data) : data);
|
|
843
1218
|
const withPhone = leads.filter((l) => l.phone).length;
|
|
844
1219
|
const withSite = leads.filter((l) => l.website).length;
|
|
845
1220
|
const withEmail = leads.filter((l) => l.email).length;
|
|
1221
|
+
const withSocial = leads.filter((l) => l.facebook || l.instagram || l.linkedin || l.twitter).length;
|
|
846
1222
|
log(`
|
|
847
1223
|
${c("green", "\u2714")} Saved ${c("bold", String(leads.length))} leads \u2192 ${c("cyan", out)}`);
|
|
848
1224
|
const stats = [`${withPhone} with a phone`, `${withSite} with a website`];
|
|
849
1225
|
if (args.emails) stats.push(`${withEmail} with an email`);
|
|
1226
|
+
if (args.socials) stats.push(`${withSocial} with a social link`);
|
|
850
1227
|
log(` ${c("dim", stats.join(" \xB7 "))}
|
|
851
1228
|
`);
|
|
852
1229
|
}
|