lacspace-leads 0.2.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -16
- package/dist/cli.js +680 -126
- package/dist/lib.cjs +577 -99
- package/dist/lib.d.cts +221 -8
- package/dist/lib.d.ts +221 -8
- package/dist/lib.js +554 -94
- package/package.json +7 -2
package/dist/lib.d.cts
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
/** The fields a {@link Lead} can carry — request any subset via `fields`. */
|
|
2
|
-
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
2
|
+
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "emailStatus" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
3
|
+
/** Deliverability verdict for a discovered email (see `verifyEmails`). */
|
|
4
|
+
type EmailStatus = "valid" | "no-mx" | "invalid-format" | "unknown";
|
|
3
5
|
/** Every field, in a sensible column order for exports. */
|
|
4
6
|
declare const ALL_FIELDS: LeadField[];
|
|
5
7
|
/** Fields that require visiting the business website (via `enrich`). */
|
|
6
8
|
declare const ENRICHED_FIELDS: LeadField[];
|
|
9
|
+
/** Named field bundles for common jobs — pass via `preset` / `--preset`. */
|
|
10
|
+
declare const FIELD_PRESETS: Record<string, LeadField[]>;
|
|
11
|
+
/**
|
|
12
|
+
* Fields collected by default — everything Google Maps shows directly, i.e.
|
|
13
|
+
* {@link ALL_FIELDS} minus the {@link ENRICHED_FIELDS} that need a website visit.
|
|
14
|
+
* Ask for the enriched ones explicitly (or via `enrich`) to opt into that work.
|
|
15
|
+
*/
|
|
16
|
+
declare const DEFAULT_FIELDS: LeadField[];
|
|
7
17
|
/** A single collected business lead. Every field is optional — Maps listings vary. */
|
|
8
18
|
interface Lead {
|
|
9
19
|
name?: string;
|
|
@@ -25,6 +35,18 @@ interface Lead {
|
|
|
25
35
|
instagram?: string;
|
|
26
36
|
/** WhatsApp number/link, from the website (needs enrichment). */
|
|
27
37
|
whatsapp?: string;
|
|
38
|
+
/** LinkedIn company/profile URL, from the website (needs enrichment). */
|
|
39
|
+
linkedin?: string;
|
|
40
|
+
/** Twitter / X URL, from the website (needs enrichment). */
|
|
41
|
+
twitter?: string;
|
|
42
|
+
/** YouTube channel URL, from the website (needs enrichment). */
|
|
43
|
+
youtube?: string;
|
|
44
|
+
/** TikTok URL, from the website (needs enrichment). */
|
|
45
|
+
tiktok?: string;
|
|
46
|
+
/** Telegram link, from the website (needs enrichment). */
|
|
47
|
+
telegram?: string;
|
|
48
|
+
/** Email deliverability verdict (set by `verifyEmails`). */
|
|
49
|
+
emailStatus?: EmailStatus;
|
|
28
50
|
/** Google Plus Code, when shown. */
|
|
29
51
|
plusCode?: string;
|
|
30
52
|
/** Latitude, parsed from the listing's Maps URL. */
|
|
@@ -48,9 +70,13 @@ interface LeadFilters {
|
|
|
48
70
|
hasWebsite?: boolean;
|
|
49
71
|
/** Keep only leads that have an email (implies enrichment). */
|
|
50
72
|
hasEmail?: boolean;
|
|
73
|
+
/** Keep only leads whose email passed MX verification (implies `verifyEmails`). */
|
|
74
|
+
hasValidEmail?: boolean;
|
|
51
75
|
}
|
|
52
76
|
/** Output formats the tool can write. */
|
|
53
|
-
type OutputFormat = "json" | "csv" | "xlsx";
|
|
77
|
+
type OutputFormat = "json" | "ndjson" | "csv" | "xlsx";
|
|
78
|
+
/** How to sort collected leads before export. */
|
|
79
|
+
type SortKey$1 = "rating" | "reviews" | "name" | "priceLevel";
|
|
54
80
|
/** Options for a lead search. */
|
|
55
81
|
interface SearchOptions {
|
|
56
82
|
/** City, e.g. "Kathmandu". */
|
|
@@ -83,15 +109,66 @@ interface SearchOptions {
|
|
|
83
109
|
enrich?: boolean;
|
|
84
110
|
/** Drop leads that don't pass these filters. */
|
|
85
111
|
filters?: LeadFilters;
|
|
86
|
-
/**
|
|
87
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Drop duplicate leads by this key. `"smart"` uses the strongest identity
|
|
114
|
+
* available per lead (website → phone → name) and is best for accumulating
|
|
115
|
+
* across runs. Default "website".
|
|
116
|
+
*/
|
|
117
|
+
dedupe?: "website" | "phone" | "name" | "smart" | "none";
|
|
118
|
+
/** Sort the results by this key before returning. */
|
|
119
|
+
sort?: SortKey$1;
|
|
120
|
+
/** Sort direction. Defaults to "desc" for numbers, "asc" for name. */
|
|
121
|
+
sortDir?: "asc" | "desc";
|
|
122
|
+
/**
|
|
123
|
+
* Normalise phone numbers to E.164 using this default country — an ISO-2 code
|
|
124
|
+
* (`"NP"`, `"US"`) or a raw calling code (`"977"`). Best-effort; numbers that
|
|
125
|
+
* can't be parsed are left as-is.
|
|
126
|
+
*/
|
|
127
|
+
country?: string;
|
|
128
|
+
/**
|
|
129
|
+
* Tidy website URLs — unwrap Google redirects and strip tracking params.
|
|
130
|
+
* Default true.
|
|
131
|
+
*/
|
|
132
|
+
cleanUrls?: boolean;
|
|
133
|
+
/** How many websites to enrich in parallel. Default 3. */
|
|
134
|
+
concurrency?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Verify each discovered email by checking its domain has MX records (DNS,
|
|
137
|
+
* no message is sent). Populates `emailStatus`. Implies `enrich`. Default off.
|
|
138
|
+
*/
|
|
139
|
+
verifyEmails?: boolean;
|
|
140
|
+
/** Browser/UI locale, e.g. "en-US", "ne-NP". Default "en-US". */
|
|
141
|
+
locale?: string;
|
|
142
|
+
/** Google region bias (ccTLD-style), e.g. "np", "us". Sets Maps `gl`. */
|
|
143
|
+
region?: string;
|
|
144
|
+
/** Route the browser through this proxy, e.g. "http://user:pass@host:port". */
|
|
145
|
+
proxy?: string;
|
|
146
|
+
/** Retry a listing that fails to open, up to this many times. Default 1. */
|
|
147
|
+
retries?: number;
|
|
148
|
+
/** Randomise the delay between listings by ±40% to look more human. */
|
|
149
|
+
jitter?: boolean;
|
|
88
150
|
/** Stop collecting after this many milliseconds (best-effort). */
|
|
89
151
|
maxMs?: number;
|
|
90
152
|
/** Called with a short progress message as the search runs. */
|
|
91
153
|
onProgress?: (message: string) => void;
|
|
154
|
+
/**
|
|
155
|
+
* Called with each lead as it's collected (before enrichment), for live UIs
|
|
156
|
+
* or crash-safe incremental writing. Never throws into the run.
|
|
157
|
+
*/
|
|
158
|
+
onLead?: (lead: Lead) => void;
|
|
92
159
|
/** An AbortSignal to cancel a running search. */
|
|
93
160
|
signal?: AbortSignal;
|
|
94
161
|
}
|
|
162
|
+
/** Aggregate counts over a lead list — see `computeStats`. */
|
|
163
|
+
interface LeadStats {
|
|
164
|
+
total: number;
|
|
165
|
+
withPhone: number;
|
|
166
|
+
withWebsite: number;
|
|
167
|
+
withEmail: number;
|
|
168
|
+
withValidEmail: number;
|
|
169
|
+
withSocial: number;
|
|
170
|
+
avgRating?: number;
|
|
171
|
+
}
|
|
95
172
|
|
|
96
173
|
/** Error thrown by the scraper. Carries a machine `code` and optional cause. */
|
|
97
174
|
declare class LeadsError extends Error {
|
|
@@ -121,6 +198,44 @@ declare function parseLatLng(url: string): {
|
|
|
121
198
|
* @throws {LeadsError} when no browser can be launched, or the search is aborted.
|
|
122
199
|
*/
|
|
123
200
|
declare function scrapeLeads(opts: SearchOptions): Promise<Lead[]>;
|
|
201
|
+
/** A search result with aggregate stats and timing. */
|
|
202
|
+
interface DetailedResult {
|
|
203
|
+
leads: Lead[];
|
|
204
|
+
stats: LeadStats;
|
|
205
|
+
/** Wall-clock time of the search, in milliseconds. */
|
|
206
|
+
elapsedMs: number;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Like {@link scrapeLeads}, but also returns aggregate {@link LeadStats} and the
|
|
210
|
+
* elapsed time — handy for dashboards, reports and CI summaries.
|
|
211
|
+
*/
|
|
212
|
+
declare function searchLeadsDetailed(opts: SearchOptions): Promise<DetailedResult>;
|
|
213
|
+
|
|
214
|
+
/** One sub-search in a batch. */
|
|
215
|
+
interface BatchQuery {
|
|
216
|
+
type?: string;
|
|
217
|
+
city?: string;
|
|
218
|
+
area?: string;
|
|
219
|
+
query?: string;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Run each `{type, city, area}` search in turn and merge the results, applying
|
|
223
|
+
* a single cross-search dedupe and sort at the end. Per-search `limit` is the
|
|
224
|
+
* cap for *each* sub-search; `opts.total`, when set, caps the merged list.
|
|
225
|
+
*
|
|
226
|
+
* Filtering and sorting are applied once, globally, here — so pass them via
|
|
227
|
+
* `opts` rather than relying on the per-search pass.
|
|
228
|
+
*/
|
|
229
|
+
declare function searchLeadsBatch(queries: BatchQuery[], opts?: Omit<SearchOptions, "type" | "city" | "area" | "query"> & {
|
|
230
|
+
total?: number;
|
|
231
|
+
}): Promise<Lead[]>;
|
|
232
|
+
/**
|
|
233
|
+
* Convenience wrapper: expand a single request with comma-separated
|
|
234
|
+
* `type`/`city`/`area` into its cross-product and run it as a batch.
|
|
235
|
+
*/
|
|
236
|
+
declare function searchLeadsMulti(opts: SearchOptions & {
|
|
237
|
+
total?: number;
|
|
238
|
+
}): Promise<Lead[]>;
|
|
124
239
|
|
|
125
240
|
/** Project leads onto exactly the requested fields, in order, as header-keyed rows. */
|
|
126
241
|
declare function toRows(leads: Lead[], fields?: LeadField[]): Record<string, string | number>[];
|
|
@@ -131,6 +246,37 @@ declare function serialize(leads: Lead[], format: OutputFormat, fields?: LeadFie
|
|
|
131
246
|
data: string | Uint8Array;
|
|
132
247
|
binary: boolean;
|
|
133
248
|
};
|
|
249
|
+
/** Aggregate counts over a lead list — for summaries and reports. Pure. */
|
|
250
|
+
declare function computeStats(leads: Lead[]): LeadStats;
|
|
251
|
+
/**
|
|
252
|
+
* Map arbitrary rows (as read back from a JSON/CSV/Excel export, keyed by field
|
|
253
|
+
* name or header label) into {@link Lead}s. Unknown columns are ignored, blanks
|
|
254
|
+
* dropped, and numeric fields coerced. Powers `--append` / resume. Pure.
|
|
255
|
+
*/
|
|
256
|
+
declare function rowsToLeads(rows: Record<string, unknown>[]): Lead[];
|
|
257
|
+
|
|
258
|
+
/** True when `email` is a plausibly-valid address shape. Pure. */
|
|
259
|
+
declare function emailFormatValid(email: string | undefined): boolean;
|
|
260
|
+
/** The domain part of an email, lower-cased, or undefined. Pure. */
|
|
261
|
+
declare function emailDomain(email: string | undefined): string | undefined;
|
|
262
|
+
/**
|
|
263
|
+
* Classify a single email: `invalid-format` → `no-mx` → `valid`. `unknown` is
|
|
264
|
+
* reserved for a missing address. Never throws.
|
|
265
|
+
*/
|
|
266
|
+
declare function verifyEmail(email: string | undefined, opts?: {
|
|
267
|
+
cache?: Map<string, boolean>;
|
|
268
|
+
timeoutMs?: number;
|
|
269
|
+
}): Promise<EmailStatus>;
|
|
270
|
+
/**
|
|
271
|
+
* Verify the `email` on each lead in place, setting `emailStatus`. Domains are
|
|
272
|
+
* looked up once (shared cache) and checked a few in parallel. Leads without an
|
|
273
|
+
* email are left untouched. Node-only; never throws.
|
|
274
|
+
*/
|
|
275
|
+
declare function verifyEmails(leads: Lead[], opts?: {
|
|
276
|
+
concurrency?: number;
|
|
277
|
+
timeoutMs?: number;
|
|
278
|
+
onProgress?: (m: string) => void;
|
|
279
|
+
}): Promise<Lead[]>;
|
|
134
280
|
|
|
135
281
|
/**
|
|
136
282
|
* Compose the human search query from options: an explicit `query` wins,
|
|
@@ -142,8 +288,32 @@ declare function composeQuery(opts: {
|
|
|
142
288
|
area?: string;
|
|
143
289
|
city?: string;
|
|
144
290
|
}): string;
|
|
145
|
-
/**
|
|
146
|
-
|
|
291
|
+
/**
|
|
292
|
+
* The Google Maps search URL for a query. `hl` (interface language) keeps the
|
|
293
|
+
* scraped aria-labels predictable; `gl` biases results to a region.
|
|
294
|
+
*/
|
|
295
|
+
declare function mapsSearchUrl(query: string, opts?: {
|
|
296
|
+
hl?: string;
|
|
297
|
+
gl?: string;
|
|
298
|
+
}): string;
|
|
299
|
+
/**
|
|
300
|
+
* Expand a possibly-multi search request into individual `{type, city, area}`
|
|
301
|
+
* queries — the cross-product of comma-separated types × cities × areas. An
|
|
302
|
+
* explicit `query` short-circuits to a single verbatim search. Used for batch.
|
|
303
|
+
*/
|
|
304
|
+
declare function expandQueries(opts: {
|
|
305
|
+
type?: string;
|
|
306
|
+
city?: string;
|
|
307
|
+
area?: string;
|
|
308
|
+
query?: string;
|
|
309
|
+
}): {
|
|
310
|
+
type?: string;
|
|
311
|
+
city?: string;
|
|
312
|
+
area?: string;
|
|
313
|
+
query?: string;
|
|
314
|
+
}[];
|
|
315
|
+
/** Resolve a preset name to its field list, case-insensitively. */
|
|
316
|
+
declare function resolvePreset(name?: string): LeadField[] | undefined;
|
|
147
317
|
/**
|
|
148
318
|
* Normalise a fields request (array or comma string) into known {@link LeadField}s
|
|
149
319
|
* in canonical order, de-duplicated. Unknown names are ignored; an empty result
|
|
@@ -161,6 +331,41 @@ declare function filterLeads(leads: Lead[], filters?: LeadFilters): Lead[];
|
|
|
161
331
|
*/
|
|
162
332
|
declare function dedupeLeads(leads: Lead[], by?: NonNullable<SearchOptions["dedupe"]>): Lead[];
|
|
163
333
|
|
|
334
|
+
/**
|
|
335
|
+
* Pure data-cleaning helpers — website tidy-up, best-effort E.164 phone
|
|
336
|
+
* normalisation, and lead sorting. All exported so they can be unit-tested and
|
|
337
|
+
* reused; none touch the network or the disk.
|
|
338
|
+
*/
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Tidy a website URL: unwrap a Google `/url?q=…` redirect, drop tracking query
|
|
342
|
+
* params (utm_*, fbclid, gclid…) and any fragment, and lower-case the host.
|
|
343
|
+
* Returns `undefined` for empty input, and the original string if it can't be
|
|
344
|
+
* parsed as a URL. Pure.
|
|
345
|
+
*/
|
|
346
|
+
declare function cleanWebsite(url?: string): string | undefined;
|
|
347
|
+
/**
|
|
348
|
+
* ISO-3166 alpha-2 → E.164 country calling code, for a curated set of common
|
|
349
|
+
* markets. Extend as needed; unknown codes fall through to best-effort.
|
|
350
|
+
*/
|
|
351
|
+
declare const CALLING_CODES: Record<string, string>;
|
|
352
|
+
/** Resolve a country argument (ISO-2 code or a raw calling code) to digits. */
|
|
353
|
+
declare function callingCode(country?: string): string | undefined;
|
|
354
|
+
/**
|
|
355
|
+
* Best-effort E.164 normalisation of a phone string given a default country
|
|
356
|
+
* (an ISO-2 code like `"NP"` or a calling code like `"977"`). Numbers that
|
|
357
|
+
* already start with `+` are kept as international. When the result isn't a
|
|
358
|
+
* plausible 8–15-digit number, the original string is returned unchanged. Pure.
|
|
359
|
+
*/
|
|
360
|
+
declare function normalizePhone(raw?: string, country?: string): string | undefined;
|
|
361
|
+
/** Keys a lead list can be sorted by. */
|
|
362
|
+
type SortKey = "rating" | "reviews" | "name" | "priceLevel";
|
|
363
|
+
/**
|
|
364
|
+
* Sort leads by a key, missing values always last. Stable, pure, new array.
|
|
365
|
+
* `dir` defaults to descending for numeric keys and ascending for `name`.
|
|
366
|
+
*/
|
|
367
|
+
declare function sortLeads(leads: Lead[], by?: SortKey, dir?: "asc" | "desc"): Lead[];
|
|
368
|
+
|
|
164
369
|
/**
|
|
165
370
|
* Website enrichment — visit a business's site and pull an email + social
|
|
166
371
|
* links. Uses global `fetch` (Node 20+); every network call is guarded so a
|
|
@@ -172,10 +377,18 @@ interface Contacts {
|
|
|
172
377
|
facebook?: string;
|
|
173
378
|
instagram?: string;
|
|
174
379
|
whatsapp?: string;
|
|
380
|
+
linkedin?: string;
|
|
381
|
+
twitter?: string;
|
|
382
|
+
youtube?: string;
|
|
383
|
+
tiktok?: string;
|
|
384
|
+
telegram?: string;
|
|
175
385
|
}
|
|
176
386
|
/** Extract the best contact email from HTML, or `undefined`. Pure. */
|
|
177
387
|
declare function extractEmails(html: string): string | undefined;
|
|
178
|
-
/**
|
|
388
|
+
/**
|
|
389
|
+
* Extract social links (Facebook, Instagram, WhatsApp, LinkedIn, Twitter/X,
|
|
390
|
+
* YouTube, TikTok, Telegram) from HTML. Skips share/intent/widget links. Pure.
|
|
391
|
+
*/
|
|
179
392
|
declare function extractSocials(html: string): Omit<Contacts, "email">;
|
|
180
393
|
/**
|
|
181
394
|
* Fetch a website (home page, then a /contact page if needed) and pull an email
|
|
@@ -212,4 +425,4 @@ declare function convertFile(input: string, opts?: {
|
|
|
212
425
|
count: number;
|
|
213
426
|
}>;
|
|
214
427
|
|
|
215
|
-
export { ALL_FIELDS, type Contacts, type DataRow, ENRICHED_FIELDS, type Lead, type LeadField, type LeadFilters, LeadsError, type OutputFormat, type SearchOptions, columnsOf, composeQuery, convertFile, dedupeLeads, defaultFilename, detectFormat, enrichContacts, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, parseLatLng, parseRating, parseReviewCount, readRows, scrapeLeads, scrapeLeads as searchLeads, serialize, serializeRows, toRows };
|
|
428
|
+
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, type DetailedResult, ENRICHED_FIELDS, type EmailStatus, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, type LeadStats, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, computeStats, convertFile, dedupeLeads, defaultFilename, detectFormat, emailDomain, emailFormatValid, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, rowsToLeads, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsDetailed, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows, verifyEmail, verifyEmails };
|
package/dist/lib.d.ts
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
/** The fields a {@link Lead} can carry — request any subset via `fields`. */
|
|
2
|
-
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
2
|
+
type LeadField = "name" | "category" | "rating" | "reviews" | "priceLevel" | "address" | "phone" | "website" | "email" | "facebook" | "instagram" | "whatsapp" | "linkedin" | "twitter" | "youtube" | "tiktok" | "telegram" | "emailStatus" | "plusCode" | "latitude" | "longitude" | "hours" | "mapsUrl";
|
|
3
|
+
/** Deliverability verdict for a discovered email (see `verifyEmails`). */
|
|
4
|
+
type EmailStatus = "valid" | "no-mx" | "invalid-format" | "unknown";
|
|
3
5
|
/** Every field, in a sensible column order for exports. */
|
|
4
6
|
declare const ALL_FIELDS: LeadField[];
|
|
5
7
|
/** Fields that require visiting the business website (via `enrich`). */
|
|
6
8
|
declare const ENRICHED_FIELDS: LeadField[];
|
|
9
|
+
/** Named field bundles for common jobs — pass via `preset` / `--preset`. */
|
|
10
|
+
declare const FIELD_PRESETS: Record<string, LeadField[]>;
|
|
11
|
+
/**
|
|
12
|
+
* Fields collected by default — everything Google Maps shows directly, i.e.
|
|
13
|
+
* {@link ALL_FIELDS} minus the {@link ENRICHED_FIELDS} that need a website visit.
|
|
14
|
+
* Ask for the enriched ones explicitly (or via `enrich`) to opt into that work.
|
|
15
|
+
*/
|
|
16
|
+
declare const DEFAULT_FIELDS: LeadField[];
|
|
7
17
|
/** A single collected business lead. Every field is optional — Maps listings vary. */
|
|
8
18
|
interface Lead {
|
|
9
19
|
name?: string;
|
|
@@ -25,6 +35,18 @@ interface Lead {
|
|
|
25
35
|
instagram?: string;
|
|
26
36
|
/** WhatsApp number/link, from the website (needs enrichment). */
|
|
27
37
|
whatsapp?: string;
|
|
38
|
+
/** LinkedIn company/profile URL, from the website (needs enrichment). */
|
|
39
|
+
linkedin?: string;
|
|
40
|
+
/** Twitter / X URL, from the website (needs enrichment). */
|
|
41
|
+
twitter?: string;
|
|
42
|
+
/** YouTube channel URL, from the website (needs enrichment). */
|
|
43
|
+
youtube?: string;
|
|
44
|
+
/** TikTok URL, from the website (needs enrichment). */
|
|
45
|
+
tiktok?: string;
|
|
46
|
+
/** Telegram link, from the website (needs enrichment). */
|
|
47
|
+
telegram?: string;
|
|
48
|
+
/** Email deliverability verdict (set by `verifyEmails`). */
|
|
49
|
+
emailStatus?: EmailStatus;
|
|
28
50
|
/** Google Plus Code, when shown. */
|
|
29
51
|
plusCode?: string;
|
|
30
52
|
/** Latitude, parsed from the listing's Maps URL. */
|
|
@@ -48,9 +70,13 @@ interface LeadFilters {
|
|
|
48
70
|
hasWebsite?: boolean;
|
|
49
71
|
/** Keep only leads that have an email (implies enrichment). */
|
|
50
72
|
hasEmail?: boolean;
|
|
73
|
+
/** Keep only leads whose email passed MX verification (implies `verifyEmails`). */
|
|
74
|
+
hasValidEmail?: boolean;
|
|
51
75
|
}
|
|
52
76
|
/** Output formats the tool can write. */
|
|
53
|
-
type OutputFormat = "json" | "csv" | "xlsx";
|
|
77
|
+
type OutputFormat = "json" | "ndjson" | "csv" | "xlsx";
|
|
78
|
+
/** How to sort collected leads before export. */
|
|
79
|
+
type SortKey$1 = "rating" | "reviews" | "name" | "priceLevel";
|
|
54
80
|
/** Options for a lead search. */
|
|
55
81
|
interface SearchOptions {
|
|
56
82
|
/** City, e.g. "Kathmandu". */
|
|
@@ -83,15 +109,66 @@ interface SearchOptions {
|
|
|
83
109
|
enrich?: boolean;
|
|
84
110
|
/** Drop leads that don't pass these filters. */
|
|
85
111
|
filters?: LeadFilters;
|
|
86
|
-
/**
|
|
87
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Drop duplicate leads by this key. `"smart"` uses the strongest identity
|
|
114
|
+
* available per lead (website → phone → name) and is best for accumulating
|
|
115
|
+
* across runs. Default "website".
|
|
116
|
+
*/
|
|
117
|
+
dedupe?: "website" | "phone" | "name" | "smart" | "none";
|
|
118
|
+
/** Sort the results by this key before returning. */
|
|
119
|
+
sort?: SortKey$1;
|
|
120
|
+
/** Sort direction. Defaults to "desc" for numbers, "asc" for name. */
|
|
121
|
+
sortDir?: "asc" | "desc";
|
|
122
|
+
/**
|
|
123
|
+
* Normalise phone numbers to E.164 using this default country — an ISO-2 code
|
|
124
|
+
* (`"NP"`, `"US"`) or a raw calling code (`"977"`). Best-effort; numbers that
|
|
125
|
+
* can't be parsed are left as-is.
|
|
126
|
+
*/
|
|
127
|
+
country?: string;
|
|
128
|
+
/**
|
|
129
|
+
* Tidy website URLs — unwrap Google redirects and strip tracking params.
|
|
130
|
+
* Default true.
|
|
131
|
+
*/
|
|
132
|
+
cleanUrls?: boolean;
|
|
133
|
+
/** How many websites to enrich in parallel. Default 3. */
|
|
134
|
+
concurrency?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Verify each discovered email by checking its domain has MX records (DNS,
|
|
137
|
+
* no message is sent). Populates `emailStatus`. Implies `enrich`. Default off.
|
|
138
|
+
*/
|
|
139
|
+
verifyEmails?: boolean;
|
|
140
|
+
/** Browser/UI locale, e.g. "en-US", "ne-NP". Default "en-US". */
|
|
141
|
+
locale?: string;
|
|
142
|
+
/** Google region bias (ccTLD-style), e.g. "np", "us". Sets Maps `gl`. */
|
|
143
|
+
region?: string;
|
|
144
|
+
/** Route the browser through this proxy, e.g. "http://user:pass@host:port". */
|
|
145
|
+
proxy?: string;
|
|
146
|
+
/** Retry a listing that fails to open, up to this many times. Default 1. */
|
|
147
|
+
retries?: number;
|
|
148
|
+
/** Randomise the delay between listings by ±40% to look more human. */
|
|
149
|
+
jitter?: boolean;
|
|
88
150
|
/** Stop collecting after this many milliseconds (best-effort). */
|
|
89
151
|
maxMs?: number;
|
|
90
152
|
/** Called with a short progress message as the search runs. */
|
|
91
153
|
onProgress?: (message: string) => void;
|
|
154
|
+
/**
|
|
155
|
+
* Called with each lead as it's collected (before enrichment), for live UIs
|
|
156
|
+
* or crash-safe incremental writing. Never throws into the run.
|
|
157
|
+
*/
|
|
158
|
+
onLead?: (lead: Lead) => void;
|
|
92
159
|
/** An AbortSignal to cancel a running search. */
|
|
93
160
|
signal?: AbortSignal;
|
|
94
161
|
}
|
|
162
|
+
/** Aggregate counts over a lead list — see `computeStats`. */
|
|
163
|
+
interface LeadStats {
|
|
164
|
+
total: number;
|
|
165
|
+
withPhone: number;
|
|
166
|
+
withWebsite: number;
|
|
167
|
+
withEmail: number;
|
|
168
|
+
withValidEmail: number;
|
|
169
|
+
withSocial: number;
|
|
170
|
+
avgRating?: number;
|
|
171
|
+
}
|
|
95
172
|
|
|
96
173
|
/** Error thrown by the scraper. Carries a machine `code` and optional cause. */
|
|
97
174
|
declare class LeadsError extends Error {
|
|
@@ -121,6 +198,44 @@ declare function parseLatLng(url: string): {
|
|
|
121
198
|
* @throws {LeadsError} when no browser can be launched, or the search is aborted.
|
|
122
199
|
*/
|
|
123
200
|
declare function scrapeLeads(opts: SearchOptions): Promise<Lead[]>;
|
|
201
|
+
/** A search result with aggregate stats and timing. */
|
|
202
|
+
interface DetailedResult {
|
|
203
|
+
leads: Lead[];
|
|
204
|
+
stats: LeadStats;
|
|
205
|
+
/** Wall-clock time of the search, in milliseconds. */
|
|
206
|
+
elapsedMs: number;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Like {@link scrapeLeads}, but also returns aggregate {@link LeadStats} and the
|
|
210
|
+
* elapsed time — handy for dashboards, reports and CI summaries.
|
|
211
|
+
*/
|
|
212
|
+
declare function searchLeadsDetailed(opts: SearchOptions): Promise<DetailedResult>;
|
|
213
|
+
|
|
214
|
+
/** One sub-search in a batch. */
|
|
215
|
+
interface BatchQuery {
|
|
216
|
+
type?: string;
|
|
217
|
+
city?: string;
|
|
218
|
+
area?: string;
|
|
219
|
+
query?: string;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Run each `{type, city, area}` search in turn and merge the results, applying
|
|
223
|
+
* a single cross-search dedupe and sort at the end. Per-search `limit` is the
|
|
224
|
+
* cap for *each* sub-search; `opts.total`, when set, caps the merged list.
|
|
225
|
+
*
|
|
226
|
+
* Filtering and sorting are applied once, globally, here — so pass them via
|
|
227
|
+
* `opts` rather than relying on the per-search pass.
|
|
228
|
+
*/
|
|
229
|
+
declare function searchLeadsBatch(queries: BatchQuery[], opts?: Omit<SearchOptions, "type" | "city" | "area" | "query"> & {
|
|
230
|
+
total?: number;
|
|
231
|
+
}): Promise<Lead[]>;
|
|
232
|
+
/**
|
|
233
|
+
* Convenience wrapper: expand a single request with comma-separated
|
|
234
|
+
* `type`/`city`/`area` into its cross-product and run it as a batch.
|
|
235
|
+
*/
|
|
236
|
+
declare function searchLeadsMulti(opts: SearchOptions & {
|
|
237
|
+
total?: number;
|
|
238
|
+
}): Promise<Lead[]>;
|
|
124
239
|
|
|
125
240
|
/** Project leads onto exactly the requested fields, in order, as header-keyed rows. */
|
|
126
241
|
declare function toRows(leads: Lead[], fields?: LeadField[]): Record<string, string | number>[];
|
|
@@ -131,6 +246,37 @@ declare function serialize(leads: Lead[], format: OutputFormat, fields?: LeadFie
|
|
|
131
246
|
data: string | Uint8Array;
|
|
132
247
|
binary: boolean;
|
|
133
248
|
};
|
|
249
|
+
/** Aggregate counts over a lead list — for summaries and reports. Pure. */
|
|
250
|
+
declare function computeStats(leads: Lead[]): LeadStats;
|
|
251
|
+
/**
|
|
252
|
+
* Map arbitrary rows (as read back from a JSON/CSV/Excel export, keyed by field
|
|
253
|
+
* name or header label) into {@link Lead}s. Unknown columns are ignored, blanks
|
|
254
|
+
* dropped, and numeric fields coerced. Powers `--append` / resume. Pure.
|
|
255
|
+
*/
|
|
256
|
+
declare function rowsToLeads(rows: Record<string, unknown>[]): Lead[];
|
|
257
|
+
|
|
258
|
+
/** True when `email` is a plausibly-valid address shape. Pure. */
|
|
259
|
+
declare function emailFormatValid(email: string | undefined): boolean;
|
|
260
|
+
/** The domain part of an email, lower-cased, or undefined. Pure. */
|
|
261
|
+
declare function emailDomain(email: string | undefined): string | undefined;
|
|
262
|
+
/**
|
|
263
|
+
* Classify a single email: `invalid-format` → `no-mx` → `valid`. `unknown` is
|
|
264
|
+
* reserved for a missing address. Never throws.
|
|
265
|
+
*/
|
|
266
|
+
declare function verifyEmail(email: string | undefined, opts?: {
|
|
267
|
+
cache?: Map<string, boolean>;
|
|
268
|
+
timeoutMs?: number;
|
|
269
|
+
}): Promise<EmailStatus>;
|
|
270
|
+
/**
|
|
271
|
+
* Verify the `email` on each lead in place, setting `emailStatus`. Domains are
|
|
272
|
+
* looked up once (shared cache) and checked a few in parallel. Leads without an
|
|
273
|
+
* email are left untouched. Node-only; never throws.
|
|
274
|
+
*/
|
|
275
|
+
declare function verifyEmails(leads: Lead[], opts?: {
|
|
276
|
+
concurrency?: number;
|
|
277
|
+
timeoutMs?: number;
|
|
278
|
+
onProgress?: (m: string) => void;
|
|
279
|
+
}): Promise<Lead[]>;
|
|
134
280
|
|
|
135
281
|
/**
|
|
136
282
|
* Compose the human search query from options: an explicit `query` wins,
|
|
@@ -142,8 +288,32 @@ declare function composeQuery(opts: {
|
|
|
142
288
|
area?: string;
|
|
143
289
|
city?: string;
|
|
144
290
|
}): string;
|
|
145
|
-
/**
|
|
146
|
-
|
|
291
|
+
/**
|
|
292
|
+
* The Google Maps search URL for a query. `hl` (interface language) keeps the
|
|
293
|
+
* scraped aria-labels predictable; `gl` biases results to a region.
|
|
294
|
+
*/
|
|
295
|
+
declare function mapsSearchUrl(query: string, opts?: {
|
|
296
|
+
hl?: string;
|
|
297
|
+
gl?: string;
|
|
298
|
+
}): string;
|
|
299
|
+
/**
|
|
300
|
+
* Expand a possibly-multi search request into individual `{type, city, area}`
|
|
301
|
+
* queries — the cross-product of comma-separated types × cities × areas. An
|
|
302
|
+
* explicit `query` short-circuits to a single verbatim search. Used for batch.
|
|
303
|
+
*/
|
|
304
|
+
declare function expandQueries(opts: {
|
|
305
|
+
type?: string;
|
|
306
|
+
city?: string;
|
|
307
|
+
area?: string;
|
|
308
|
+
query?: string;
|
|
309
|
+
}): {
|
|
310
|
+
type?: string;
|
|
311
|
+
city?: string;
|
|
312
|
+
area?: string;
|
|
313
|
+
query?: string;
|
|
314
|
+
}[];
|
|
315
|
+
/** Resolve a preset name to its field list, case-insensitively. */
|
|
316
|
+
declare function resolvePreset(name?: string): LeadField[] | undefined;
|
|
147
317
|
/**
|
|
148
318
|
* Normalise a fields request (array or comma string) into known {@link LeadField}s
|
|
149
319
|
* in canonical order, de-duplicated. Unknown names are ignored; an empty result
|
|
@@ -161,6 +331,41 @@ declare function filterLeads(leads: Lead[], filters?: LeadFilters): Lead[];
|
|
|
161
331
|
*/
|
|
162
332
|
declare function dedupeLeads(leads: Lead[], by?: NonNullable<SearchOptions["dedupe"]>): Lead[];
|
|
163
333
|
|
|
334
|
+
/**
|
|
335
|
+
* Pure data-cleaning helpers — website tidy-up, best-effort E.164 phone
|
|
336
|
+
* normalisation, and lead sorting. All exported so they can be unit-tested and
|
|
337
|
+
* reused; none touch the network or the disk.
|
|
338
|
+
*/
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Tidy a website URL: unwrap a Google `/url?q=…` redirect, drop tracking query
|
|
342
|
+
* params (utm_*, fbclid, gclid…) and any fragment, and lower-case the host.
|
|
343
|
+
* Returns `undefined` for empty input, and the original string if it can't be
|
|
344
|
+
* parsed as a URL. Pure.
|
|
345
|
+
*/
|
|
346
|
+
declare function cleanWebsite(url?: string): string | undefined;
|
|
347
|
+
/**
|
|
348
|
+
* ISO-3166 alpha-2 → E.164 country calling code, for a curated set of common
|
|
349
|
+
* markets. Extend as needed; unknown codes fall through to best-effort.
|
|
350
|
+
*/
|
|
351
|
+
declare const CALLING_CODES: Record<string, string>;
|
|
352
|
+
/** Resolve a country argument (ISO-2 code or a raw calling code) to digits. */
|
|
353
|
+
declare function callingCode(country?: string): string | undefined;
|
|
354
|
+
/**
|
|
355
|
+
* Best-effort E.164 normalisation of a phone string given a default country
|
|
356
|
+
* (an ISO-2 code like `"NP"` or a calling code like `"977"`). Numbers that
|
|
357
|
+
* already start with `+` are kept as international. When the result isn't a
|
|
358
|
+
* plausible 8–15-digit number, the original string is returned unchanged. Pure.
|
|
359
|
+
*/
|
|
360
|
+
declare function normalizePhone(raw?: string, country?: string): string | undefined;
|
|
361
|
+
/** Keys a lead list can be sorted by. */
|
|
362
|
+
type SortKey = "rating" | "reviews" | "name" | "priceLevel";
|
|
363
|
+
/**
|
|
364
|
+
* Sort leads by a key, missing values always last. Stable, pure, new array.
|
|
365
|
+
* `dir` defaults to descending for numeric keys and ascending for `name`.
|
|
366
|
+
*/
|
|
367
|
+
declare function sortLeads(leads: Lead[], by?: SortKey, dir?: "asc" | "desc"): Lead[];
|
|
368
|
+
|
|
164
369
|
/**
|
|
165
370
|
* Website enrichment — visit a business's site and pull an email + social
|
|
166
371
|
* links. Uses global `fetch` (Node 20+); every network call is guarded so a
|
|
@@ -172,10 +377,18 @@ interface Contacts {
|
|
|
172
377
|
facebook?: string;
|
|
173
378
|
instagram?: string;
|
|
174
379
|
whatsapp?: string;
|
|
380
|
+
linkedin?: string;
|
|
381
|
+
twitter?: string;
|
|
382
|
+
youtube?: string;
|
|
383
|
+
tiktok?: string;
|
|
384
|
+
telegram?: string;
|
|
175
385
|
}
|
|
176
386
|
/** Extract the best contact email from HTML, or `undefined`. Pure. */
|
|
177
387
|
declare function extractEmails(html: string): string | undefined;
|
|
178
|
-
/**
|
|
388
|
+
/**
|
|
389
|
+
* Extract social links (Facebook, Instagram, WhatsApp, LinkedIn, Twitter/X,
|
|
390
|
+
* YouTube, TikTok, Telegram) from HTML. Skips share/intent/widget links. Pure.
|
|
391
|
+
*/
|
|
179
392
|
declare function extractSocials(html: string): Omit<Contacts, "email">;
|
|
180
393
|
/**
|
|
181
394
|
* Fetch a website (home page, then a /contact page if needed) and pull an email
|
|
@@ -212,4 +425,4 @@ declare function convertFile(input: string, opts?: {
|
|
|
212
425
|
count: number;
|
|
213
426
|
}>;
|
|
214
427
|
|
|
215
|
-
export { ALL_FIELDS, type Contacts, type DataRow, ENRICHED_FIELDS, type Lead, type LeadField, type LeadFilters, LeadsError, type OutputFormat, type SearchOptions, columnsOf, composeQuery, convertFile, dedupeLeads, defaultFilename, detectFormat, enrichContacts, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, parseLatLng, parseRating, parseReviewCount, readRows, scrapeLeads, scrapeLeads as searchLeads, serialize, serializeRows, toRows };
|
|
428
|
+
export { ALL_FIELDS, type BatchQuery, CALLING_CODES, type Contacts, DEFAULT_FIELDS, type DataRow, type DetailedResult, ENRICHED_FIELDS, type EmailStatus, FIELD_PRESETS, type Lead, type LeadField, type LeadFilters, type LeadStats, LeadsError, type OutputFormat, type SearchOptions, type SortKey$1 as SortKey, callingCode, cleanWebsite, columnsOf, composeQuery, computeStats, convertFile, dedupeLeads, defaultFilename, detectFormat, emailDomain, emailFormatValid, enrichContacts, expandQueries, extractEmails, extractSocials, filterLeads, mapsSearchUrl, normalizeFields, normalizePhone, parseLatLng, parseRating, parseReviewCount, readRows, resolvePreset, rowsToLeads, scrapeLeads, scrapeLeads as searchLeads, searchLeadsBatch, searchLeadsDetailed, searchLeadsMulti, serialize, serializeRows, sortLeads, toRows, verifyEmail, verifyEmails };
|