linkedin-secret-sauce 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +512 -232
- package/dist/enrichment/auth/smartlead-auth.d.ts +3 -3
- package/dist/enrichment/auth/smartlead-auth.js +25 -25
- package/dist/enrichment/index.d.ts +6 -4
- package/dist/enrichment/index.js +25 -24
- package/dist/enrichment/matching.d.ts +8 -3
- package/dist/enrichment/matching.js +7 -5
- package/dist/enrichment/orchestrator.js +44 -14
- package/dist/enrichment/providers/construct.js +72 -14
- package/dist/enrichment/providers/hunter.js +6 -60
- package/dist/enrichment/providers/index.d.ts +0 -1
- package/dist/enrichment/providers/index.js +1 -3
- package/dist/enrichment/providers/ldd.js +5 -47
- package/dist/enrichment/providers/smartprospect.js +9 -14
- package/dist/enrichment/types.d.ts +23 -24
- package/dist/enrichment/types.js +22 -21
- package/dist/enrichment/utils/http-retry.d.ts +96 -0
- package/dist/enrichment/utils/http-retry.js +162 -0
- package/dist/enrichment/verification/index.d.ts +1 -1
- package/dist/enrichment/verification/index.js +3 -1
- package/dist/enrichment/verification/mx.d.ts +33 -0
- package/dist/enrichment/verification/mx.js +367 -7
- package/dist/index.d.ts +196 -6
- package/dist/index.js +159 -12
- package/dist/parsers/search-parser.js +7 -3
- package/package.json +30 -22
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Email Enrichment Providers
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.clearSnovioTokenCache = exports.verifyEmailWithSnovio = exports.findEmailsWithSnovio = exports.createSnovioProvider = exports.verifyEmailsBatch = exports.checkCatchAllDomain = exports.verifyEmailWithBouncer = exports.createBouncerProvider = exports.createDropcontactProvider = exports.
|
|
6
|
+
exports.clearSnovioTokenCache = exports.verifyEmailWithSnovio = exports.findEmailsWithSnovio = exports.createSnovioProvider = exports.verifyEmailsBatch = exports.checkCatchAllDomain = exports.verifyEmailWithBouncer = exports.createBouncerProvider = exports.createDropcontactProvider = exports.createHunterProvider = exports.createSmartProspectProvider = exports.createLddProvider = exports.createConstructProvider = void 0;
|
|
7
7
|
var construct_1 = require("./construct");
|
|
8
8
|
Object.defineProperty(exports, "createConstructProvider", { enumerable: true, get: function () { return construct_1.createConstructProvider; } });
|
|
9
9
|
var ldd_1 = require("./ldd");
|
|
@@ -12,8 +12,6 @@ var smartprospect_1 = require("./smartprospect");
|
|
|
12
12
|
Object.defineProperty(exports, "createSmartProspectProvider", { enumerable: true, get: function () { return smartprospect_1.createSmartProspectProvider; } });
|
|
13
13
|
var hunter_1 = require("./hunter");
|
|
14
14
|
Object.defineProperty(exports, "createHunterProvider", { enumerable: true, get: function () { return hunter_1.createHunterProvider; } });
|
|
15
|
-
var apollo_1 = require("./apollo");
|
|
16
|
-
Object.defineProperty(exports, "createApolloProvider", { enumerable: true, get: function () { return apollo_1.createApolloProvider; } });
|
|
17
15
|
var dropcontact_1 = require("./dropcontact");
|
|
18
16
|
Object.defineProperty(exports, "createDropcontactProvider", { enumerable: true, get: function () { return dropcontact_1.createDropcontactProvider; } });
|
|
19
17
|
var bouncer_1 = require("./bouncer");
|
|
@@ -13,43 +13,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
13
13
|
exports.extractNumericLinkedInId = extractNumericLinkedInId;
|
|
14
14
|
exports.createLddProvider = createLddProvider;
|
|
15
15
|
const validation_1 = require("../utils/validation");
|
|
16
|
-
|
|
17
|
-
* Delay helper for retry logic
|
|
18
|
-
*/
|
|
19
|
-
async function delay(ms) {
|
|
20
|
-
return new Promise((r) => setTimeout(r, ms));
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* HTTP request with retry on 429 rate limit
|
|
24
|
-
*/
|
|
25
|
-
async function requestWithRetry(url, token, retries = 1, backoffMs = 200) {
|
|
26
|
-
let lastErr;
|
|
27
|
-
for (let i = 0; i <= retries; i++) {
|
|
28
|
-
try {
|
|
29
|
-
const res = await fetch(url, {
|
|
30
|
-
method: "GET",
|
|
31
|
-
headers: {
|
|
32
|
-
Authorization: `Bearer ${token}`,
|
|
33
|
-
"Content-Type": "application/json",
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
// Retry on rate limit
|
|
37
|
-
if (res?.status === 429 && i < retries) {
|
|
38
|
-
await delay(backoffMs * Math.pow(2, i));
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
return res;
|
|
42
|
-
}
|
|
43
|
-
catch (err) {
|
|
44
|
-
lastErr = err;
|
|
45
|
-
if (i < retries) {
|
|
46
|
-
await delay(backoffMs * Math.pow(2, i));
|
|
47
|
-
continue;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
throw lastErr ?? new Error("ldd_http_error");
|
|
52
|
-
}
|
|
16
|
+
const http_retry_1 = require("../utils/http-retry");
|
|
53
17
|
/**
|
|
54
18
|
* Extract numeric LinkedIn ID from various formats:
|
|
55
19
|
* - Direct number: "307567"
|
|
@@ -162,11 +126,8 @@ function createLddProvider(config) {
|
|
|
162
126
|
async function lookupByNumericId(numericId) {
|
|
163
127
|
try {
|
|
164
128
|
const endpoint = `${apiUrl}/api/v1/profiles/by-numeric-id/${encodeURIComponent(numericId)}`;
|
|
165
|
-
const response = await
|
|
166
|
-
|
|
167
|
-
return null;
|
|
168
|
-
}
|
|
169
|
-
return parseResponse(await response.json());
|
|
129
|
+
const response = await (0, http_retry_1.getWithRetry)(endpoint, { Authorization: `Bearer ${apiToken}` }, { retries: 1, backoffMs: 100 });
|
|
130
|
+
return parseResponse(response);
|
|
170
131
|
}
|
|
171
132
|
catch {
|
|
172
133
|
return null;
|
|
@@ -175,11 +136,8 @@ function createLddProvider(config) {
|
|
|
175
136
|
async function lookupByUsername(username) {
|
|
176
137
|
try {
|
|
177
138
|
const endpoint = `${apiUrl}/api/v1/profiles/by-username/${encodeURIComponent(username)}`;
|
|
178
|
-
const response = await
|
|
179
|
-
|
|
180
|
-
return null;
|
|
181
|
-
}
|
|
182
|
-
return parseResponse(await response.json());
|
|
139
|
+
const response = await (0, http_retry_1.getWithRetry)(endpoint, { Authorization: `Bearer ${apiToken}` }, { retries: 1, backoffMs: 100 });
|
|
140
|
+
return parseResponse(response);
|
|
183
141
|
}
|
|
184
142
|
catch {
|
|
185
143
|
return null;
|
|
@@ -17,13 +17,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
17
17
|
exports.createSmartProspectProvider = createSmartProspectProvider;
|
|
18
18
|
exports.createSmartProspectClient = createSmartProspectClient;
|
|
19
19
|
const smartlead_auth_1 = require("../auth/smartlead-auth");
|
|
20
|
+
const http_retry_1 = require("../utils/http-retry");
|
|
20
21
|
const DEFAULT_API_URL = "https://prospect-api.smartlead.ai/api/search-email-leads";
|
|
21
|
-
/**
|
|
22
|
-
* Delay helper for retry logic
|
|
23
|
-
*/
|
|
24
|
-
async function delay(ms) {
|
|
25
|
-
return new Promise((r) => setTimeout(r, ms));
|
|
26
|
-
}
|
|
27
22
|
const DEFAULT_POLLING_CONFIG = {
|
|
28
23
|
initialDelay: 500,
|
|
29
24
|
pollInterval: 1000,
|
|
@@ -40,7 +35,7 @@ async function requestWithRetry(url, options, retries = 2, backoffMs = 300) {
|
|
|
40
35
|
const res = await fetch(url, options);
|
|
41
36
|
// Retry on rate limit
|
|
42
37
|
if (res.status === 429 && i < retries) {
|
|
43
|
-
await delay(backoffMs * Math.pow(2, i));
|
|
38
|
+
await (0, http_retry_1.delay)(backoffMs * Math.pow(2, i));
|
|
44
39
|
continue;
|
|
45
40
|
}
|
|
46
41
|
if (!res.ok) {
|
|
@@ -52,7 +47,7 @@ async function requestWithRetry(url, options, retries = 2, backoffMs = 300) {
|
|
|
52
47
|
catch (err) {
|
|
53
48
|
lastErr = err;
|
|
54
49
|
if (i < retries) {
|
|
55
|
-
await delay(backoffMs * Math.pow(2, i));
|
|
50
|
+
await (0, http_retry_1.delay)(backoffMs * Math.pow(2, i));
|
|
56
51
|
continue;
|
|
57
52
|
}
|
|
58
53
|
}
|
|
@@ -214,7 +209,7 @@ function createSmartProspectProvider(config) {
|
|
|
214
209
|
async function pollForContactResults(filterId, expectedCount) {
|
|
215
210
|
const { initialDelay, pollInterval, maxAttempts, maxWaitTime } = DEFAULT_POLLING_CONFIG;
|
|
216
211
|
const startTime = Date.now();
|
|
217
|
-
await delay(initialDelay);
|
|
212
|
+
await (0, http_retry_1.delay)(initialDelay);
|
|
218
213
|
const makeRequest = async (token) => {
|
|
219
214
|
return requestWithRetry(`${apiUrl}/get-contacts`, {
|
|
220
215
|
method: "POST",
|
|
@@ -243,13 +238,13 @@ function createSmartProspectProvider(config) {
|
|
|
243
238
|
return result;
|
|
244
239
|
}
|
|
245
240
|
}
|
|
246
|
-
await delay(pollInterval);
|
|
241
|
+
await (0, http_retry_1.delay)(pollInterval);
|
|
247
242
|
}
|
|
248
243
|
catch (err) {
|
|
249
244
|
if (err instanceof Error && err.message.includes("401") && hasCredentials) {
|
|
250
245
|
await handleAuthError();
|
|
251
246
|
}
|
|
252
|
-
await delay(pollInterval);
|
|
247
|
+
await (0, http_retry_1.delay)(pollInterval);
|
|
253
248
|
}
|
|
254
249
|
}
|
|
255
250
|
// Return last result
|
|
@@ -628,7 +623,7 @@ function createSmartProspectClient(config) {
|
|
|
628
623
|
const { initialDelay, pollInterval, maxAttempts, maxWaitTime } = pollingConfig;
|
|
629
624
|
const startTime = Date.now();
|
|
630
625
|
// Wait initial delay before first poll
|
|
631
|
-
await delay(initialDelay);
|
|
626
|
+
await (0, http_retry_1.delay)(initialDelay);
|
|
632
627
|
const makeRequest = async (token) => {
|
|
633
628
|
return requestWithRetry(`${apiUrl}/get-contacts`, {
|
|
634
629
|
method: "POST",
|
|
@@ -660,7 +655,7 @@ function createSmartProspectClient(config) {
|
|
|
660
655
|
}
|
|
661
656
|
}
|
|
662
657
|
// Wait before next poll
|
|
663
|
-
await delay(pollInterval);
|
|
658
|
+
await (0, http_retry_1.delay)(pollInterval);
|
|
664
659
|
}
|
|
665
660
|
catch (err) {
|
|
666
661
|
// On 401, clear token and retry
|
|
@@ -670,7 +665,7 @@ function createSmartProspectClient(config) {
|
|
|
670
665
|
await handleAuthError();
|
|
671
666
|
}
|
|
672
667
|
// Continue polling on errors
|
|
673
|
-
await delay(pollInterval);
|
|
668
|
+
await (0, http_retry_1.delay)(pollInterval);
|
|
674
669
|
}
|
|
675
670
|
}
|
|
676
671
|
// Return last result even if not complete
|
|
@@ -126,12 +126,6 @@ export interface EnrichmentCandidate {
|
|
|
126
126
|
export interface HunterConfig {
|
|
127
127
|
apiKey: string;
|
|
128
128
|
}
|
|
129
|
-
/**
|
|
130
|
-
* Apollo.io provider configuration
|
|
131
|
-
*/
|
|
132
|
-
export interface ApolloConfig {
|
|
133
|
-
apiKey: string;
|
|
134
|
-
}
|
|
135
129
|
/**
|
|
136
130
|
* SmartProspect/Smartlead provider configuration
|
|
137
131
|
*
|
|
@@ -277,6 +271,8 @@ export interface ConstructConfig {
|
|
|
277
271
|
maxAttempts?: number;
|
|
278
272
|
/** Timeout for MX verification in ms (default: 5000) */
|
|
279
273
|
timeoutMs?: number;
|
|
274
|
+
/** Delay between SMTP verification checks in ms (default: 2000) */
|
|
275
|
+
smtpVerifyDelayMs?: number;
|
|
280
276
|
}
|
|
281
277
|
/**
|
|
282
278
|
* All provider configurations
|
|
@@ -286,7 +282,6 @@ export interface ProvidersConfig {
|
|
|
286
282
|
ldd?: LddConfig;
|
|
287
283
|
smartprospect?: SmartProspectConfig;
|
|
288
284
|
hunter?: HunterConfig;
|
|
289
|
-
apollo?: ApolloConfig;
|
|
290
285
|
dropcontact?: DropcontactConfig;
|
|
291
286
|
/** Bouncer.io for SMTP email verification (99%+ accuracy) */
|
|
292
287
|
bouncer?: BouncerConfig;
|
|
@@ -301,7 +296,7 @@ export interface EnrichmentOptions {
|
|
|
301
296
|
maxCostPerEmail?: number;
|
|
302
297
|
/** Minimum confidence threshold 0-100 (default: 0) */
|
|
303
298
|
confidenceThreshold?: number;
|
|
304
|
-
/** Provider order (default: ['
|
|
299
|
+
/** Provider order (default: ['ldd', 'smartprospect', 'construct', 'bouncer', 'snovio', 'hunter']) */
|
|
305
300
|
providerOrder?: ProviderName[];
|
|
306
301
|
/** Retry delay in ms on transient errors (default: 200) */
|
|
307
302
|
retryMs?: number;
|
|
@@ -370,30 +365,34 @@ export interface EnrichmentClient {
|
|
|
370
365
|
/**
|
|
371
366
|
* Available provider names
|
|
372
367
|
*/
|
|
373
|
-
export type ProviderName = "construct" | "ldd" | "smartprospect" | "hunter" | "
|
|
368
|
+
export type ProviderName = "construct" | "ldd" | "smartprospect" | "hunter" | "dropcontact" | "bouncer" | "snovio";
|
|
374
369
|
/**
|
|
375
|
-
* Default provider order
|
|
370
|
+
* Default provider order - 2-Phase Strategy
|
|
371
|
+
*
|
|
372
|
+
* PHASE 1 - Free lookups (run in parallel):
|
|
373
|
+
* - ldd: LinkedIn Data Dump - real verified emails (FREE with subscription)
|
|
374
|
+
* - smartprospect: SmartLead API - real verified emails (FREE with subscription)
|
|
375
|
+
* - construct: Pattern guessing + MX check (FREE)
|
|
376
|
+
*
|
|
377
|
+
* PHASE 2 - Paid verification/finding (only if Phase 1 inconclusive):
|
|
378
|
+
* - bouncer: SMTP verify constructed emails ($0.006/email)
|
|
379
|
+
* - snovio: Email finder for catch-all domains ($0.02/email)
|
|
380
|
+
* - hunter: Hunter.io fallback ($0.005/email)
|
|
376
381
|
*
|
|
377
|
-
*
|
|
378
|
-
* 1. construct - FREE pattern guessing with MX check
|
|
379
|
-
* 2. bouncer - SMTP verification of construct results ($0.006/email)
|
|
380
|
-
* 3. ldd - FREE LinkedIn data dump lookup
|
|
381
|
-
* 4. smartprospect - Paid SmartLead lookup ($0.01/email)
|
|
382
|
-
* 5. snovio - Email finder for catch-all domains ($0.02/email)
|
|
383
|
-
* 6. hunter - Hunter.io API ($0.005/email)
|
|
384
|
-
* 7. apollo - FREE Apollo.io lookup
|
|
385
|
-
* 8. dropcontact - Dropcontact API ($0.01/email)
|
|
382
|
+
* Note: dropcontact available but not in default order (expensive at $0.01)
|
|
386
383
|
*/
|
|
387
384
|
export declare const DEFAULT_PROVIDER_ORDER: ProviderName[];
|
|
388
385
|
/**
|
|
389
386
|
* Provider costs in USD per lookup
|
|
390
387
|
*
|
|
391
388
|
* Costs based on 2025 pricing:
|
|
392
|
-
* -
|
|
393
|
-
* -
|
|
394
|
-
* -
|
|
395
|
-
* -
|
|
396
|
-
* -
|
|
389
|
+
* - ldd: FREE (subscription-based)
|
|
390
|
+
* - smartprospect: FREE (included in SmartLead subscription)
|
|
391
|
+
* - construct: FREE (pattern guessing + MX check)
|
|
392
|
+
* - bouncer: $0.006/email (SMTP verification, 99%+ accuracy)
|
|
393
|
+
* - snovio: $0.02/email (email finding + verification)
|
|
394
|
+
* - hunter: $0.005/email
|
|
395
|
+
* - dropcontact: $0.01/email (not in default order)
|
|
397
396
|
*/
|
|
398
397
|
export declare const PROVIDER_COSTS: Record<ProviderName, number>;
|
|
399
398
|
/**
|
package/dist/enrichment/types.js
CHANGED
|
@@ -8,44 +8,45 @@
|
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.SMARTPROSPECT_SUB_INDUSTRIES = exports.PROVIDER_COSTS = exports.DEFAULT_PROVIDER_ORDER = void 0;
|
|
10
10
|
/**
|
|
11
|
-
* Default provider order
|
|
11
|
+
* Default provider order - 2-Phase Strategy
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
13
|
+
* PHASE 1 - Free lookups (run in parallel):
|
|
14
|
+
* - ldd: LinkedIn Data Dump - real verified emails (FREE with subscription)
|
|
15
|
+
* - smartprospect: SmartLead API - real verified emails (FREE with subscription)
|
|
16
|
+
* - construct: Pattern guessing + MX check (FREE)
|
|
17
|
+
*
|
|
18
|
+
* PHASE 2 - Paid verification/finding (only if Phase 1 inconclusive):
|
|
19
|
+
* - bouncer: SMTP verify constructed emails ($0.006/email)
|
|
20
|
+
* - snovio: Email finder for catch-all domains ($0.02/email)
|
|
21
|
+
* - hunter: Hunter.io fallback ($0.005/email)
|
|
22
|
+
*
|
|
23
|
+
* Note: dropcontact available but not in default order (expensive at $0.01)
|
|
22
24
|
*/
|
|
23
25
|
exports.DEFAULT_PROVIDER_ORDER = [
|
|
24
|
-
"construct",
|
|
25
|
-
"bouncer",
|
|
26
26
|
"ldd",
|
|
27
27
|
"smartprospect",
|
|
28
|
+
"construct",
|
|
29
|
+
"bouncer",
|
|
28
30
|
"snovio",
|
|
29
31
|
"hunter",
|
|
30
|
-
"apollo",
|
|
31
|
-
"dropcontact",
|
|
32
32
|
];
|
|
33
33
|
/**
|
|
34
34
|
* Provider costs in USD per lookup
|
|
35
35
|
*
|
|
36
36
|
* Costs based on 2025 pricing:
|
|
37
|
-
* -
|
|
38
|
-
* -
|
|
39
|
-
* -
|
|
40
|
-
* -
|
|
41
|
-
* -
|
|
37
|
+
* - ldd: FREE (subscription-based)
|
|
38
|
+
* - smartprospect: FREE (included in SmartLead subscription)
|
|
39
|
+
* - construct: FREE (pattern guessing + MX check)
|
|
40
|
+
* - bouncer: $0.006/email (SMTP verification, 99%+ accuracy)
|
|
41
|
+
* - snovio: $0.02/email (email finding + verification)
|
|
42
|
+
* - hunter: $0.005/email
|
|
43
|
+
* - dropcontact: $0.01/email (not in default order)
|
|
42
44
|
*/
|
|
43
45
|
exports.PROVIDER_COSTS = {
|
|
44
46
|
construct: 0,
|
|
45
47
|
ldd: 0,
|
|
46
|
-
smartprospect: 0
|
|
48
|
+
smartprospect: 0,
|
|
47
49
|
hunter: 0.005,
|
|
48
|
-
apollo: 0,
|
|
49
50
|
dropcontact: 0.01,
|
|
50
51
|
bouncer: 0.006,
|
|
51
52
|
snovio: 0.02,
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP Retry Utilities
|
|
3
|
+
*
|
|
4
|
+
* Shared utilities for HTTP requests with retry logic, rate limit handling,
|
|
5
|
+
* and exponential backoff. Used across all enrichment providers.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Delay execution for specified milliseconds
|
|
9
|
+
*/
|
|
10
|
+
export declare function delay(ms: number): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Check if a value is truthy (not null, undefined, or empty string)
|
|
13
|
+
*/
|
|
14
|
+
export declare function truthy(v: unknown): boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Map common verification status strings to boolean
|
|
17
|
+
*/
|
|
18
|
+
export declare function mapVerifiedStatus(status: string | undefined | null): boolean | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* Options for fetch with retry
|
|
21
|
+
*/
|
|
22
|
+
export interface FetchWithRetryOptions {
|
|
23
|
+
/** Number of retries (default: 1) */
|
|
24
|
+
retries?: number;
|
|
25
|
+
/** Initial backoff in ms (default: 200) */
|
|
26
|
+
backoffMs?: number;
|
|
27
|
+
/** Timeout in ms (default: 30000) */
|
|
28
|
+
timeoutMs?: number;
|
|
29
|
+
/** HTTP status codes that should trigger retry (default: [429, 502, 503, 504]) */
|
|
30
|
+
retryOnStatus?: number[];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Generic API response type
|
|
34
|
+
*/
|
|
35
|
+
export interface ApiResponse {
|
|
36
|
+
data?: unknown;
|
|
37
|
+
result?: unknown;
|
|
38
|
+
people?: unknown[];
|
|
39
|
+
matches?: unknown[];
|
|
40
|
+
emails?: unknown[];
|
|
41
|
+
success?: boolean;
|
|
42
|
+
message?: string;
|
|
43
|
+
[key: string]: unknown;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* HTTP request with retry on rate limit and transient errors
|
|
47
|
+
*
|
|
48
|
+
* Features:
|
|
49
|
+
* - Exponential backoff on rate limit (429)
|
|
50
|
+
* - Retry on server errors (502, 503, 504)
|
|
51
|
+
* - Configurable timeout
|
|
52
|
+
* - Returns typed response
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```typescript
|
|
56
|
+
* const response = await fetchWithRetry<MyApiResponse>(
|
|
57
|
+
* 'https://api.example.com/data',
|
|
58
|
+
* {
|
|
59
|
+
* method: 'POST',
|
|
60
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
61
|
+
* body: JSON.stringify({ query: 'test' }),
|
|
62
|
+
* },
|
|
63
|
+
* { retries: 2, backoffMs: 300 }
|
|
64
|
+
* );
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export declare function fetchWithRetry<T = ApiResponse>(url: string, init?: RequestInit, options?: FetchWithRetryOptions): Promise<T>;
|
|
68
|
+
/**
|
|
69
|
+
* Simpler GET request with retry (no request body)
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* ```typescript
|
|
73
|
+
* const data = await getWithRetry<UserData>(
|
|
74
|
+
* `https://api.example.com/users/${id}`,
|
|
75
|
+
* { 'Authorization': `Bearer ${token}` }
|
|
76
|
+
* );
|
|
77
|
+
* ```
|
|
78
|
+
*/
|
|
79
|
+
export declare function getWithRetry<T = ApiResponse>(url: string, headers?: Record<string, string>, options?: FetchWithRetryOptions): Promise<T>;
|
|
80
|
+
/**
|
|
81
|
+
* POST request with JSON body and retry
|
|
82
|
+
*
|
|
83
|
+
* @example
|
|
84
|
+
* ```typescript
|
|
85
|
+
* const result = await postWithRetry<SearchResult>(
|
|
86
|
+
* 'https://api.example.com/search',
|
|
87
|
+
* { query: 'test', limit: 10 },
|
|
88
|
+
* { 'X-Api-Key': apiKey }
|
|
89
|
+
* );
|
|
90
|
+
* ```
|
|
91
|
+
*/
|
|
92
|
+
export declare function postWithRetry<T = ApiResponse>(url: string, body: Record<string, unknown>, headers?: Record<string, string>, options?: FetchWithRetryOptions): Promise<T>;
|
|
93
|
+
/**
|
|
94
|
+
* Safe JSON parse with fallback
|
|
95
|
+
*/
|
|
96
|
+
export declare function safeJsonParse<T>(str: string, fallback: T): T;
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* HTTP Retry Utilities
|
|
4
|
+
*
|
|
5
|
+
* Shared utilities for HTTP requests with retry logic, rate limit handling,
|
|
6
|
+
* and exponential backoff. Used across all enrichment providers.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.delay = delay;
|
|
10
|
+
exports.truthy = truthy;
|
|
11
|
+
exports.mapVerifiedStatus = mapVerifiedStatus;
|
|
12
|
+
exports.fetchWithRetry = fetchWithRetry;
|
|
13
|
+
exports.getWithRetry = getWithRetry;
|
|
14
|
+
exports.postWithRetry = postWithRetry;
|
|
15
|
+
exports.safeJsonParse = safeJsonParse;
|
|
16
|
+
/**
|
|
17
|
+
* Delay execution for specified milliseconds
|
|
18
|
+
*/
|
|
19
|
+
async function delay(ms) {
|
|
20
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Check if a value is truthy (not null, undefined, or empty string)
|
|
24
|
+
*/
|
|
25
|
+
function truthy(v) {
|
|
26
|
+
return v !== undefined && v !== null && String(v).length > 0;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Map common verification status strings to boolean
|
|
30
|
+
*/
|
|
31
|
+
function mapVerifiedStatus(status) {
|
|
32
|
+
if (!status)
|
|
33
|
+
return undefined;
|
|
34
|
+
const s = String(status).toLowerCase();
|
|
35
|
+
if (s === 'valid' || s === 'verified' || s === 'deliverable')
|
|
36
|
+
return true;
|
|
37
|
+
if (s === 'invalid' || s === 'unverified' || s === 'undeliverable')
|
|
38
|
+
return false;
|
|
39
|
+
return undefined; // catch-all/unknown/webmail -> leave undefined
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* HTTP request with retry on rate limit and transient errors
|
|
43
|
+
*
|
|
44
|
+
* Features:
|
|
45
|
+
* - Exponential backoff on rate limit (429)
|
|
46
|
+
* - Retry on server errors (502, 503, 504)
|
|
47
|
+
* - Configurable timeout
|
|
48
|
+
* - Returns typed response
|
|
49
|
+
*
|
|
50
|
+
* @example
|
|
51
|
+
* ```typescript
|
|
52
|
+
* const response = await fetchWithRetry<MyApiResponse>(
|
|
53
|
+
* 'https://api.example.com/data',
|
|
54
|
+
* {
|
|
55
|
+
* method: 'POST',
|
|
56
|
+
* headers: { 'Content-Type': 'application/json' },
|
|
57
|
+
* body: JSON.stringify({ query: 'test' }),
|
|
58
|
+
* },
|
|
59
|
+
* { retries: 2, backoffMs: 300 }
|
|
60
|
+
* );
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
async function fetchWithRetry(url, init, options) {
|
|
64
|
+
const { retries = 1, backoffMs = 200, timeoutMs = 30000, retryOnStatus = [429, 502, 503, 504], } = options ?? {};
|
|
65
|
+
let lastErr = null;
|
|
66
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
67
|
+
const controller = new AbortController();
|
|
68
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
69
|
+
try {
|
|
70
|
+
const res = await fetch(url, {
|
|
71
|
+
...init,
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
});
|
|
74
|
+
clearTimeout(timeoutId);
|
|
75
|
+
// Check if we should retry based on status
|
|
76
|
+
if (retryOnStatus.includes(res.status) && attempt < retries) {
|
|
77
|
+
const waitMs = backoffMs * Math.pow(2, attempt);
|
|
78
|
+
await delay(waitMs);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
// Handle non-OK responses
|
|
82
|
+
if (!res.ok) {
|
|
83
|
+
const errorText = await res.text().catch(() => '');
|
|
84
|
+
lastErr = new Error(`HTTP ${res.status}: ${res.statusText}${errorText ? ` - ${errorText.slice(0, 200)}` : ''}`);
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
const json = await res.json();
|
|
88
|
+
return json;
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
clearTimeout(timeoutId);
|
|
92
|
+
// Handle abort/timeout
|
|
93
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
94
|
+
lastErr = new Error(`Request timeout after ${timeoutMs}ms`);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
lastErr = err instanceof Error ? err : new Error(String(err));
|
|
98
|
+
}
|
|
99
|
+
// Retry on network errors
|
|
100
|
+
if (attempt < retries) {
|
|
101
|
+
const waitMs = backoffMs * Math.pow(2, attempt);
|
|
102
|
+
await delay(waitMs);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
throw lastErr ?? new Error('Request failed');
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Simpler GET request with retry (no request body)
|
|
111
|
+
*
|
|
112
|
+
* @example
|
|
113
|
+
* ```typescript
|
|
114
|
+
* const data = await getWithRetry<UserData>(
|
|
115
|
+
* `https://api.example.com/users/${id}`,
|
|
116
|
+
* { 'Authorization': `Bearer ${token}` }
|
|
117
|
+
* );
|
|
118
|
+
* ```
|
|
119
|
+
*/
|
|
120
|
+
async function getWithRetry(url, headers, options) {
|
|
121
|
+
return fetchWithRetry(url, {
|
|
122
|
+
method: 'GET',
|
|
123
|
+
headers: {
|
|
124
|
+
'Accept': 'application/json',
|
|
125
|
+
...headers,
|
|
126
|
+
},
|
|
127
|
+
}, options);
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* POST request with JSON body and retry
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```typescript
|
|
134
|
+
* const result = await postWithRetry<SearchResult>(
|
|
135
|
+
* 'https://api.example.com/search',
|
|
136
|
+
* { query: 'test', limit: 10 },
|
|
137
|
+
* { 'X-Api-Key': apiKey }
|
|
138
|
+
* );
|
|
139
|
+
* ```
|
|
140
|
+
*/
|
|
141
|
+
async function postWithRetry(url, body, headers, options) {
|
|
142
|
+
return fetchWithRetry(url, {
|
|
143
|
+
method: 'POST',
|
|
144
|
+
headers: {
|
|
145
|
+
'Content-Type': 'application/json',
|
|
146
|
+
'Accept': 'application/json',
|
|
147
|
+
...headers,
|
|
148
|
+
},
|
|
149
|
+
body: JSON.stringify(body),
|
|
150
|
+
}, options);
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Safe JSON parse with fallback
|
|
154
|
+
*/
|
|
155
|
+
function safeJsonParse(str, fallback) {
|
|
156
|
+
try {
|
|
157
|
+
return JSON.parse(str);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
return fallback;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
* Email Verification Module
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.verifyEmailMx = void 0;
|
|
6
|
+
exports.verifyEmailsExist = exports.checkDomainCatchAll = exports.verifyEmailMx = void 0;
|
|
7
7
|
var mx_1 = require("./mx");
|
|
8
8
|
Object.defineProperty(exports, "verifyEmailMx", { enumerable: true, get: function () { return mx_1.verifyEmailMx; } });
|
|
9
|
+
Object.defineProperty(exports, "checkDomainCatchAll", { enumerable: true, get: function () { return mx_1.checkDomainCatchAll; } });
|
|
10
|
+
Object.defineProperty(exports, "verifyEmailsExist", { enumerable: true, get: function () { return mx_1.verifyEmailsExist; } });
|
|
@@ -2,8 +2,25 @@
|
|
|
2
2
|
* MX Record Resolution and Email Verification
|
|
3
3
|
*
|
|
4
4
|
* Provides MX record resolution with timeout support and confidence scoring.
|
|
5
|
+
* Includes SMTP-based catch-all detection using RCPT TO verification.
|
|
5
6
|
*/
|
|
6
7
|
import type { VerificationResult } from '../types';
|
|
8
|
+
/**
|
|
9
|
+
* Verify multiple email addresses with rate limiting
|
|
10
|
+
*
|
|
11
|
+
* @param emails - Array of email addresses to verify
|
|
12
|
+
* @param options - Options including delay between checks
|
|
13
|
+
* @returns Array of verification results
|
|
14
|
+
*/
|
|
15
|
+
export declare function verifyEmailsExist(emails: string[], options?: {
|
|
16
|
+
delayMs?: number;
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
}): Promise<Array<{
|
|
19
|
+
email: string;
|
|
20
|
+
exists: boolean | null;
|
|
21
|
+
mxHost: string | null;
|
|
22
|
+
error?: string;
|
|
23
|
+
}>>;
|
|
7
24
|
/**
|
|
8
25
|
* Verify an email address via MX record lookup
|
|
9
26
|
*
|
|
@@ -11,6 +28,22 @@ import type { VerificationResult } from '../types';
|
|
|
11
28
|
* @param options - Verification options
|
|
12
29
|
* @returns Verification result with confidence score
|
|
13
30
|
*/
|
|
31
|
+
/**
|
|
32
|
+
* Check if a domain is a catch-all (accepts all email addresses)
|
|
33
|
+
*
|
|
34
|
+
* @param domain - Domain to check
|
|
35
|
+
* @param options - Options including timeout
|
|
36
|
+
* @returns Object with isCatchAll status and MX records
|
|
37
|
+
*/
|
|
38
|
+
export declare function checkDomainCatchAll(domain: string, options?: {
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
}): Promise<{
|
|
41
|
+
isCatchAll: boolean | null;
|
|
42
|
+
mxRecords: string[];
|
|
43
|
+
mxHost: string | null;
|
|
44
|
+
error?: string;
|
|
45
|
+
}>;
|
|
14
46
|
export declare function verifyEmailMx(email: string, options?: {
|
|
15
47
|
timeoutMs?: number;
|
|
48
|
+
checkCatchAll?: boolean;
|
|
16
49
|
}): Promise<VerificationResult>;
|