linkedin-secret-sauce 0.11.0 → 0.12.0

package/dist/enrichment/utils/http-retry.js

@@ -0,0 +1,162 @@
+"use strict";
+/**
+ * HTTP Retry Utilities
+ *
+ * Shared utilities for HTTP requests with retry logic, rate limit handling,
+ * and exponential backoff. Used across all enrichment providers.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.delay = delay;
+exports.truthy = truthy;
+exports.mapVerifiedStatus = mapVerifiedStatus;
+exports.fetchWithRetry = fetchWithRetry;
+exports.getWithRetry = getWithRetry;
+exports.postWithRetry = postWithRetry;
+exports.safeJsonParse = safeJsonParse;
+/**
+ * Delay execution for specified milliseconds
+ */
+async function delay(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+/**
+ * Check if a value is truthy (not null, undefined, or empty string)
+ */
+function truthy(v) {
+    return v !== undefined && v !== null && String(v).length > 0;
+}
+/**
+ * Map common verification status strings to boolean
+ */
+function mapVerifiedStatus(status) {
+    if (!status)
+        return undefined;
+    const s = String(status).toLowerCase();
+    if (s === 'valid' || s === 'verified' || s === 'deliverable')
+        return true;
+    if (s === 'invalid' || s === 'unverified' || s === 'undeliverable')
+        return false;
+    return undefined; // catch-all/unknown/webmail -> leave undefined
+}
+/**
+ * HTTP request with retry on rate limit and transient errors
+ *
+ * Features:
+ * - Exponential backoff on rate limit (429)
+ * - Retry on server errors (502, 503, 504)
+ * - Configurable timeout
+ * - Returns typed response
+ *
+ * @example
+ * ```typescript
+ * const response = await fetchWithRetry<MyApiResponse>(
+ *   'https://api.example.com/data',
+ *   {
+ *     method: 'POST',
+ *     headers: { 'Content-Type': 'application/json' },
+ *     body: JSON.stringify({ query: 'test' }),
+ *   },
+ *   { retries: 2, backoffMs: 300 }
+ * );
+ * ```
+ */
+async function fetchWithRetry(url, init, options) {
+    const { retries = 1, backoffMs = 200, timeoutMs = 30000, retryOnStatus = [429, 502, 503, 504], } = options ?? {};
+    let lastErr = null;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const res = await fetch(url, {
+                ...init,
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            // Check if we should retry based on status
+            if (retryOnStatus.includes(res.status) && attempt < retries) {
+                const waitMs = backoffMs * Math.pow(2, attempt);
+                await delay(waitMs);
+                continue;
+            }
+            // Handle non-OK responses
+            if (!res.ok) {
+                const errorText = await res.text().catch(() => '');
+                lastErr = new Error(`HTTP ${res.status}: ${res.statusText}${errorText ? ` - ${errorText.slice(0, 200)}` : ''}`);
+                break;
+            }
+            const json = await res.json();
+            return json;
+        }
+        catch (err) {
+            clearTimeout(timeoutId);
+            // Handle abort/timeout
+            if (err instanceof Error && err.name === 'AbortError') {
+                lastErr = new Error(`Request timeout after ${timeoutMs}ms`);
+            }
+            else {
+                lastErr = err instanceof Error ? err : new Error(String(err));
+            }
+            // Retry on network errors
+            if (attempt < retries) {
+                const waitMs = backoffMs * Math.pow(2, attempt);
+                await delay(waitMs);
+                continue;
+            }
+        }
+    }
+    throw lastErr ?? new Error('Request failed');
+}
+/**
+ * Simpler GET request with retry (no request body)
+ *
+ * @example
+ * ```typescript
+ * const data = await getWithRetry<UserData>(
+ *   `https://api.example.com/users/${id}`,
+ *   { 'Authorization': `Bearer ${token}` }
+ * );
+ * ```
+ */
+async function getWithRetry(url, headers, options) {
+    return fetchWithRetry(url, {
+        method: 'GET',
+        headers: {
+            'Accept': 'application/json',
+            ...headers,
+        },
+    }, options);
+}
+/**
+ * POST request with JSON body and retry
+ *
+ * @example
+ * ```typescript
+ * const result = await postWithRetry<SearchResult>(
+ *   'https://api.example.com/search',
+ *   { query: 'test', limit: 10 },
+ *   { 'X-Api-Key': apiKey }
+ * );
+ * ```
+ */
+async function postWithRetry(url, body, headers, options) {
+    return fetchWithRetry(url, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            ...headers,
+        },
+        body: JSON.stringify(body),
+    }, options);
+}
+/**
+ * Safe JSON parse with fallback
+ */
+function safeJsonParse(str, fallback) {
+    try {
+        return JSON.parse(str);
+    }
+    catch {
+        return fallback;
+    }
+}

package/dist/enrichment/verification/index.d.ts

@@ -1,4 +1,4 @@
 /**
  * Email Verification Module
  */
-export { verifyEmailMx } from './mx';
+export { verifyEmailMx, checkDomainCatchAll, verifyEmailsExist } from './mx';

package/dist/enrichment/verification/index.js

@@ -3,6 +3,8 @@
  * Email Verification Module
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.verifyEmailMx = void 0;
+exports.verifyEmailsExist = exports.checkDomainCatchAll = exports.verifyEmailMx = void 0;
 var mx_1 = require("./mx");
 Object.defineProperty(exports, "verifyEmailMx", { enumerable: true, get: function () { return mx_1.verifyEmailMx; } });
+Object.defineProperty(exports, "checkDomainCatchAll", { enumerable: true, get: function () { return mx_1.checkDomainCatchAll; } });
+Object.defineProperty(exports, "verifyEmailsExist", { enumerable: true, get: function () { return mx_1.verifyEmailsExist; } });

package/dist/enrichment/verification/mx.d.ts

@@ -2,8 +2,25 @@
  * MX Record Resolution and Email Verification
  *
  * Provides MX record resolution with timeout support and confidence scoring.
+ * Includes SMTP-based catch-all detection using RCPT TO verification.
  */
 import type { VerificationResult } from '../types';
+/**
+ * Verify multiple email addresses with rate limiting
+ *
+ * @param emails - Array of email addresses to verify
+ * @param options - Options including delay between checks
+ * @returns Array of verification results
+ */
+export declare function verifyEmailsExist(emails: string[], options?: {
+    delayMs?: number;
+    timeoutMs?: number;
+}): Promise<Array<{
+    email: string;
+    exists: boolean | null;
+    mxHost: string | null;
+    error?: string;
+}>>;
 /**
  * Verify an email address via MX record lookup
  *
@@ -11,6 +28,22 @@ import type { VerificationResult } from '../types';
  * @param options - Verification options
  * @returns Verification result with confidence score
  */
+/**
+ * Check if a domain is a catch-all (accepts all email addresses)
+ *
+ * @param domain - Domain to check
+ * @param options - Options including timeout
+ * @returns Object with isCatchAll status and MX records
+ */
+export declare function checkDomainCatchAll(domain: string, options?: {
+    timeoutMs?: number;
+}): Promise<{
+    isCatchAll: boolean | null;
+    mxRecords: string[];
+    mxHost: string | null;
+    error?: string;
+}>;
 export declare function verifyEmailMx(email: string, options?: {
     timeoutMs?: number;
+    checkCatchAll?: boolean;
 }): Promise<VerificationResult>;

package/dist/enrichment/verification/mx.js

@@ -3,12 +3,52 @@
  * MX Record Resolution and Email Verification
  *
  * Provides MX record resolution with timeout support and confidence scoring.
+ * Includes SMTP-based catch-all detection using RCPT TO verification.
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.verifyEmailsExist = verifyEmailsExist;
+exports.checkDomainCatchAll = checkDomainCatchAll;
 exports.verifyEmailMx = verifyEmailMx;
 const promises_1 = require("node:dns/promises");
+const net = __importStar(require("node:net"));
 const disposable_domains_1 = require("../utils/disposable-domains");
 const validation_1 = require("../utils/validation");
+// Cache for catch-all results (domain -> isCatchAll)
+const catchAllCache = new Map();
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
 /**
  * Resolve MX records with timeout
  *
@@ -30,6 +70,273 @@ async function resolveMxWithTimeout(domain, timeoutMs) {
         clearTimeout(timeoutId);
     }
 }
+/**
+ * Generate a random string for catch-all testing
+ */
+function generateRandomLocalPart() {
+    const chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
+    let result = 'catchalltest_';
+    for (let i = 0; i < 16; i++) {
+        result += chars.charAt(Math.floor(Math.random() * chars.length));
+    }
+    return result;
+}
+/**
+ * Detect if a domain is a catch-all by testing with a random email address
+ * Uses SMTP RCPT TO verification
+ *
+ * @param domain - Domain to test
+ * @param mxHost - MX server hostname
+ * @param timeoutMs - Timeout in milliseconds
+ * @returns true if catch-all, false if not, null if unable to determine
+ */
+async function detectCatchAll(domain, mxHost, timeoutMs = 10000) {
+    // Check cache first
+    const cached = catchAllCache.get(domain);
+    if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
+        return cached.isCatchAll;
+    }
+    return new Promise((resolve) => {
+        const randomEmail = `${generateRandomLocalPart()}@${domain}`;
+        const socket = new net.Socket();
+        let step = 0;
+        let responseBuffer = '';
+        const cleanup = () => {
+            socket.removeAllListeners();
+            socket.destroy();
+        };
+        const timeout = setTimeout(() => {
+            cleanup();
+            resolve(null); // Unable to determine
+        }, timeoutMs);
+        socket.on('error', () => {
+            clearTimeout(timeout);
+            cleanup();
+            resolve(null); // Unable to determine
+        });
+        socket.on('close', () => {
+            clearTimeout(timeout);
+            cleanup();
+        });
+        socket.on('data', (data) => {
+            responseBuffer += data.toString();
+            // Process complete lines
+            const lines = responseBuffer.split('\r\n');
+            responseBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+            for (const line of lines) {
+                if (!line)
+                    continue;
+                const code = parseInt(line.substring(0, 3), 10);
+                switch (step) {
+                    case 0: // Waiting for greeting
+                        if (code === 220) {
+                            step = 1;
+                            socket.write(`HELO verify.local\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 1: // Waiting for HELO response
+                        if (code === 250) {
+                            step = 2;
+                            socket.write(`MAIL FROM:<test@verify.local>\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 2: // Waiting for MAIL FROM response
+                        if (code === 250) {
+                            step = 3;
+                            socket.write(`RCPT TO:<${randomEmail}>\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 3: // Waiting for RCPT TO response - this is the key!
+                        clearTimeout(timeout);
+                        socket.write('QUIT\r\n');
+                        // 250 = accepted (catch-all)
+                        // 550, 551, 552, 553 = rejected (not catch-all)
+                        // Other codes = unable to determine
+                        const isCatchAll = code === 250;
+                        const isRejected = code >= 550 && code <= 559;
+                        if (isCatchAll || isRejected) {
+                            // Cache the result
+                            catchAllCache.set(domain, {
+                                isCatchAll,
+                                timestamp: Date.now(),
+                            });
+                            cleanup();
+                            resolve(isCatchAll);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                }
+            }
+        });
+        // Connect to MX server on port 25
+        socket.connect(25, mxHost);
+    });
+}
+/**
+ * Verify if a specific email address exists via SMTP RCPT TO
+ *
+ * @param email - Email address to verify
+ * @param mxHost - MX server hostname
+ * @param timeoutMs - Timeout in milliseconds
+ * @returns true if exists, false if not, null if unable to determine
+ */
+async function verifyEmailExists(email, mxHost, timeoutMs = 10000) {
+    return new Promise((resolve) => {
+        const socket = new net.Socket();
+        let step = 0;
+        let responseBuffer = '';
+        const cleanup = () => {
+            socket.removeAllListeners();
+            socket.destroy();
+        };
+        const timeout = setTimeout(() => {
+            cleanup();
+            resolve(null);
+        }, timeoutMs);
+        socket.on('error', () => {
+            clearTimeout(timeout);
+            cleanup();
+            resolve(null);
+        });
+        socket.on('close', () => {
+            clearTimeout(timeout);
+            cleanup();
+        });
+        socket.on('data', (data) => {
+            responseBuffer += data.toString();
+            const lines = responseBuffer.split('\r\n');
+            responseBuffer = lines.pop() || '';
+            for (const line of lines) {
+                if (!line)
+                    continue;
+                const code = parseInt(line.substring(0, 3), 10);
+                switch (step) {
+                    case 0: // Waiting for greeting
+                        if (code === 220) {
+                            step = 1;
+                            socket.write(`HELO verify.local\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 1: // Waiting for HELO response
+                        if (code === 250) {
+                            step = 2;
+                            socket.write(`MAIL FROM:<test@verify.local>\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 2: // Waiting for MAIL FROM response
+                        if (code === 250) {
+                            step = 3;
+                            socket.write(`RCPT TO:<${email}>\r\n`);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                    case 3: // Waiting for RCPT TO response
+                        clearTimeout(timeout);
+                        socket.write('QUIT\r\n');
+                        // 250 = accepted (email exists or catch-all)
+                        // 550, 551, 552, 553 = rejected (email doesn't exist)
+                        const exists = code === 250;
+                        const rejected = code >= 550 && code <= 559;
+                        if (exists || rejected) {
+                            cleanup();
+                            resolve(exists);
+                        }
+                        else {
+                            cleanup();
+                            resolve(null);
+                        }
+                        break;
+                }
+            }
+        });
+        socket.connect(25, mxHost);
+    });
+}
+/**
+ * Verify multiple email addresses with rate limiting
+ *
+ * @param emails - Array of email addresses to verify
+ * @param options - Options including delay between checks
+ * @returns Array of verification results
+ */
+async function verifyEmailsExist(emails, options) {
+    const delayMs = options?.delayMs ?? 3000; // 3 second delay between checks
+    const timeoutMs = options?.timeoutMs ?? 10000;
+    const results = [];
+    // Group emails by domain to reuse MX lookups
+    const emailsByDomain = new Map();
+    for (const email of emails) {
+        const domain = email.split('@')[1]?.toLowerCase();
+        if (domain) {
+            const existing = emailsByDomain.get(domain) || [];
+            existing.push(email);
+            emailsByDomain.set(domain, existing);
+        }
+        else {
+            results.push({ email, exists: null, mxHost: null, error: 'Invalid email format' });
+        }
+    }
+    // Process each domain
+    for (const [domain, domainEmails] of emailsByDomain) {
+        // Get MX records for domain
+        const mx = await resolveMxWithTimeout(domain, 5000);
+        if (!mx || mx.length === 0) {
+            for (const email of domainEmails) {
+                results.push({ email, exists: null, mxHost: null, error: 'No MX records found' });
+            }
+            continue;
+        }
+        const sortedMx = mx.sort((a, b) => a.priority - b.priority);
+        const mxHost = sortedMx[0].exchange;
+        // Verify each email with delay
+        for (let i = 0; i < domainEmails.length; i++) {
+            const email = domainEmails[i];
+            // Add delay between checks (except for first one)
+            if (i > 0) {
+                await new Promise((resolve) => setTimeout(resolve, delayMs));
+            }
+            try {
+                const exists = await verifyEmailExists(email, mxHost, timeoutMs);
+                results.push({ email, exists, mxHost });
+            }
+            catch (err) {
+                results.push({
+                    email,
+                    exists: null,
+                    mxHost,
+                    error: err instanceof Error ? err.message : 'Verification failed',
+                });
+            }
+        }
+    }
+    return results;
+}
 /**
  * Calculate confidence score based on verification factors
  */
@@ -43,9 +350,14 @@ function calculateConfidence(params) {
         score += 20;
     else if (mxCount === 1)
         score += 10;
-    // Non-catch-all bonus (we assume catch-all if MX exists but can't verify)
-    if (!isCatchAll)
+    // Catch-all scoring:
+    // - Not catch-all (false): +30 bonus
+    // - Is catch-all (true): no bonus
+    // - Unknown (null): +15 (partial bonus)
+    if (isCatchAll === false)
         score += 30;
+    else if (isCatchAll === null)
+        score += 15;
     // Non-role account bonus
     if (!isRoleAccount)
         score += 20;
@@ -61,8 +373,50 @@ function calculateConfidence(params) {
  * @param options - Verification options
  * @returns Verification result with confidence score
  */
+/**
+ * Check if a domain is a catch-all (accepts all email addresses)
+ *
+ * @param domain - Domain to check
+ * @param options - Options including timeout
+ * @returns Object with isCatchAll status and MX records
+ */
+async function checkDomainCatchAll(domain, options) {
+    const timeoutMs = options?.timeoutMs ?? 10000;
+    try {
+        // Get MX records
+        const mx = await resolveMxWithTimeout(domain, 5000);
+        if (!mx || mx.length === 0) {
+            return {
+                isCatchAll: null,
+                mxRecords: [],
+                mxHost: null,
+                error: 'No MX records found',
+            };
+        }
+        // Sort by priority and get primary
+        const sortedMx = mx.sort((a, b) => a.priority - b.priority);
+        const primaryMxHost = sortedMx[0].exchange;
+        const mxRecords = sortedMx.map((m) => m.exchange);
+        // Detect catch-all
+        const isCatchAll = await detectCatchAll(domain, primaryMxHost, timeoutMs);
+        return {
+            isCatchAll,
+            mxRecords,
+            mxHost: primaryMxHost,
+        };
+    }
+    catch (err) {
+        return {
+            isCatchAll: null,
+            mxRecords: [],
+            mxHost: null,
+            error: err instanceof Error ? err.message : 'Unknown error',
+        };
+    }
+}
 async function verifyEmailMx(email, options) {
     const timeoutMs = options?.timeoutMs ?? 5000;
+    const checkCatchAll = options?.checkCatchAll ?? false;
     // Step 1: Syntax validation
     if (!(0, validation_1.isValidEmailSyntax)(email)) {
         return {
@@ -113,8 +467,14 @@ async function verifyEmailMx(email, options) {
                 mxRecords: [],
             };
         }
-        // Conservative: assume catch-all if MX exists (full detection requires SMTP handshake)
-        const catchAll = true;
+        // Sort MX records by priority (lowest first)
+        const sortedMx = mx.sort((a, b) => a.priority - b.priority);
+        const primaryMxHost = sortedMx[0].exchange;
+        // Detect catch-all if requested (uses SMTP RCPT TO verification)
+        let catchAll = null;
+        if (checkCatchAll) {
+            catchAll = await detectCatchAll(domain, primaryMxHost, timeoutMs);
+        }
         // Calculate confidence
         const confidence = calculateConfidence({
             mxValid: true,
@@ -127,17 +487,17 @@ async function verifyEmailMx(email, options) {
         // Determine reason
         let reason = 'valid';
         if (!valid) {
-            if (catchAll)
+            if (catchAll === true)
                 reason = 'catch_all';
             else if (roleAccount)
                 reason = 'role_account';
         }
-        const mxRecords = mx.map((m) => m.exchange);
+        const mxRecords = sortedMx.map((m) => m.exchange);
         return {
             valid,
             confidence,
             reason,
-            isCatchAll: catchAll,
+            isCatchAll: catchAll ?? false, // Default to false if unknown
             isRoleAccount: roleAccount,
             isDisposable: false,
             mxRecords,