mailpop 1.0.0

package/dist/csv.js

@@ -0,0 +1,115 @@
+import * as fs from 'fs';
+import fsPromises from 'fs/promises';
+import path from 'path';
+import * as csv from 'fast-csv';
+import { Logger } from './logger.js';
+/**
+ * Creates an async generator to stream rows from an input CSV file.
+ * This guarantees memory-efficient processing for up to 50k+ rows.
+ * @param filePath - Path to the input CSV file.
+ */
+export async function* readCsvGenerator(filePath) {
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`Input CSV file not found: ${filePath}`);
+    }
+    const stream = fs.createReadStream(filePath).pipe(csv.parse({
+        headers: true,
+        trim: true,
+        discardUnmappedColumns: false,
+    }));
+    let index = 0;
+    for await (const row of stream) {
+        yield { row: row, index };
+        index++;
+    }
+}
+/**
+ * Reads only the header row of a CSV file.
+ * Helper to dynamically extract the input schema.
+ */
+export async function getCsvHeaders(filePath) {
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`Input CSV file not found: ${filePath}`);
+    }
+    return new Promise((resolve, reject) => {
+        const stream = fs.createReadStream(filePath);
+        const parser = csv.parse({ headers: true });
+        stream
+            .pipe(parser)
+            .on('headers', (headers) => {
+            stream.destroy();
+            resolve(headers);
+        })
+            .on('error', (err) => {
+            reject(err);
+        });
+    });
+}
+/**
+ * Appends a single row to the output CSV file in a thread-safe and escaped manner.
+ * @param filePath - Path to the output CSV file.
+ * @param row - Output row data.
+ * @param headers - Complete ordered headers array for output alignment.
+ * @param writeHeader - Whether to prefix the row with a header line.
+ */
+export async function appendCsvRow(filePath, row, headers, writeHeader = false) {
+    const dir = path.dirname(filePath);
+    try {
+        await fsPromises.mkdir(dir, { recursive: true });
+    }
+    catch (_e) {
+        // Ignore folder creation errors if it already exists
+    }
+    const csvLine = await new Promise((resolve, reject) => {
+        csv
+            .writeToString([row], {
+            headers: headers,
+            includeEndRowDelimiter: true,
+            writeHeaders: writeHeader,
+        })
+            .then(resolve)
+            .catch(reject);
+    });
+    await fsPromises.appendFile(filePath, csvLine, 'utf-8');
+}
+/**
+ * Loads checkpoint progress from disk. Returns null if missing/corrupt.
+ */
+export async function loadCheckpoint(filePath) {
+    try {
+        const content = await fsPromises.readFile(filePath, 'utf-8');
+        const data = JSON.parse(content);
+        if (typeof data.lastProcessedIndex === 'number' && Array.isArray(data.completedUrls)) {
+            return data;
+        }
+        return null;
+    }
+    catch (_e) {
+        return null;
+    }
+}
+/**
+ * Saves checkpoint progress to disk.
+ */
+export async function saveCheckpoint(filePath, data) {
+    const dir = path.dirname(filePath);
+    try {
+        await fsPromises.mkdir(dir, { recursive: true });
+        await fsPromises.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8');
+    }
+    catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        await Logger.error('checkpoint-save-fail', undefined, undefined, `Failed to save checkpoint: ${errorMsg}`);
+    }
+}
+/**
+ * Deletes the checkpoint file (used upon successful completion of run).
+ */
+export async function clearCheckpoint(filePath) {
+    try {
+        await fsPromises.unlink(filePath);
+    }
+    catch (_e) {
+        // Ignore if checkpoint file doesn't exist
+    }
+}

package/dist/extractor.js

@@ -0,0 +1,290 @@
+import { load } from 'cheerio';
+import { normalizeEmail } from './utils/normalize.js';
+import { isValidEmail } from './utils/validators.js';
+// Standard email regex for searching inside strings
+const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
+// Obfuscated email regex matching "name [at] domain [dot] com", "name(at)domain(dot)com", "name AT domain DOT com"
+const OBFUSCATED_REGEX = /([a-zA-Z0-9._%+-]+)\s*(?:\[at\]|\(at\)|\s+at\s+)\s*([a-zA-Z0-9.-]+)\s*(?:\[dot\]|\(dot\)|\s+dot\s+)\s*([a-zA-Z]{2,})/gi;
+// Base64 candidate regex for extracting potential base64 encoded strings
+const BASE64_CANDIDATE_REGEX = /\b[a-zA-Z0-9+/]{12,80}={0,2}\b/g;
+/**
+ * Decodes a Cloudflare email protection hex string.
+ */
+export function decodeCloudflareEmail(hex) {
+    try {
+        let email = '';
+        const r = parseInt(hex.substring(0, 2), 16);
+        for (let i = 2; i < hex.length; i += 2) {
+            const c = parseInt(hex.substring(i, i + 2), 16) ^ r;
+            email += String.fromCharCode(c);
+        }
+        return email;
+    }
+    catch (_e) {
+        return '';
+    }
+}
+/**
+ * Decodes HTML Unicode entities (e.g. &#x63; or &#99;).
+ * Cheerio generally decodes these, but this serves as a fallback.
+ */
+export function decodeUnicodeEntities(text) {
+    return text
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
+        .replace(/&#([0-9]+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)));
+}
+/**
+ * Attempts to extract Base64 encoded email addresses.
+ */
+export function extractBase64Emails(text) {
+    const emails = [];
+    let match;
+    // Reset regex index
+    BASE64_CANDIDATE_REGEX.lastIndex = 0;
+    while ((match = BASE64_CANDIDATE_REGEX.exec(text)) !== null) {
+        try {
+            const decoded = Buffer.from(match[0], 'base64').toString('utf-8');
+            const normalized = normalizeEmail(decoded);
+            if (isValidEmail(normalized)) {
+                emails.push(normalized);
+            }
+        }
+        catch (_e) {
+            // Not a valid base64 string or decode failed
+        }
+    }
+    return emails;
+}
+/**
+ * Maps a URL path to a discovery method / page type.
+ */
+function getDiscoveryMethod(url) {
+    try {
+        const path = new URL(url).pathname.toLowerCase();
+        if (path.includes('contact'))
+            return 'contact-page';
+        if (path.includes('about'))
+            return 'about-page';
+        if (path.includes('sitemap'))
+            return 'sitemap';
+        return 'general-page';
+    }
+    catch (_e) {
+        return 'general-page';
+    }
+}
+/**
+ * Classifies an email as 'role', 'personal', or 'automated'.
+ */
+export function classifyEmailType(email) {
+    const localPart = email.split('@')[0].toLowerCase();
+    const rolePrefixes = [
+        'contact',
+        'info',
+        'hello',
+        'support',
+        'sales',
+        'partnerships',
+        'partnership',
+        'business',
+        'team',
+        'founder',
+        'ceo',
+        'media',
+        'press',
+        'jobs',
+        'careers',
+        'admin',
+        'office',
+        'help',
+        'inquiries',
+        'inquiry',
+        'hi',
+        'welcome',
+        'hr',
+        'marketing',
+        'privacy',
+        'legal',
+        'billing',
+        'finance',
+    ];
+    const automatedPrefixes = [
+        'noreply',
+        'no-reply',
+        'donotreply',
+        'do-not-reply',
+        'mailer-daemon',
+        'postmaster',
+        'abuse',
+        'security',
+        'spam',
+        'bot',
+        'system',
+        'notification',
+        'notifications',
+        'alert',
+        'alerts',
+    ];
+    if (automatedPrefixes.some((prefix) => localPart === prefix || localPart.startsWith(prefix + '-'))) {
+        return 'automated';
+    }
+    if (rolePrefixes.some((prefix) => localPart === prefix || localPart.startsWith(prefix + '.'))) {
+        return 'role';
+    }
+    return 'personal';
+}
+/**
+ * Extracts all unique emails from a given HTML string and URL.
+ */
+export function extractEmails(html, url, pageTitle, crawlDurationMs) {
+    const discovered = new Map();
+    const $ = load(html);
+    const discoveryMethod = getDiscoveryMethod(url);
+    const addEmail = (rawEmail, sourceType, obfuscatedMethod) => {
+        const email = normalizeEmail(rawEmail);
+        if (!isValidEmail(email)) {
+            return;
+        }
+        const type = classifyEmailType(email);
+        const timestamp = new Date().toISOString();
+        const metadata = {
+            sourceUrl: url,
+            sourceType,
+            pageTitle,
+            discoveryTimestamp: timestamp,
+            crawlDurationMs,
+        };
+        // Calculate preliminary confidence score based on extraction details
+        // Final confidence is refined in scorer.ts
+        let initialConfidence = 60;
+        if (sourceType === 'footer' || sourceType === 'header') {
+            initialConfidence = 85;
+        }
+        else if (sourceType === 'mailto') {
+            initialConfidence = 90;
+        }
+        else if (sourceType === 'obfuscated') {
+            initialConfidence = 50;
+        }
+        else if (sourceType === 'script') {
+            initialConfidence = 40;
+        }
+        if (obfuscatedMethod) {
+            initialConfidence -= 10; // Obfuscated discovery has slightly lower confidence
+        }
+        const item = {
+            email,
+            emailSource: url,
+            emailType: type,
+            confidenceScore: Math.max(10, Math.min(100, initialConfidence)),
+            discoveryMethod: obfuscatedMethod === 'cloudflare'
+                ? 'obscure-js'
+                : sourceType === 'mailto'
+                    ? 'mailto-link'
+                    : discoveryMethod,
+            metadata,
+        };
+        const existing = discovered.get(email);
+        if (!existing || existing.confidenceScore < item.confidenceScore) {
+            discovered.set(email, item);
+        }
+    };
+    // 1. Cloudflare email protection decoding
+    // Search tags containing cfemail
+    $('[data-cfemail]').each((_, el) => {
+        const hex = $(el).attr('data-cfemail');
+        if (hex) {
+            const email = decodeCloudflareEmail(hex);
+            if (email)
+                addEmail(email, 'obfuscated', 'cloudflare');
+        }
+    });
+    // Search links containing cloudflare email-protection path
+    $('a[href*="/cdn-cgi/l/email-protection#"]').each((_, el) => {
+        const href = $(el).attr('href') || '';
+        const match = href.match(/\/cdn-cgi\/l\/email-protection#([a-fA-F0-9]+)/);
+        if (match && match[1]) {
+            const email = decodeCloudflareEmail(match[1]);
+            if (email)
+                addEmail(email, 'obfuscated', 'cloudflare');
+        }
+    });
+    // 2. Mailto links
+    $('a[href^="mailto:"]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (href) {
+            addEmail(href, 'mailto');
+        }
+    });
+    // 3. Header section extraction
+    const headerElem = $('header, [id*="header"], [class*="header"]');
+    if (headerElem.length > 0) {
+        const headerText = decodeUnicodeEntities(headerElem.text());
+        let match;
+        EMAIL_REGEX.lastIndex = 0;
+        while ((match = EMAIL_REGEX.exec(headerText)) !== null) {
+            addEmail(match[0], 'header');
+        }
+    }
+    // 4. Footer section extraction
+    const footerElem = $('footer, [id*="footer"], [class*="footer"]');
+    if (footerElem.length > 0) {
+        const footerText = decodeUnicodeEntities(footerElem.text());
+        let match;
+        EMAIL_REGEX.lastIndex = 0;
+        while ((match = EMAIL_REGEX.exec(footerText)) !== null) {
+            addEmail(match[0], 'footer');
+        }
+    }
+    // 5. Meta tags
+    $('meta').each((_, el) => {
+        const content = $(el).attr('content');
+        if (content) {
+            const cleanedContent = decodeUnicodeEntities(content);
+            let match;
+            EMAIL_REGEX.lastIndex = 0;
+            while ((match = EMAIL_REGEX.exec(cleanedContent)) !== null) {
+                addEmail(match[0], 'meta');
+            }
+        }
+    });
+    // 6. Scripts (JSON-LD, Inline JavaScript)
+    $('script').each((_, el) => {
+        const scriptContent = $(el).html();
+        if (scriptContent) {
+            const decodedScript = decodeUnicodeEntities(scriptContent);
+            // Look for standard emails in scripts
+            let match;
+            EMAIL_REGEX.lastIndex = 0;
+            while ((match = EMAIL_REGEX.exec(decodedScript)) !== null) {
+                addEmail(match[0], 'script');
+            }
+            // Look for obfuscated pattern emails in scripts
+            OBFUSCATED_REGEX.lastIndex = 0;
+            let obfMatch;
+            while ((obfMatch = OBFUSCATED_REGEX.exec(decodedScript)) !== null) {
+                addEmail(`${obfMatch[1]}@${obfMatch[2]}.${obfMatch[3]}`, 'script', 'text-obfuscation');
+            }
+            // Look for base64 encoded emails in scripts
+            const base64Emails = extractBase64Emails(decodedScript);
+            for (const b64Email of base64Emails) {
+                addEmail(b64Email, 'script', 'base64');
+            }
+        }
+    });
+    // 7. Visible Body Text & Obfuscated matches in body
+    const bodyText = decodeUnicodeEntities($('body').text());
+    // Standard matches in body text
+    let bodyMatch;
+    EMAIL_REGEX.lastIndex = 0;
+    while ((bodyMatch = EMAIL_REGEX.exec(bodyText)) !== null) {
+        addEmail(bodyMatch[0], 'text');
+    }
+    // Obfuscated matches in body text
+    let obfBodyMatch;
+    OBFUSCATED_REGEX.lastIndex = 0;
+    while ((obfBodyMatch = OBFUSCATED_REGEX.exec(bodyText)) !== null) {
+        addEmail(`${obfBodyMatch[1]}@${obfBodyMatch[2]}.${obfBodyMatch[3]}`, 'text', 'text-obfuscation');
+    }
+    return Array.from(discovered.values());
+}

package/dist/index.js

@@ -0,0 +1,247 @@
+#!/usr/bin/env node
+import { config } from './config.js';
+import { readCsvGenerator, appendCsvRow, loadCheckpoint, saveCheckpoint, clearCheckpoint, getCsvHeaders, } from './csv.js';
+import { Crawler } from './crawler.js';
+import { Logger } from './logger.js';
+import pLimit from 'p-limit';
+import fs from 'fs/promises';
+import path from 'path';
+import { normalizeDomain, findWebsiteInRow } from './utils/normalize.js';
+let highestContiguousIndex = -1;
+const completedIndices = new Set();
+let completedUrls = [];
+let crawlerInstance = null;
+let isShuttingDown = false;
+/**
+ * Handles graceful shutdown on SIGINT / SIGTERM signals.
+ */
+async function handleShutdown(signal) {
+    if (isShuttingDown) {
+        return;
+    }
+    isShuttingDown = true;
+    process.stdout.write(`\n[SHUTDOWN] Received ${signal}. Saving checkpoints and shutting down...\n`);
+    if (crawlerInstance) {
+        try {
+            await crawlerInstance.close();
+        }
+        catch (_e) {
+            /* ignore */
+        }
+    }
+    if (highestContiguousIndex >= 0) {
+        try {
+            await saveCheckpoint(config.checkpointFile, {
+                lastProcessedIndex: highestContiguousIndex,
+                completedUrls,
+                timestamp: new Date().toISOString(),
+            });
+            process.stdout.write(`[SHUTDOWN] Checkpoint persisted at index ${highestContiguousIndex}.\n`);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            process.stderr.write(`[SHUTDOWN] Failed to write checkpoint: ${msg}\n`);
+        }
+    }
+    process.exit(0);
+}
+// Register signal listeners
+process.on('SIGINT', () => {
+    handleShutdown('SIGINT').catch(() => process.exit(1));
+});
+process.on('SIGTERM', () => {
+    handleShutdown('SIGTERM').catch(() => process.exit(1));
+});
+/**
+ * Marks an index as completed and advances the highest contiguous completed index.
+ */
+function markIndexCompleted(index, url) {
+    completedIndices.add(index);
+    completedUrls.push(url);
+    while (completedIndices.has(highestContiguousIndex + 1)) {
+        highestContiguousIndex++;
+    }
+}
+/**
+ * Main application runner.
+ */
+async function main() {
+    const startRunTime = Date.now();
+    // Parse CLI flags and arguments
+    const args = process.argv.slice(2);
+    let inputPath = config.inputCsv;
+    let outputPath = config.outputCsv;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '-i' || args[i] === '--input') {
+            inputPath = path.resolve(args[i + 1]);
+            i++;
+        }
+        else if (args[i] === '-o' || args[i] === '--output') {
+            outputPath = path.resolve(args[i + 1]);
+            i++;
+        }
+        else if (args[i] === '-h' || args[i] === '--help') {
+            process.stdout.write(`
+mailpop - CLI Guide
+Usage: npx mailpop [options] [input.csv] [output.csv]
+
+Options:
+  -i, --input <path>     Path to the input CSV file
+  -o, --output <path>    Path to the output CSV file
+  -h, --help             Display this help message
+\n`);
+            process.exit(0);
+        }
+    }
+    // Fallback to positional arguments
+    const positionals = args.filter((a) => !a.startsWith('-'));
+    if (positionals.length >= 1) {
+        inputPath = path.resolve(positionals[0]);
+    }
+    if (positionals.length >= 2) {
+        outputPath = path.resolve(positionals[1]);
+    }
+    await Logger.info('app-initialize', undefined, undefined, 'Running', `Initializing mailpop (Input: ${path.basename(inputPath)}, Output: ${path.basename(outputPath)})...`);
+    // 1. Extract dynamic headers from the input CSV
+    let inputHeaders = [];
+    try {
+        inputHeaders = await getCsvHeaders(inputPath);
+    }
+    catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        await Logger.error('app-initialize-fail', undefined, undefined, `Failed to read input CSV headers: ${errorMsg}`);
+        process.exit(1);
+    }
+    // Construct combined output headers, preserving original columns and adding new ones
+    const outputHeaders = [...inputHeaders];
+    const newColumns = [
+        'email',
+        'email_source',
+        'email_type',
+        'confidence_score',
+        'discovery_method',
+    ];
+    for (const col of newColumns) {
+        if (!outputHeaders.includes(col)) {
+            outputHeaders.push(col);
+        }
+    }
+    // 2. Initialize crawler and browser
+    crawlerInstance = new Crawler();
+    await crawlerInstance.initialize(config.headless);
+    // 3. Determine if resume is available
+    let checkpointIndex = -1;
+    const checkpoint = await loadCheckpoint(config.checkpointFile);
+    if (checkpoint) {
+        checkpointIndex = checkpoint.lastProcessedIndex;
+        highestContiguousIndex = checkpointIndex;
+        completedUrls = checkpoint.completedUrls;
+        // Prime completed indices from checkpoint history to ensure tracking consistency
+        for (let i = 0; i <= checkpointIndex; i++) {
+            completedIndices.add(i);
+        }
+        await Logger.info('runner-resume', undefined, undefined, 'Resume', `Checkpoint found. Resuming crawl from row index ${checkpointIndex + 1}`);
+    }
+    else {
+        // New run. Setup fresh output file with CSV header line
+        await Logger.info('runner-fresh-start', undefined, undefined, 'Fresh', 'Starting fresh run. Writing CSV headers...');
+        try {
+            const outDir = path.dirname(outputPath);
+            await fs.mkdir(outDir, { recursive: true }).catch(() => { });
+            const headerLine = outputHeaders.join(',') + '\n';
+            await fs.writeFile(outputPath, headerLine, 'utf-8');
+        }
+        catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            await Logger.error('app-initialize-fail', undefined, undefined, `Failed to setup output file: ${errorMsg}`);
+            await crawlerInstance.close();
+            process.exit(1);
+        }
+    }
+    // 3. Process Input CSV with Concurrency Throttling
+    const limit = pLimit(config.concurrency);
+    const tasks = [];
+    let processedCount = 0;
+    try {
+        for await (const { row, index } of readCsvGenerator(inputPath)) {
+            if (isShuttingDown) {
+                break;
+            }
+            // Skip previously processed indices
+            if (index <= checkpointIndex) {
+                continue;
+            }
+            // Detect website URL column dynamically (Website, URL, Domain, Site, Web)
+            const websiteUrl = findWebsiteInRow(row);
+            if (!websiteUrl) {
+                await Logger.error('csv-row-skip', undefined, undefined, `Row ${index} is missing a website/url/domain column. Skipping.`);
+                markIndexCompleted(index, 'skipped-missing-url');
+                continue;
+            }
+            // Schedule the crawl target
+            const target = {
+                name: row.Name || row.name || 'Unknown Company',
+                website: websiteUrl,
+                domain: row.Domain || row.domain || normalizeDomain(websiteUrl),
+            };
+            const task = limit(async () => {
+                if (isShuttingDown) {
+                    return;
+                }
+                const result = await crawlerInstance.crawlWebsite(target, config);
+                if (isShuttingDown) {
+                    return;
+                }
+                // Map crawling result, retaining all original row keys
+                const outputRow = {
+                    ...row,
+                    email: result.selectedEmail ? result.selectedEmail.email : '',
+                    email_source: result.selectedEmail ? result.selectedEmail.emailSource : '',
+                    email_type: result.selectedEmail ? result.selectedEmail.emailType : '',
+                    confidence_score: result.selectedEmail
+                        ? String(result.selectedEmail.confidenceScore)
+                        : '',
+                    discovery_method: result.selectedEmail ? result.selectedEmail.discoveryMethod : '',
+                };
+                // Append output row incrementally matching the dynamic headers list
+                await appendCsvRow(outputPath, outputRow, outputHeaders, false);
+                // Mark index completed and log status
+                markIndexCompleted(index, target.website);
+                processedCount++;
+                // Save progress checkpoints every 10 companies processed
+                if (processedCount % 10 === 0) {
+                    await saveCheckpoint(config.checkpointFile, {
+                        lastProcessedIndex: highestContiguousIndex,
+                        completedUrls,
+                        timestamp: new Date().toISOString(),
+                    });
+                    await Logger.info('runner-checkpoint', undefined, undefined, 'Progress', `Checkpoint persisted. Processed: ${processedCount} in this run (Last index: ${highestContiguousIndex})`);
+                }
+            });
+            tasks.push(task);
+        }
+        // Wait for all scheduled crawler tasks to finish
+        await Promise.all(tasks);
+        // 4. Successful Finish
+        if (!isShuttingDown) {
+            await clearCheckpoint(config.checkpointFile);
+            const totalDuration = Date.now() - startRunTime;
+            await Logger.info('runner-complete', undefined, totalDuration, 'Success', `Crawl completed successfully. Processed ${processedCount} targets in ${Math.round(totalDuration / 1000)}s.`);
+        }
+    }
+    catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        await Logger.error('runner-fatal', undefined, Date.now() - startRunTime, errorMsg);
+    }
+    finally {
+        if (crawlerInstance) {
+            await crawlerInstance.close();
+        }
+    }
+}
+// Start application
+main().catch((err) => {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`Fatal error during application execution: ${errorMsg}\n`);
+    process.exit(1);
+});