mailpop 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.js +12 -2
- package/dist/logger.js +76 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -91,9 +91,9 @@ RETRY_MAX_DELAY_MS=10000
|
|
|
91
91
|
|
|
92
92
|
## Installation & Setup
|
|
93
93
|
|
|
94
|
-
1. **
|
|
94
|
+
1. **Navigate to the project directory**:
|
|
95
95
|
```bash
|
|
96
|
-
cd
|
|
96
|
+
cd mailpop
|
|
97
97
|
```
|
|
98
98
|
|
|
99
99
|
2. **Install Node dependencies**:
|
package/dist/index.js
CHANGED
|
@@ -5,8 +5,13 @@ import { Crawler } from './crawler.js';
|
|
|
5
5
|
import { Logger } from './logger.js';
|
|
6
6
|
import pLimit from 'p-limit';
|
|
7
7
|
import fs from 'fs/promises';
|
|
8
|
+
import { readFileSync } from 'fs';
|
|
8
9
|
import path from 'path';
|
|
10
|
+
import { fileURLToPath } from 'url';
|
|
9
11
|
import { normalizeDomain, findWebsiteInRow } from './utils/normalize.js';
|
|
12
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const pkg = JSON.parse(readFileSync(path.resolve(__dirname, '../package.json'), 'utf-8'));
|
|
14
|
+
const version = pkg.version || 'unknown';
|
|
10
15
|
let highestContiguousIndex = -1;
|
|
11
16
|
const completedIndices = new Set();
|
|
12
17
|
let completedUrls = [];
|
|
@@ -90,6 +95,10 @@ async function main() {
|
|
|
90
95
|
config.excludePrefixes = Array.from(new Set([...config.excludePrefixes, ...list]));
|
|
91
96
|
i++;
|
|
92
97
|
}
|
|
98
|
+
else if (args[i] === '-v' || args[i] === '--version') {
|
|
99
|
+
process.stdout.write(`mailpop v${version}\n`);
|
|
100
|
+
process.exit(0);
|
|
101
|
+
}
|
|
93
102
|
else if (args[i] === '-h' || args[i] === '--help') {
|
|
94
103
|
process.stdout.write(`
|
|
95
104
|
mailpop - CLI Guide
|
|
@@ -99,6 +108,7 @@ Options:
|
|
|
99
108
|
-i, --input <path> Path to the input CSV file
|
|
100
109
|
-o, --output <path> Path to the output CSV file
|
|
101
110
|
-e, --exclude <list> Comma-separated list of email local-parts to exclude
|
|
111
|
+
-v, --version Display the version number
|
|
102
112
|
-h, --help Display this help message
|
|
103
113
|
\n`);
|
|
104
114
|
process.exit(0);
|
|
@@ -114,7 +124,7 @@ Options:
|
|
|
114
124
|
if (positionals.length >= 2) {
|
|
115
125
|
outputPath = path.resolve(positionals[1]);
|
|
116
126
|
}
|
|
117
|
-
await Logger.
|
|
127
|
+
await Logger.intro(version, inputPath, outputPath, config.concurrency, config.excludePrefixes, config.checkpointFile);
|
|
118
128
|
// 1. Extract dynamic headers from the input CSV
|
|
119
129
|
let inputHeaders = [];
|
|
120
130
|
try {
|
|
@@ -239,7 +249,7 @@ Options:
|
|
|
239
249
|
if (!isShuttingDown) {
|
|
240
250
|
await clearCheckpoint(config.checkpointFile);
|
|
241
251
|
const totalDuration = Date.now() - startRunTime;
|
|
242
|
-
await Logger.
|
|
252
|
+
await Logger.outro(processedCount, totalDuration, outputPath);
|
|
243
253
|
}
|
|
244
254
|
}
|
|
245
255
|
catch (err) {
|
package/dist/logger.js
CHANGED
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
import fs from 'fs/promises';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
const LOGS_DIR = path.resolve('logs');
|
|
4
|
+
// ANSI escape codes for styling
|
|
5
|
+
const RESET = '\x1b[0m';
|
|
6
|
+
const BOLD = '\x1b[1m';
|
|
7
|
+
const FG_CYAN = '\x1b[36m';
|
|
8
|
+
const FG_GREEN = '\x1b[32m';
|
|
9
|
+
const FG_RED = '\x1b[31m';
|
|
10
|
+
const FG_YELLOW = '\x1b[33m';
|
|
11
|
+
const FG_MAGENTA = '\x1b[35m';
|
|
12
|
+
const FG_GRAY = '\x1b[90m';
|
|
13
|
+
const FG_WHITE = '\x1b[37m';
|
|
14
|
+
const TEXT_BLACK = '\x1b[30m';
|
|
15
|
+
const BG_CYAN = '\x1b[46m';
|
|
16
|
+
const BG_RED = '\x1b[41m';
|
|
17
|
+
const BG_MAGENTA = '\x1b[45m';
|
|
4
18
|
/**
|
|
5
19
|
* Ensures that the logs directory exists on disk.
|
|
6
20
|
*/
|
|
@@ -28,6 +42,52 @@ async function writeLog(filename, data) {
|
|
|
28
42
|
}
|
|
29
43
|
}
|
|
30
44
|
export class Logger {
|
|
45
|
+
/**
|
|
46
|
+
* Logs a beautiful intro banner with all configuration settings.
|
|
47
|
+
*/
|
|
48
|
+
static async intro(version, inputPath, outputPath, concurrency, excludePrefixes, checkpointFile) {
|
|
49
|
+
const border = `${BOLD}${FG_CYAN}┌────────────────────────────────────────────────────────┐${RESET}`;
|
|
50
|
+
const title = `${BOLD}${FG_CYAN}│${RESET} ${BOLD}${FG_MAGENTA}⚡ MAILPOP${RESET} ${FG_GRAY}v${version}${RESET} ${BOLD}${FG_CYAN}│${RESET}`;
|
|
51
|
+
const desc = `${BOLD}${FG_CYAN}│${RESET} ${FG_WHITE}Production-Ready Contact Email Scraper${RESET} ${BOLD}${FG_CYAN}│${RESET}`;
|
|
52
|
+
const bottom = `${BOLD}${FG_CYAN}└────────────────────────────────────────────────────────┘${RESET}`;
|
|
53
|
+
const infoLines = [
|
|
54
|
+
` ${BOLD}📂 Input CSV:${RESET} ${FG_GREEN}${inputPath}${RESET}`,
|
|
55
|
+
` ${BOLD}📂 Output CSV:${RESET} ${FG_GREEN}${outputPath}${RESET}`,
|
|
56
|
+
` ${BOLD}⚙️ Concurrency:${RESET} ${FG_CYAN}${concurrency} targets${RESET}`,
|
|
57
|
+
` ${BOLD}🚫 Exclusions:${RESET} ${FG_YELLOW}${excludePrefixes.join(', ') || 'None'}${RESET}`,
|
|
58
|
+
` ${BOLD}🔄 Checkpoint:${RESET} ${FG_GRAY}${checkpointFile}${RESET}`,
|
|
59
|
+
];
|
|
60
|
+
const fullConsoleMsg = `\n${border}\n${title}\n${desc}\n${bottom}\n\n${infoLines.join('\n')}\n\n`;
|
|
61
|
+
process.stdout.write(fullConsoleMsg);
|
|
62
|
+
await writeLog('app.log', {
|
|
63
|
+
timestamp: new Date().toISOString(),
|
|
64
|
+
level: 'INFO',
|
|
65
|
+
action: 'app-initialize',
|
|
66
|
+
message: `Mailpop v${version} initialized. Input: ${inputPath}, Output: ${outputPath}, Concurrency: ${concurrency}, Exclusions: ${excludePrefixes.join(',')}`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Logs a beautiful outro banner with summary metrics.
|
|
71
|
+
*/
|
|
72
|
+
static async outro(processedCount, totalDurationMs, outputPath) {
|
|
73
|
+
const border = `${BOLD}${FG_GREEN}┌────────────────────────────────────────────────────────┐${RESET}`;
|
|
74
|
+
const title = `${BOLD}${FG_GREEN}│${RESET} ${BOLD}${FG_GREEN}🎉 CRAWL COMPLETED SUCCESSFULLY!${RESET} ${BOLD}${FG_GREEN}│${RESET}`;
|
|
75
|
+
const bottom = `${BOLD}${FG_GREEN}└────────────────────────────────────────────────────────┘${RESET}`;
|
|
76
|
+
const infoLines = [
|
|
77
|
+
` ${BOLD}✨ Total Targets:${RESET} ${FG_CYAN}${processedCount}${RESET}`,
|
|
78
|
+
` ${BOLD}⏱️ Time Elapsed:${RESET} ${FG_CYAN}${(totalDurationMs / 1000).toFixed(1)}s${RESET}`,
|
|
79
|
+
` ${BOLD}💾 Output File:${RESET} ${FG_GREEN}${outputPath}${RESET}`,
|
|
80
|
+
];
|
|
81
|
+
const fullConsoleMsg = `\n${border}\n${title}\n${bottom}\n\n${infoLines.join('\n')}\n\n`;
|
|
82
|
+
process.stdout.write(fullConsoleMsg);
|
|
83
|
+
await writeLog('app.log', {
|
|
84
|
+
timestamp: new Date().toISOString(),
|
|
85
|
+
level: 'INFO',
|
|
86
|
+
action: 'runner-complete',
|
|
87
|
+
duration: totalDurationMs,
|
|
88
|
+
message: `Crawl completed successfully. Processed ${processedCount} targets in ${(totalDurationMs / 1000).toFixed(1)}s. Output: ${outputPath}`,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
31
91
|
/**
|
|
32
92
|
* Logs general information events.
|
|
33
93
|
*/
|
|
@@ -41,7 +101,12 @@ export class Logger {
|
|
|
41
101
|
result,
|
|
42
102
|
message,
|
|
43
103
|
};
|
|
44
|
-
const
|
|
104
|
+
const levelTag = `${BOLD}${BG_CYAN}${TEXT_BLACK} INFO ${RESET}`;
|
|
105
|
+
const domainStr = domain ? ` ${FG_GRAY}[${FG_CYAN}${domain}${FG_GRAY}]${RESET}` : '';
|
|
106
|
+
const actionStr = ` ${BOLD}${action}${RESET}`;
|
|
107
|
+
const resultStr = result ? ` -> ${FG_GREEN}${result}${RESET}` : '';
|
|
108
|
+
const msgStr = message ? ` | ${FG_GRAY}${message}${RESET}` : '';
|
|
109
|
+
const consoleMsg = `${levelTag}${domainStr}${actionStr}${resultStr}${msgStr}`;
|
|
45
110
|
process.stdout.write(consoleMsg + '\n');
|
|
46
111
|
await writeLog('app.log', entry);
|
|
47
112
|
}
|
|
@@ -58,7 +123,11 @@ export class Logger {
|
|
|
58
123
|
error: errorMsg,
|
|
59
124
|
stack,
|
|
60
125
|
};
|
|
61
|
-
const
|
|
126
|
+
const levelTag = `${BOLD}${BG_RED}${FG_WHITE} ERROR ${RESET}`;
|
|
127
|
+
const domainStr = domain ? ` ${FG_GRAY}[${FG_RED}${domain}${FG_GRAY}]${RESET}` : '';
|
|
128
|
+
const actionStr = ` ${BOLD}${action}${RESET}`;
|
|
129
|
+
const errorStr = errorMsg ? `: ${FG_RED}${errorMsg}${RESET}` : '';
|
|
130
|
+
const consoleMsg = `${levelTag}${domainStr}${actionStr}${errorStr}`;
|
|
62
131
|
process.stderr.write(consoleMsg + '\n');
|
|
63
132
|
await writeLog('app.log', entry);
|
|
64
133
|
await writeLog('errors.log', entry);
|
|
@@ -75,7 +144,11 @@ export class Logger {
|
|
|
75
144
|
confidenceScore: confidence,
|
|
76
145
|
discoveryMethod: method,
|
|
77
146
|
};
|
|
78
|
-
const
|
|
147
|
+
const levelTag = `${BOLD}${BG_MAGENTA}${TEXT_BLACK} EMAIL ${RESET}`;
|
|
148
|
+
const domainStr = ` ${FG_GRAY}[${FG_CYAN}${domain}${FG_GRAY}]${RESET}`;
|
|
149
|
+
const emailStr = ` Found ${BOLD}${FG_GREEN}${email}${RESET}`;
|
|
150
|
+
const detailsStr = ` (${FG_YELLOW}${method}${RESET}, confidence: ${BOLD}${confidence}${RESET}) at ${FG_GRAY}${source}${RESET}`;
|
|
151
|
+
const consoleMsg = `${levelTag}${domainStr}${emailStr}${detailsStr}`;
|
|
79
152
|
process.stdout.write(consoleMsg + '\n');
|
|
80
153
|
await writeLog('discovered-emails.log', entry);
|
|
81
154
|
}
|