mailpop 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config.js +9 -1
- package/dist/index.js +2 -2
- package/dist/logger.js +49 -2
- package/dist/utils/validators.js +7 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -91,9 +91,9 @@ RETRY_MAX_DELAY_MS=10000
|
|
|
91
91
|
|
|
92
92
|
## Installation & Setup
|
|
93
93
|
|
|
94
|
-
1. **
|
|
94
|
+
1. **Navigate to the project directory**:
|
|
95
95
|
```bash
|
|
96
|
-
cd
|
|
96
|
+
cd mailpop
|
|
97
97
|
```
|
|
98
98
|
|
|
99
99
|
2. **Install Node dependencies**:
|
package/dist/config.js
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import dotenv from 'dotenv';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
|
|
3
|
+
import { existsSync } from 'fs';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
5
|
+
// 1. Load environment variables from CWD .env (if present)
|
|
4
6
|
dotenv.config();
|
|
7
|
+
// 2. Load from package directory .env as a fallback for defaults
|
|
8
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
const packageEnv = path.resolve(__dirname, '../.env');
|
|
10
|
+
if (existsSync(packageEnv)) {
|
|
11
|
+
dotenv.config({ path: packageEnv });
|
|
12
|
+
}
|
|
5
13
|
const getEnvNumber = (key, defaultValue) => {
|
|
6
14
|
const val = process.env[key];
|
|
7
15
|
if (val === undefined)
|
package/dist/index.js
CHANGED
|
@@ -124,7 +124,7 @@ Options:
|
|
|
124
124
|
if (positionals.length >= 2) {
|
|
125
125
|
outputPath = path.resolve(positionals[1]);
|
|
126
126
|
}
|
|
127
|
-
await Logger.
|
|
127
|
+
await Logger.intro(version, inputPath, outputPath, config.concurrency, config.excludePrefixes, config.checkpointFile);
|
|
128
128
|
// 1. Extract dynamic headers from the input CSV
|
|
129
129
|
let inputHeaders = [];
|
|
130
130
|
try {
|
|
@@ -249,7 +249,7 @@ Options:
|
|
|
249
249
|
if (!isShuttingDown) {
|
|
250
250
|
await clearCheckpoint(config.checkpointFile);
|
|
251
251
|
const totalDuration = Date.now() - startRunTime;
|
|
252
|
-
await Logger.
|
|
252
|
+
await Logger.outro(processedCount, totalDuration, outputPath);
|
|
253
253
|
}
|
|
254
254
|
}
|
|
255
255
|
catch (err) {
|
package/dist/logger.js
CHANGED
|
@@ -8,12 +8,13 @@ const FG_CYAN = '\x1b[36m';
|
|
|
8
8
|
const FG_GREEN = '\x1b[32m';
|
|
9
9
|
const FG_RED = '\x1b[31m';
|
|
10
10
|
const FG_YELLOW = '\x1b[33m';
|
|
11
|
+
const FG_MAGENTA = '\x1b[35m';
|
|
11
12
|
const FG_GRAY = '\x1b[90m';
|
|
12
13
|
const FG_WHITE = '\x1b[37m';
|
|
13
14
|
const TEXT_BLACK = '\x1b[30m';
|
|
14
15
|
const BG_CYAN = '\x1b[46m';
|
|
15
|
-
const BG_GREEN = '\x1b[42m';
|
|
16
16
|
const BG_RED = '\x1b[41m';
|
|
17
|
+
const BG_MAGENTA = '\x1b[45m';
|
|
17
18
|
/**
|
|
18
19
|
* Ensures that the logs directory exists on disk.
|
|
19
20
|
*/
|
|
@@ -41,6 +42,52 @@ async function writeLog(filename, data) {
|
|
|
41
42
|
}
|
|
42
43
|
}
|
|
43
44
|
export class Logger {
|
|
45
|
+
/**
|
|
46
|
+
* Logs a beautiful intro banner with all configuration settings.
|
|
47
|
+
*/
|
|
48
|
+
static async intro(version, inputPath, outputPath, concurrency, excludePrefixes, checkpointFile) {
|
|
49
|
+
const border = `${BOLD}${FG_CYAN}┌────────────────────────────────────────────────────────┐${RESET}`;
|
|
50
|
+
const title = `${BOLD}${FG_CYAN}│${RESET} ${BOLD}${FG_MAGENTA}⚡ MAILPOP${RESET} ${FG_GRAY}v${version}${RESET} ${BOLD}${FG_CYAN}│${RESET}`;
|
|
51
|
+
const desc = `${BOLD}${FG_CYAN}│${RESET} ${FG_WHITE}Production-Ready Contact Email Scraper${RESET} ${BOLD}${FG_CYAN}│${RESET}`;
|
|
52
|
+
const bottom = `${BOLD}${FG_CYAN}└────────────────────────────────────────────────────────┘${RESET}`;
|
|
53
|
+
const infoLines = [
|
|
54
|
+
` ${BOLD}📂 Input CSV:${RESET} ${FG_GREEN}${inputPath}${RESET}`,
|
|
55
|
+
` ${BOLD}📂 Output CSV:${RESET} ${FG_GREEN}${outputPath}${RESET}`,
|
|
56
|
+
` ${BOLD}⚙️ Concurrency:${RESET} ${FG_CYAN}${concurrency} targets${RESET}`,
|
|
57
|
+
` ${BOLD}🚫 Exclusions:${RESET} ${FG_YELLOW}${excludePrefixes.join(', ') || 'None'}${RESET}`,
|
|
58
|
+
` ${BOLD}🔄 Checkpoint:${RESET} ${FG_GRAY}${checkpointFile}${RESET}`,
|
|
59
|
+
];
|
|
60
|
+
const fullConsoleMsg = `\n${border}\n${title}\n${desc}\n${bottom}\n\n${infoLines.join('\n')}\n\n`;
|
|
61
|
+
process.stdout.write(fullConsoleMsg);
|
|
62
|
+
await writeLog('app.log', {
|
|
63
|
+
timestamp: new Date().toISOString(),
|
|
64
|
+
level: 'INFO',
|
|
65
|
+
action: 'app-initialize',
|
|
66
|
+
message: `Mailpop v${version} initialized. Input: ${inputPath}, Output: ${outputPath}, Concurrency: ${concurrency}, Exclusions: ${excludePrefixes.join(',')}`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Logs a beautiful outro banner with summary metrics.
|
|
71
|
+
*/
|
|
72
|
+
static async outro(processedCount, totalDurationMs, outputPath) {
|
|
73
|
+
const border = `${BOLD}${FG_GREEN}┌────────────────────────────────────────────────────────┐${RESET}`;
|
|
74
|
+
const title = `${BOLD}${FG_GREEN}│${RESET} ${BOLD}${FG_GREEN}🎉 CRAWL COMPLETED SUCCESSFULLY!${RESET} ${BOLD}${FG_GREEN}│${RESET}`;
|
|
75
|
+
const bottom = `${BOLD}${FG_GREEN}└────────────────────────────────────────────────────────┘${RESET}`;
|
|
76
|
+
const infoLines = [
|
|
77
|
+
` ${BOLD}✨ Total Targets:${RESET} ${FG_CYAN}${processedCount}${RESET}`,
|
|
78
|
+
` ${BOLD}⏱️ Time Elapsed:${RESET} ${FG_CYAN}${(totalDurationMs / 1000).toFixed(1)}s${RESET}`,
|
|
79
|
+
` ${BOLD}💾 Output File:${RESET} ${FG_GREEN}${outputPath}${RESET}`,
|
|
80
|
+
];
|
|
81
|
+
const fullConsoleMsg = `\n${border}\n${title}\n${bottom}\n\n${infoLines.join('\n')}\n\n`;
|
|
82
|
+
process.stdout.write(fullConsoleMsg);
|
|
83
|
+
await writeLog('app.log', {
|
|
84
|
+
timestamp: new Date().toISOString(),
|
|
85
|
+
level: 'INFO',
|
|
86
|
+
action: 'runner-complete',
|
|
87
|
+
duration: totalDurationMs,
|
|
88
|
+
message: `Crawl completed successfully. Processed ${processedCount} targets in ${(totalDurationMs / 1000).toFixed(1)}s. Output: ${outputPath}`,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
44
91
|
/**
|
|
45
92
|
* Logs general information events.
|
|
46
93
|
*/
|
|
@@ -97,7 +144,7 @@ export class Logger {
|
|
|
97
144
|
confidenceScore: confidence,
|
|
98
145
|
discoveryMethod: method,
|
|
99
146
|
};
|
|
100
|
-
const levelTag = `${BOLD}${
|
|
147
|
+
const levelTag = `${BOLD}${BG_MAGENTA}${TEXT_BLACK} EMAIL ${RESET}`;
|
|
101
148
|
const domainStr = ` ${FG_GRAY}[${FG_CYAN}${domain}${FG_GRAY}]${RESET}`;
|
|
102
149
|
const emailStr = ` Found ${BOLD}${FG_GREEN}${email}${RESET}`;
|
|
103
150
|
const detailsStr = ` (${FG_YELLOW}${method}${RESET}, confidence: ${BOLD}${confidence}${RESET}) at ${FG_GRAY}${source}${RESET}`;
|
package/dist/utils/validators.js
CHANGED
|
@@ -46,8 +46,13 @@ export function isValidEmail(email) {
|
|
|
46
46
|
if (REJECTED_PREFIXES.includes(localPart)) {
|
|
47
47
|
return false;
|
|
48
48
|
}
|
|
49
|
-
// Reject user-configured excluded prefixes
|
|
50
|
-
|
|
49
|
+
// Reject user-configured excluded prefixes (matches exact, or delimited by -, ., _, +)
|
|
50
|
+
const isExcluded = config.excludePrefixes.some((prefix) => {
|
|
51
|
+
const escaped = prefix.replace(/[-\\^$*+?.()|[\]{}]/g, '\\$&');
|
|
52
|
+
const regex = new RegExp(`(^|[-._+])` + escaped + `($|[-._+])`, 'i');
|
|
53
|
+
return regex.test(localPart);
|
|
54
|
+
});
|
|
55
|
+
if (isExcluded) {
|
|
51
56
|
return false;
|
|
52
57
|
}
|
|
53
58
|
// Reject blacklisted domains
|