@fanboynz/network-scanner 2.0.57 → 2.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/npm-publish.yml +3 -3
- package/CHANGELOG.md +940 -0
- package/CLAUDE.md +65 -0
- package/README.md +31 -0
- package/lib/adblock.js +4 -3
- package/lib/browserexit.js +61 -96
- package/lib/browserhealth.js +16 -4
- package/lib/cdp.js +17 -169
- package/lib/compare.js +0 -4
- package/lib/compress.js +6 -15
- package/lib/dry-run.js +1 -1
- package/lib/fingerprint.js +47 -37
- package/lib/flowproxy.js +8 -8
- package/lib/ghost-cursor.js +258 -0
- package/lib/grep.js +1 -1
- package/lib/interaction.js +23 -45
- package/lib/openvpn_vpn.js +16 -21
- package/lib/output.js +12 -6
- package/lib/validate_rules.js +12 -27
- package/nwss.js +147 -52
- package/package.json +5 -1
- package/.clauderc +0 -30
package/lib/openvpn_vpn.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// NOTE: Like wireguard_vpn.js, OpenVPN modifies system-level routing.
|
|
7
7
|
// When running concurrent scans, all traffic routes through the active
|
|
8
|
-
// VPN tunnel � not just the site that requested it. For isolated
|
|
8
|
+
// VPN tunnel � not just the site that requested it. For isolated
|
|
9
9
|
// per-site VPN with concurrency, a SOCKS proxy approach is needed.
|
|
10
10
|
|
|
11
11
|
const { execSync, spawn } = require('child_process');
|
|
@@ -77,13 +77,17 @@ function hasRootPrivileges() {
|
|
|
77
77
|
* Detect if running inside WSL
|
|
78
78
|
* @returns {boolean}
|
|
79
79
|
*/
|
|
80
|
-
|
|
80
|
+
const _isWSL = (() => {
|
|
81
81
|
try {
|
|
82
82
|
const release = fs.readFileSync('/proc/version', 'utf8').toLowerCase();
|
|
83
83
|
return release.includes('microsoft') || release.includes('wsl');
|
|
84
84
|
} catch {
|
|
85
85
|
return false;
|
|
86
86
|
}
|
|
87
|
+
})();
|
|
88
|
+
|
|
89
|
+
function isWSL() {
|
|
90
|
+
return _isWSL;
|
|
87
91
|
}
|
|
88
92
|
|
|
89
93
|
/**
|
|
@@ -122,9 +126,7 @@ function checkTunDevice() {
|
|
|
122
126
|
* Ensure temp directory exists with secure permissions
|
|
123
127
|
*/
|
|
124
128
|
function ensureTempDir() {
|
|
125
|
-
|
|
126
|
-
fs.mkdirSync(TEMP_DIR, { recursive: true, mode: 0o755 });
|
|
127
|
-
}
|
|
129
|
+
fs.mkdirSync(TEMP_DIR, { recursive: true, mode: 0o755 });
|
|
128
130
|
}
|
|
129
131
|
|
|
130
132
|
/**
|
|
@@ -369,7 +371,7 @@ async function startConnection(configPath, vpnConfig, forceDebug = false) {
|
|
|
369
371
|
const logPath = path.join(TEMP_DIR, `${connectionName}.log`);
|
|
370
372
|
|
|
371
373
|
// Clean stale log
|
|
372
|
-
try {
|
|
374
|
+
try { fs.unlinkSync(logPath); } catch {}
|
|
373
375
|
|
|
374
376
|
// Pre-create log file writable by all so sudo openvpn can write and user can read
|
|
375
377
|
try { fs.writeFileSync(logPath, '', { mode: 0o666 }); } catch {}
|
|
@@ -380,9 +382,8 @@ async function startConnection(configPath, vpnConfig, forceDebug = false) {
|
|
|
380
382
|
console.log(formatLogMessage('debug', `[openvpn] Starting: openvpn ${args.join(' ')}`));
|
|
381
383
|
}
|
|
382
384
|
|
|
383
|
-
// Spawn OpenVPN
|
|
385
|
+
// Spawn OpenVPN — it daemonizes itself via --daemon, but we spawn
|
|
384
386
|
// without --daemon so we can track the process directly
|
|
385
|
-
const filteredArgs = args.filter(a => a !== '--daemon' && a !== connectionName);
|
|
386
387
|
// Remove --daemon and its argument from args, run in foreground
|
|
387
388
|
const fgArgs = [];
|
|
388
389
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -508,9 +509,7 @@ function cleanupConnectionFiles(connectionName) {
|
|
|
508
509
|
];
|
|
509
510
|
|
|
510
511
|
for (const file of filesToClean) {
|
|
511
|
-
try {
|
|
512
|
-
if (fs.existsSync(file)) fs.unlinkSync(file);
|
|
513
|
-
} catch {}
|
|
512
|
+
try { fs.unlinkSync(file); } catch {}
|
|
514
513
|
}
|
|
515
514
|
}
|
|
516
515
|
|
|
@@ -582,11 +581,9 @@ function getConnectionStatus(connectionName) {
|
|
|
582
581
|
|
|
583
582
|
// Read last few lines of log
|
|
584
583
|
try {
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
status.lastLog = lines.slice(-3).join('\n');
|
|
589
|
-
}
|
|
584
|
+
const log = fs.readFileSync(info.logPath, 'utf8');
|
|
585
|
+
const lines = log.trim().split('\n');
|
|
586
|
+
status.lastLog = lines.slice(-3).join('\n');
|
|
590
587
|
} catch {}
|
|
591
588
|
|
|
592
589
|
return status;
|
|
@@ -694,12 +691,12 @@ function validateOvpnConfig(ovpnConfig) {
|
|
|
694
691
|
|
|
695
692
|
// Privilege check
|
|
696
693
|
if (!hasRootPrivileges()) {
|
|
697
|
-
result.warnings.push('OpenVPN requires root privileges � run with sudo');
|
|
694
|
+
result.warnings.push('OpenVPN requires root privileges � run with sudo');
|
|
698
695
|
}
|
|
699
696
|
|
|
700
697
|
// WSL checks
|
|
701
698
|
if (isWSL()) {
|
|
702
|
-
result.warnings.push('Running on WSL2 � ensure TUN module is loaded: sudo modprobe tun');
|
|
699
|
+
result.warnings.push('Running on WSL2 � ensure TUN module is loaded: sudo modprobe tun');
|
|
703
700
|
const tunCheck = checkTunDevice();
|
|
704
701
|
if (!tunCheck.available) {
|
|
705
702
|
result.warnings.push(tunCheck.error);
|
|
@@ -862,9 +859,7 @@ function disconnectAll(forceDebug = false) {
|
|
|
862
859
|
}
|
|
863
860
|
|
|
864
861
|
// Clean up entire temp directory
|
|
865
|
-
|
|
866
|
-
try { fs.rmSync(TEMP_DIR, { recursive: true, force: true }); } catch {}
|
|
867
|
-
}
|
|
862
|
+
try { fs.rmSync(TEMP_DIR, { recursive: true, force: true }); } catch {}
|
|
868
863
|
|
|
869
864
|
if (forceDebug && results.tornDown > 0) {
|
|
870
865
|
console.log(formatLogMessage('debug',
|
package/lib/output.js
CHANGED
|
@@ -5,6 +5,9 @@ const { getTotalDomainsSkipped } = require('./domain-cache');
|
|
|
5
5
|
const { loadComparisonRules, filterUniqueRules } = require('./compare');
|
|
6
6
|
const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
|
|
7
7
|
|
|
8
|
+
// Cache for compiled wildcard regex patterns in matchesIgnoreDomain
|
|
9
|
+
const wildcardRegexCache = new Map();
|
|
10
|
+
|
|
8
11
|
/**
|
|
9
12
|
* Check if domain matches any ignore patterns (supports wildcards)
|
|
10
13
|
* @param {string} domain - Domain to check
|
|
@@ -37,11 +40,14 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
|
|
|
37
40
|
const baseDomain = pattern.slice(0, -2); // Remove ".*"
|
|
38
41
|
return domain.startsWith(baseDomain + '.');
|
|
39
42
|
} else {
|
|
40
|
-
// Complex wildcard pattern
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
// Complex wildcard pattern (cached)
|
|
44
|
+
if (!wildcardRegexCache.has(pattern)) {
|
|
45
|
+
const regexPattern = pattern
|
|
46
|
+
.replace(/\./g, '\\.') // Escape dots
|
|
47
|
+
.replace(/\*/g, '.*'); // Convert * to .*
|
|
48
|
+
wildcardRegexCache.set(pattern, new RegExp(`^${regexPattern}$`));
|
|
49
|
+
}
|
|
50
|
+
return wildcardRegexCache.get(pattern).test(domain);
|
|
45
51
|
}
|
|
46
52
|
} else {
|
|
47
53
|
// Exact pattern matching
|
|
@@ -435,7 +441,7 @@ function writeOutput(lines, outputFile = null, silentMode = false) {
|
|
|
435
441
|
if (outputFile) {
|
|
436
442
|
// Ensure output directory exists
|
|
437
443
|
const outputDir = path.dirname(outputFile);
|
|
438
|
-
if (outputDir !== '.'
|
|
444
|
+
if (outputDir !== '.') {
|
|
439
445
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
440
446
|
}
|
|
441
447
|
|
package/lib/validate_rules.js
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
const { formatLogMessage } = require('./colorize');
|
|
2
2
|
|
|
3
|
+
// Pre-compiled regex constants for validation
|
|
4
|
+
const REGEX_LABEL = /^[a-zA-Z0-9-]+$/;
|
|
5
|
+
const REGEX_TLD = /^[a-zA-Z][a-zA-Z0-9]*$/;
|
|
6
|
+
const REGEX_IPv4 = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
|
|
7
|
+
const REGEX_IPv6 = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::$|^::1$|^(?:[0-9a-fA-F]{1,4}:)*::(?:[0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$/;
|
|
8
|
+
|
|
3
9
|
/**
|
|
4
10
|
* Enhanced domain validation function
|
|
5
11
|
* @param {string} domain - The domain to validate
|
|
@@ -88,8 +94,7 @@ function isValidDomainLabel(label) {
|
|
|
88
94
|
}
|
|
89
95
|
|
|
90
96
|
// Label can only contain alphanumeric characters and hyphens
|
|
91
|
-
|
|
92
|
-
if (!labelRegex.test(label)) {
|
|
97
|
+
if (!REGEX_LABEL.test(label)) {
|
|
93
98
|
return false;
|
|
94
99
|
}
|
|
95
100
|
|
|
@@ -115,8 +120,7 @@ function isValidTLD(tld) {
|
|
|
115
120
|
// but still validate structure
|
|
116
121
|
|
|
117
122
|
// TLD can contain letters and numbers, but must start with letter
|
|
118
|
-
|
|
119
|
-
if (!tldRegex.test(tld)) {
|
|
123
|
+
if (!REGEX_TLD.test(tld)) {
|
|
120
124
|
return false;
|
|
121
125
|
}
|
|
122
126
|
|
|
@@ -138,8 +142,7 @@ function isIPAddress(str) {
|
|
|
138
142
|
* @returns {boolean} True if valid IPv4
|
|
139
143
|
*/
|
|
140
144
|
function isIPv4(str) {
|
|
141
|
-
|
|
142
|
-
return ipv4Regex.test(str);
|
|
145
|
+
return REGEX_IPv4.test(str);
|
|
143
146
|
}
|
|
144
147
|
|
|
145
148
|
/**
|
|
@@ -148,9 +151,7 @@ function isIPv4(str) {
|
|
|
148
151
|
* @returns {boolean} True if valid IPv6
|
|
149
152
|
*/
|
|
150
153
|
function isIPv6(str) {
|
|
151
|
-
|
|
152
|
-
const ipv6Regex = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::$|^::1$|^(?:[0-9a-fA-F]{1,4}:)*::(?:[0-9a-fA-F]{1,4}:)*[0-9a-fA-F]{1,4}$/;
|
|
153
|
-
return ipv6Regex.test(str);
|
|
154
|
+
return REGEX_IPv6.test(str);
|
|
154
155
|
}
|
|
155
156
|
|
|
156
157
|
/**
|
|
@@ -467,15 +468,7 @@ function validateRulesetFile(filePath, options = {}) {
|
|
|
467
468
|
} = options;
|
|
468
469
|
|
|
469
470
|
const fs = require('fs');
|
|
470
|
-
|
|
471
|
-
if (!fs.existsSync(filePath)) {
|
|
472
|
-
return {
|
|
473
|
-
isValid: false,
|
|
474
|
-
error: `File not found: ${filePath}`,
|
|
475
|
-
stats: { total: 0, valid: 0, invalid: 0, comments: 0 }
|
|
476
|
-
};
|
|
477
|
-
}
|
|
478
|
-
|
|
471
|
+
|
|
479
472
|
let content;
|
|
480
473
|
try {
|
|
481
474
|
content = fs.readFileSync(filePath, 'utf8');
|
|
@@ -721,15 +714,7 @@ function cleanRulesetFile(filePath, outputPath = null, options = {}) {
|
|
|
721
714
|
|
|
722
715
|
const fs = require('fs');
|
|
723
716
|
const path = require('path');
|
|
724
|
-
|
|
725
|
-
if (!fs.existsSync(filePath)) {
|
|
726
|
-
return {
|
|
727
|
-
success: false,
|
|
728
|
-
error: `File not found: ${filePath}`,
|
|
729
|
-
stats: { total: 0, valid: 0, invalid: 0, removed: 0, duplicates: 0 }
|
|
730
|
-
};
|
|
731
|
-
}
|
|
732
|
-
|
|
717
|
+
|
|
733
718
|
let content;
|
|
734
719
|
try {
|
|
735
720
|
content = fs.readFileSync(filePath, 'utf8');
|
package/nwss.js
CHANGED
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
// puppeteer for browser automation, fs for file system operations, psl for domain parsing.
|
|
4
4
|
// const pLimit = require('p-limit'); // Will be dynamically imported
|
|
5
|
-
const
|
|
5
|
+
const usePuppeteerCore = process.argv.includes('--use-puppeteer-core');
|
|
6
|
+
const puppeteer = usePuppeteerCore ? require('puppeteer-core') : require('puppeteer');
|
|
6
7
|
const fs = require('fs');
|
|
8
|
+
const os = require('os');
|
|
7
9
|
const psl = require('psl');
|
|
8
10
|
const path = require('path');
|
|
9
11
|
const { createGrepHandler, validateGrepAvailability } = require('./lib/grep');
|
|
@@ -41,6 +43,8 @@ const { processResults } = require('./lib/post-processing');
|
|
|
41
43
|
const { colorize, colors, messageColors, tags, formatLogMessage } = require('./lib/colorize');
|
|
42
44
|
// Enhanced mouse interaction and page simulation
|
|
43
45
|
const { performPageInteraction, createInteractionConfig, performContentClicks, humanLikeMouseMove } = require('./lib/interaction');
|
|
46
|
+
// Optional ghost-cursor support for advanced Bezier-based mouse movements
|
|
47
|
+
const { isGhostCursorAvailable, createGhostCursor, ghostMove, ghostClick, ghostRandomMove, resolveGhostCursorConfig } = require('./lib/ghost-cursor');
|
|
44
48
|
// Domain detection cache for performance optimization
|
|
45
49
|
const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } = require('./lib/domain-cache');
|
|
46
50
|
const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
|
|
@@ -117,7 +121,7 @@ const REALTIME_CLEANUP_THRESHOLD = 8; // Default pages to keep for realtime clea
|
|
|
117
121
|
*/
|
|
118
122
|
function detectPuppeteerVersion() {
|
|
119
123
|
try {
|
|
120
|
-
const puppeteer = require('puppeteer');
|
|
124
|
+
const puppeteer = usePuppeteerCore ? require('puppeteer-core') : require('puppeteer');
|
|
121
125
|
let versionString = null;
|
|
122
126
|
|
|
123
127
|
// Try multiple methods to get version
|
|
@@ -151,7 +155,7 @@ function detectPuppeteerVersion() {
|
|
|
151
155
|
// Enhanced redirect handling
|
|
152
156
|
const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/redirect');
|
|
153
157
|
// Ensure web browser is working correctly
|
|
154
|
-
const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload, purgeStaleTrackers } = require('./lib/browserhealth');
|
|
158
|
+
const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, untrackPage, cleanupPageBeforeReload, purgeStaleTrackers } = require('./lib/browserhealth');
|
|
155
159
|
|
|
156
160
|
// --- Script Configuration & Constants ---
|
|
157
161
|
const VERSION = '2.0.33'; // Script version
|
|
@@ -203,7 +207,9 @@ const localhostIndex = args.findIndex(arg => arg.startsWith('--localhost'));
|
|
|
203
207
|
if (localhostIndex !== -1) {
|
|
204
208
|
localhostIP = args[localhostIndex].includes('=') ? args[localhostIndex].split('=')[1] : '127.0.0.1';
|
|
205
209
|
}
|
|
210
|
+
const keepBrowserOpen = args.includes('--keep-open');
|
|
206
211
|
const disableInteract = args.includes('--no-interact');
|
|
212
|
+
const globalGhostCursor = args.includes('--ghost-cursor');
|
|
207
213
|
const plainOutput = args.includes('--plain');
|
|
208
214
|
const enableCDP = args.includes('--cdp');
|
|
209
215
|
const dnsmasqMode = args.includes('--dnsmasq');
|
|
@@ -545,8 +551,11 @@ General Options:
|
|
|
545
551
|
--compress-logs Compress log files with gzip (requires --dumpurls)
|
|
546
552
|
--sub-domains Output full subdomains instead of collapsing to root
|
|
547
553
|
--no-interact Disable page interactions globally
|
|
554
|
+
--ghost-cursor Use ghost-cursor Bezier mouse movements (requires: npm i ghost-cursor)
|
|
548
555
|
--custom-json <file> Use a custom config JSON file instead of config.json
|
|
549
556
|
--headful Launch browser with GUI (not headless)
|
|
557
|
+
--keep-open Keep browser open after scan completes (use with --headful)
|
|
558
|
+
--use-puppeteer-core Use puppeteer-core with system Chrome instead of bundled Chromium
|
|
550
559
|
--cdp Enable Chrome DevTools Protocol logging (now per-page if enabled)
|
|
551
560
|
--remove-dupes Remove duplicate domains from output (only with -o)
|
|
552
561
|
--eval-on-doc Globally enable evaluateOnNewDocument() for Fetch/XHR interception
|
|
@@ -657,6 +666,11 @@ Advanced Options:
|
|
|
657
666
|
interact_scrolling: true/false Enable scrolling simulation (default: true)
|
|
658
667
|
interact_clicks: true/false Enable element clicking simulation (default: false)
|
|
659
668
|
interact_typing: true/false Enable typing simulation (default: false)
|
|
669
|
+
cursor_mode: "ghost" Use ghost-cursor Bezier mouse (requires: npm i ghost-cursor)
|
|
670
|
+
ghost_cursor_speed: <number> Ghost-cursor speed multiplier (default: auto)
|
|
671
|
+
ghost_cursor_hesitate: <milliseconds> Delay before ghost-cursor clicks (default: 50)
|
|
672
|
+
ghost_cursor_overshoot: <pixels> Max ghost-cursor overshoot distance (default: auto)
|
|
673
|
+
ghost_cursor_duration: <milliseconds> Ghost-cursor interaction duration (default: interact_duration or 2000)
|
|
660
674
|
whois: ["term1", "term2"] Check whois data for ALL specified terms (AND logic)
|
|
661
675
|
whois-or: ["term1", "term2"] Check whois data for ANY specified term (OR logic)
|
|
662
676
|
whois_server_mode: "random" or "cycle" Server selection mode: random (default) or cycle through list
|
|
@@ -1375,7 +1389,7 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
1375
1389
|
*/
|
|
1376
1390
|
async function createBrowser(extraArgs = []) {
|
|
1377
1391
|
// Create temporary user data directory that we can fully control and clean up
|
|
1378
|
-
const tempUserDataDir =
|
|
1392
|
+
const tempUserDataDir = path.join(os.tmpdir(), `puppeteer-${Date.now()}-${Math.random().toString(36).substring(7)}`);
|
|
1379
1393
|
userDataDir = tempUserDataDir; // Store for cleanup tracking (use outer scope variable)
|
|
1380
1394
|
|
|
1381
1395
|
// Try to find system Chrome installation to avoid Puppeteer downloads
|
|
@@ -1405,11 +1419,15 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
1405
1419
|
}
|
|
1406
1420
|
|
|
1407
1421
|
const systemChromePaths = [
|
|
1422
|
+
// Linux / WSL
|
|
1408
1423
|
'/usr/bin/google-chrome-stable',
|
|
1409
1424
|
'/usr/bin/google-chrome',
|
|
1410
1425
|
'/usr/bin/chromium-browser',
|
|
1411
1426
|
'/usr/bin/chromium',
|
|
1412
|
-
'/snap/bin/chromium'
|
|
1427
|
+
'/snap/bin/chromium',
|
|
1428
|
+
// macOS
|
|
1429
|
+
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
1430
|
+
'/Applications/Chromium.app/Contents/MacOS/Chromium'
|
|
1413
1431
|
];
|
|
1414
1432
|
// V8 Optimization: Freeze the Chrome paths array since it's constant
|
|
1415
1433
|
Object.freeze(systemChromePaths);
|
|
@@ -1425,6 +1443,10 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
1425
1443
|
break;
|
|
1426
1444
|
}
|
|
1427
1445
|
}
|
|
1446
|
+
if (usePuppeteerCore && !executablePath) {
|
|
1447
|
+
console.error(formatLogMessage('error', '--use-puppeteer-core requires a system Chrome installation. No Chrome found in standard paths.'));
|
|
1448
|
+
process.exit(1);
|
|
1449
|
+
}
|
|
1428
1450
|
const browser = await puppeteer.launch({
|
|
1429
1451
|
// Use system Chrome if available to avoid downloads
|
|
1430
1452
|
executablePath: executablePath,
|
|
@@ -3415,53 +3437,115 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
3415
3437
|
}
|
|
3416
3438
|
}
|
|
3417
3439
|
|
|
3418
|
-
if (interactEnabled && !disableInteract) {
|
|
3419
|
-
if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
|
|
3420
|
-
|
|
3421
|
-
// Mark page as processing during interactions
|
|
3422
|
-
updatePageUsage(page, true);
|
|
3423
|
-
// Use enhanced interaction module with hard abort timeout
|
|
3424
|
-
const INTERACTION_HARD_TIMEOUT = 15000;
|
|
3425
|
-
try {
|
|
3426
|
-
await Promise.race([
|
|
3427
|
-
performPageInteraction(page, currentUrl, interactionConfig, forceDebug),
|
|
3428
|
-
new Promise((_, reject) => setTimeout(() => reject(new Error('interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
|
|
3429
|
-
]);
|
|
3430
|
-
} catch (interactTimeoutErr) {
|
|
3431
|
-
if (forceDebug) console.log(formatLogMessage('debug', `[interaction] Aborted after ${INTERACTION_HARD_TIMEOUT}ms: ${interactTimeoutErr.message}`));
|
|
3432
|
-
}
|
|
3433
|
-
}
|
|
3434
|
-
|
|
3435
3440
|
const delayMs = DEFAULT_DELAY;
|
|
3436
|
-
|
|
3441
|
+
|
|
3437
3442
|
// Optimized delays for Puppeteer 23.x performance
|
|
3438
3443
|
const isFastSite = timeout <= TIMEOUTS.FAST_SITE_THRESHOLD;
|
|
3439
3444
|
const networkIdleTime = TIMEOUTS.NETWORK_IDLE; // Balanced: 2s for reliable network detection
|
|
3440
3445
|
const networkIdleTimeout = Math.min(timeout / 2, TIMEOUTS.NETWORK_IDLE_MAX); // Balanced: 10s timeout
|
|
3441
3446
|
const actualDelay = Math.min(delayMs, TIMEOUTS.NETWORK_IDLE); // Balanced: 2s delay for stability
|
|
3442
|
-
|
|
3443
|
-
//
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3447
|
+
|
|
3448
|
+
// Build delay promise (networkIdle + delay + optional flowProxy delay)
|
|
3449
|
+
const delayPromise = (async () => {
|
|
3450
|
+
if (page && !page.isClosed()) {
|
|
3451
|
+
try {
|
|
3452
|
+
await page.waitForNetworkIdle({
|
|
3453
|
+
idleTime: networkIdleTime,
|
|
3454
|
+
timeout: networkIdleTimeout
|
|
3455
|
+
});
|
|
3456
|
+
} catch (networkIdleErr) {
|
|
3457
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Network idle wait failed: ${networkIdleErr.message}`));
|
|
3458
|
+
}
|
|
3453
3459
|
}
|
|
3454
|
-
|
|
3460
|
+
await fastTimeout(actualDelay);
|
|
3461
|
+
if (flowproxyDetection) {
|
|
3462
|
+
const additionalDelay = Math.min(siteConfig.flowproxy_additional_delay || 3000, 3000);
|
|
3463
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Applying flowProxy additional delay: ${additionalDelay}ms`));
|
|
3464
|
+
await fastTimeout(additionalDelay);
|
|
3465
|
+
}
|
|
3466
|
+
})();
|
|
3455
3467
|
|
|
3456
|
-
//
|
|
3457
|
-
|
|
3468
|
+
// Build interaction promise — runs concurrently with delay
|
|
3469
|
+
const interactPromise = (async () => {
|
|
3470
|
+
if (!(interactEnabled && !disableInteract)) return;
|
|
3471
|
+
if (forceDebug) console.log(formatLogMessage('debug', `interaction simulation enabled for ${currentUrl}`));
|
|
3458
3472
|
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
const
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3473
|
+
// Mark page as processing during interactions
|
|
3474
|
+
updatePageUsage(page, true);
|
|
3475
|
+
const INTERACTION_HARD_TIMEOUT = 15000;
|
|
3476
|
+
|
|
3477
|
+
// Check if ghost-cursor mode is enabled for this site
|
|
3478
|
+
const ghostConfig = resolveGhostCursorConfig(siteConfig, globalGhostCursor, forceDebug);
|
|
3479
|
+
|
|
3480
|
+
try {
|
|
3481
|
+
if (ghostConfig) {
|
|
3482
|
+
// Ghost-cursor mode: Bezier-based mouse movements
|
|
3483
|
+
if (forceDebug) console.log(formatLogMessage('debug', `[ghost-cursor] Using ghost-cursor for ${currentUrl}`));
|
|
3484
|
+
const cursor = createGhostCursor(page, { forceDebug });
|
|
3485
|
+
if (cursor) {
|
|
3486
|
+
await Promise.race([
|
|
3487
|
+
(async () => {
|
|
3488
|
+
const viewport = page.viewport() || { width: 1200, height: 800 };
|
|
3489
|
+
const ghostDuration = ghostConfig.duration || 2000;
|
|
3490
|
+
const ghostStart = Date.now();
|
|
3491
|
+
const ghostTimeLeft = () => ghostDuration - (Date.now() - ghostStart);
|
|
3492
|
+
|
|
3493
|
+
// Time-based Bezier mouse movements — runs for ghostDuration ms
|
|
3494
|
+
while (ghostTimeLeft() > 200) {
|
|
3495
|
+
const toX = Math.floor(Math.random() * (viewport.width - 100)) + 50;
|
|
3496
|
+
const toY = Math.floor(Math.random() * (viewport.height - 100)) + 50;
|
|
3497
|
+
await ghostMove(cursor, toX, toY, {
|
|
3498
|
+
moveSpeed: ghostConfig.moveSpeed,
|
|
3499
|
+
overshootThreshold: ghostConfig.overshootThreshold,
|
|
3500
|
+
forceDebug
|
|
3501
|
+
});
|
|
3502
|
+
if (ghostTimeLeft() > 100) {
|
|
3503
|
+
await new Promise(r => setTimeout(r, 25 + Math.random() * 75));
|
|
3504
|
+
}
|
|
3505
|
+
}
|
|
3506
|
+
if (ghostTimeLeft() > 100 && Math.random() < 0.3) {
|
|
3507
|
+
await ghostRandomMove(cursor, { forceDebug });
|
|
3508
|
+
}
|
|
3509
|
+
if (interactionConfig.includeElementClicks && ghostTimeLeft() > 100) {
|
|
3510
|
+
const clickX = Math.floor(viewport.width * 0.2 + Math.random() * viewport.width * 0.6);
|
|
3511
|
+
const clickY = Math.floor(viewport.height * 0.2 + Math.random() * viewport.height * 0.6);
|
|
3512
|
+
await ghostClick(cursor, { x: clickX, y: clickY }, {
|
|
3513
|
+
hesitate: ghostConfig.hesitate,
|
|
3514
|
+
forceDebug
|
|
3515
|
+
});
|
|
3516
|
+
}
|
|
3517
|
+
if (interactionConfig.includeScrolling) {
|
|
3518
|
+
await performPageInteraction(page, currentUrl, {
|
|
3519
|
+
...interactionConfig,
|
|
3520
|
+
mouseMovements: 0,
|
|
3521
|
+
includeElementClicks: false,
|
|
3522
|
+
includeTyping: false
|
|
3523
|
+
}, forceDebug);
|
|
3524
|
+
}
|
|
3525
|
+
})(),
|
|
3526
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('ghost-cursor interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
|
|
3527
|
+
]);
|
|
3528
|
+
} else {
|
|
3529
|
+
if (forceDebug) console.log(formatLogMessage('debug', '[ghost-cursor] Falling back to built-in mouse'));
|
|
3530
|
+
await Promise.race([
|
|
3531
|
+
performPageInteraction(page, currentUrl, interactionConfig, forceDebug),
|
|
3532
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
|
|
3533
|
+
]);
|
|
3534
|
+
}
|
|
3535
|
+
} else {
|
|
3536
|
+
// Standard built-in mouse interaction
|
|
3537
|
+
await Promise.race([
|
|
3538
|
+
performPageInteraction(page, currentUrl, interactionConfig, forceDebug),
|
|
3539
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('interaction hard timeout')), INTERACTION_HARD_TIMEOUT))
|
|
3540
|
+
]);
|
|
3541
|
+
}
|
|
3542
|
+
} catch (interactTimeoutErr) {
|
|
3543
|
+
if (forceDebug) console.log(formatLogMessage('debug', `[interaction] Aborted after ${INTERACTION_HARD_TIMEOUT}ms: ${interactTimeoutErr.message}`));
|
|
3544
|
+
}
|
|
3545
|
+
})();
|
|
3546
|
+
|
|
3547
|
+
// Run delay and mouse interaction concurrently — mouse moves while page settles
|
|
3548
|
+
await Promise.all([delayPromise, interactPromise]);
|
|
3465
3549
|
|
|
3466
3550
|
// Use fast timeout helper for consistent Puppeteer 23.x compatibility
|
|
3467
3551
|
|
|
@@ -3893,11 +3977,14 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
3893
3977
|
}
|
|
3894
3978
|
}
|
|
3895
3979
|
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3980
|
+
if (!keepBrowserOpen) {
|
|
3981
|
+
try {
|
|
3982
|
+
untrackPage(page);
|
|
3983
|
+
await page.close();
|
|
3984
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page closed for ${currentUrl}`));
|
|
3985
|
+
} catch (pageCloseErr) {
|
|
3986
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Failed to close page for ${currentUrl}: ${pageCloseErr.message}`));
|
|
3987
|
+
}
|
|
3901
3988
|
}
|
|
3902
3989
|
}
|
|
3903
3990
|
}
|
|
@@ -4511,6 +4598,14 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
4511
4598
|
wgDisconnectAll(forceDebug);
|
|
4512
4599
|
ovpnDisconnectAll(forceDebug);
|
|
4513
4600
|
|
|
4601
|
+
// Keep browser open if --keep-open flag is set (useful with --headful for inspection)
|
|
4602
|
+
if (keepBrowserOpen && !launchHeadless) {
|
|
4603
|
+
console.log(messageColors.info('Browser kept open.') + ' Close the browser window or press Ctrl+C to exit.');
|
|
4604
|
+
await new Promise((resolve) => {
|
|
4605
|
+
browser.on('disconnected', resolve);
|
|
4606
|
+
});
|
|
4607
|
+
}
|
|
4608
|
+
|
|
4514
4609
|
// Perform comprehensive final cleanup using enhanced browserexit module
|
|
4515
4610
|
if (forceDebug) console.log(formatLogMessage('debug', `Starting comprehensive browser cleanup...`));
|
|
4516
4611
|
|
|
@@ -4535,7 +4630,7 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
4535
4630
|
if (forceDebug) {
|
|
4536
4631
|
console.log(formatLogMessage('debug', `Final cleanup results: ${cleanupResult.success ? 'success' : 'failed'}`));
|
|
4537
4632
|
console.log(formatLogMessage('debug', `Browser closed: ${cleanupResult.browserClosed}, Temp files cleaned: ${cleanupResult.tempFilesCleanedCount || 0}, User data cleaned: ${cleanupResult.userDataCleaned}`));
|
|
4538
|
-
|
|
4633
|
+
|
|
4539
4634
|
if (cleanupResult.errors.length > 0) {
|
|
4540
4635
|
cleanupResult.errors.forEach(err => console.log(formatLogMessage('debug', `Cleanup error: ${err}`)));
|
|
4541
4636
|
}
|
|
@@ -4543,10 +4638,10 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
4543
4638
|
|
|
4544
4639
|
// Final aggressive cleanup to catch any remaining temp files
|
|
4545
4640
|
if (forceDebug) console.log(formatLogMessage('debug', 'Performing final aggressive temp file cleanup...'));
|
|
4546
|
-
await cleanupChromeTempFiles({
|
|
4547
|
-
includeSnapTemp: true,
|
|
4641
|
+
await cleanupChromeTempFiles({
|
|
4642
|
+
includeSnapTemp: true,
|
|
4548
4643
|
forceDebug,
|
|
4549
|
-
comprehensive: true
|
|
4644
|
+
comprehensive: true
|
|
4550
4645
|
});
|
|
4551
4646
|
await fastTimeout(TIMEOUTS.BROWSER_STABILIZE_DELAY); // Give filesystem time to sync
|
|
4552
4647
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.59",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"lint": "eslint *.js lib/*.js"
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
|
+
"ghost-cursor": "^1.4.2",
|
|
13
14
|
"lru-cache": "^10.4.3",
|
|
14
15
|
"p-limit": "^4.0.0",
|
|
15
16
|
"psl": "^1.15.0",
|
|
@@ -48,6 +49,9 @@
|
|
|
48
49
|
"url": "https://github.com/ryanbr/network-scanner/issues"
|
|
49
50
|
},
|
|
50
51
|
"homepage": "https://github.com/ryanbr/network-scanner",
|
|
52
|
+
"optionalDependencies": {
|
|
53
|
+
"puppeteer-core": ">=20.0.0"
|
|
54
|
+
},
|
|
51
55
|
"devDependencies": {
|
|
52
56
|
"eslint": "^10.0.2",
|
|
53
57
|
"globals": "^16.3.0"
|
package/.clauderc
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"description": "Network scanner that monitors website requests and generates blocking rules. Uses Puppeteer to load sites, intercepts network traffic, matches patterns, and outputs rules in various formats (adblock, dnsmasq, hosts file, etc.).",
|
|
3
|
-
|
|
4
|
-
"conventions": [
|
|
5
|
-
"Store modular functionality in ./lib/ directory with focused, single-purpose modules",
|
|
6
|
-
"Use messageColors and formatLogMessage from ./lib/colorize for consistent console output",
|
|
7
|
-
"Implement timeout protection for all Puppeteer operations using Promise.race patterns",
|
|
8
|
-
"Handle browser lifecycle with comprehensive cleanup in try-finally blocks",
|
|
9
|
-
"Validate all external tool availability before use (grep, curl, whois, dig)",
|
|
10
|
-
"Use forceDebug flag for detailed logging, silentMode for minimal output"
|
|
11
|
-
],
|
|
12
|
-
|
|
13
|
-
"files": {
|
|
14
|
-
"important": [
|
|
15
|
-
"nwss.js",
|
|
16
|
-
"config.json",
|
|
17
|
-
"lib/*.js",
|
|
18
|
-
"*.md",
|
|
19
|
-
"nwss.1"
|
|
20
|
-
],
|
|
21
|
-
"ignore": [
|
|
22
|
-
"node_modules/**",
|
|
23
|
-
"logs/**",
|
|
24
|
-
"sources/**",
|
|
25
|
-
".cache/**",
|
|
26
|
-
"*.log",
|
|
27
|
-
"*.gz"
|
|
28
|
-
]
|
|
29
|
-
}
|
|
30
|
-
}
|