@fanboynz/network-scanner 2.0.59 → 2.0.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +51 -1
- package/lib/adblock.js +215 -179
- package/lib/colorize.js +3 -1
- package/lib/compare.js +19 -32
- package/lib/domain-cache.js +9 -7
- package/lib/fingerprint.js +41 -11
- package/lib/grep.js +9 -13
- package/lib/nettools.js +177 -42
- package/lib/output.js +17 -30
- package/nwss.js +290 -62
- package/package.json +1 -1
package/lib/compare.js
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
2
|
const path = require('path');
|
|
3
3
|
|
|
4
|
+
// Pre-compiled regexes for rule normalization (shared across functions)
|
|
5
|
+
const RE_ADBLOCK_PREFIX = /^\|\|?/;
|
|
6
|
+
const RE_LOCALHOST_127 = /^127\.0\.0\.1\s+/;
|
|
7
|
+
const RE_LOCALHOST_000 = /^0\.0\.0\.0\s+/;
|
|
8
|
+
const RE_CARET_SUFFIX = /\^.*$/;
|
|
9
|
+
const RE_DOLLAR_SUFFIX = /\$.*$/;
|
|
10
|
+
|
|
11
|
+
function normalizeRuleInline(rule) {
|
|
12
|
+
return rule
|
|
13
|
+
.replace(RE_ADBLOCK_PREFIX, '')
|
|
14
|
+
.replace(RE_LOCALHOST_127, '')
|
|
15
|
+
.replace(RE_LOCALHOST_000, '')
|
|
16
|
+
.replace(RE_CARET_SUFFIX, '')
|
|
17
|
+
.replace(RE_DOLLAR_SUFFIX, '')
|
|
18
|
+
.trim();
|
|
19
|
+
}
|
|
20
|
+
|
|
4
21
|
/**
|
|
5
22
|
* Loads rules from a comparison file and returns them as a Set for fast lookup
|
|
6
23
|
* @param {string} compareFilePath - Path to the file containing existing rules
|
|
@@ -17,23 +34,7 @@ function loadComparisonRules(compareFilePath, forceDebug = false) {
|
|
|
17
34
|
const rules = new Set();
|
|
18
35
|
|
|
19
36
|
for (const line of lines) {
|
|
20
|
-
|
|
21
|
-
let normalizedRule = line;
|
|
22
|
-
|
|
23
|
-
// Remove adblock prefixes (||, |, etc.)
|
|
24
|
-
normalizedRule = normalizedRule.replace(/^\|\|/, '');
|
|
25
|
-
normalizedRule = normalizedRule.replace(/^\|/, '');
|
|
26
|
-
|
|
27
|
-
// Remove localhost prefixes
|
|
28
|
-
normalizedRule = normalizedRule.replace(/^127\.0\.0\.1\s+/, '');
|
|
29
|
-
normalizedRule = normalizedRule.replace(/^0\.0\.0\.0\s+/, '');
|
|
30
|
-
|
|
31
|
-
// Remove adblock suffixes and modifiers
|
|
32
|
-
normalizedRule = normalizedRule.replace(/\^.*$/, ''); // Remove ^ and everything after
|
|
33
|
-
normalizedRule = normalizedRule.replace(/\$.*$/, ''); // Remove $ and everything after
|
|
34
|
-
|
|
35
|
-
// Clean up and add to set
|
|
36
|
-
normalizedRule = normalizedRule.trim();
|
|
37
|
+
const normalizedRule = normalizeRuleInline(line);
|
|
37
38
|
if (normalizedRule) {
|
|
38
39
|
rules.add(normalizedRule);
|
|
39
40
|
}
|
|
@@ -55,21 +56,7 @@ function loadComparisonRules(compareFilePath, forceDebug = false) {
|
|
|
55
56
|
* @returns {string} Normalized rule
|
|
56
57
|
*/
|
|
57
58
|
function normalizeRule(rule) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
// Remove adblock prefixes
|
|
61
|
-
normalized = normalized.replace(/^\|\|/, '');
|
|
62
|
-
normalized = normalized.replace(/^\|/, '');
|
|
63
|
-
|
|
64
|
-
// Remove localhost prefixes
|
|
65
|
-
normalized = normalized.replace(/^127\.0\.0\.1\s+/, '');
|
|
66
|
-
normalized = normalized.replace(/^0\.0\.0\.0\s+/, '');
|
|
67
|
-
|
|
68
|
-
// Remove adblock suffixes and modifiers
|
|
69
|
-
normalized = normalized.replace(/\^.*$/, '');
|
|
70
|
-
normalized = normalized.replace(/\$.*$/, '');
|
|
71
|
-
|
|
72
|
-
return normalized.trim();
|
|
59
|
+
return normalizeRuleInline(rule);
|
|
73
60
|
}
|
|
74
61
|
|
|
75
62
|
/**
|
package/lib/domain-cache.js
CHANGED
|
@@ -148,14 +148,16 @@ class DomainCache {
|
|
|
148
148
|
*/
|
|
149
149
|
clearOldestEntries(count) {
|
|
150
150
|
if (count <= 0) return;
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
151
|
+
|
|
152
|
+
let removed = 0;
|
|
153
|
+
for (const domain of this.cache) {
|
|
154
|
+
if (removed >= count) break;
|
|
155
|
+
this.cache.delete(domain);
|
|
156
|
+
removed++;
|
|
157
|
+
}
|
|
158
|
+
|
|
157
159
|
if (this.enableLogging) {
|
|
158
|
-
console.log(formatLogMessage('debug', `${this.logPrefix} Cleared ${
|
|
160
|
+
console.log(formatLogMessage('debug', `${this.logPrefix} Cleared ${removed} old entries, cache size now: ${this.cache.size}`));
|
|
159
161
|
}
|
|
160
162
|
}
|
|
161
163
|
|
package/lib/fingerprint.js
CHANGED
|
@@ -22,6 +22,7 @@ function seededRandom(seed) {
|
|
|
22
22
|
|
|
23
23
|
// Cache fingerprints per domain so reloads and multi-page visits stay consistent
|
|
24
24
|
const _fingerprintCache = new Map();
|
|
25
|
+
const FINGERPRINT_CACHE_MAX = 500;
|
|
25
26
|
|
|
26
27
|
// Type-specific property spoofing functions for monomorphic optimization
|
|
27
28
|
// Built-in properties that should not be modified
|
|
@@ -32,12 +33,12 @@ const BUILT_IN_PROPERTIES = new Set([
|
|
|
32
33
|
|
|
33
34
|
// User agent collections with latest versions
|
|
34
35
|
const USER_AGENT_COLLECTIONS = Object.freeze(new Map([
|
|
35
|
-
['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
36
|
-
['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
37
|
-
['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
38
|
-
['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:
|
|
39
|
-
['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:
|
|
40
|
-
['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:
|
|
36
|
+
['chrome', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
|
|
37
|
+
['chrome_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
|
|
38
|
+
['chrome_linux', "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"],
|
|
39
|
+
['firefox', "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"],
|
|
40
|
+
['firefox_mac', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0"],
|
|
41
|
+
['firefox_linux', "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0"],
|
|
41
42
|
['safari', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15"]
|
|
42
43
|
]));
|
|
43
44
|
|
|
@@ -237,7 +238,12 @@ function generateRealisticFingerprint(userAgent, domain = '') {
|
|
|
237
238
|
};
|
|
238
239
|
|
|
239
240
|
// Cache for this domain
|
|
240
|
-
if (domain)
|
|
241
|
+
if (domain) {
|
|
242
|
+
if (_fingerprintCache.size >= FINGERPRINT_CACHE_MAX) {
|
|
243
|
+
_fingerprintCache.delete(_fingerprintCache.keys().next().value);
|
|
244
|
+
}
|
|
245
|
+
_fingerprintCache.set(domain, fingerprint);
|
|
246
|
+
}
|
|
241
247
|
|
|
242
248
|
return fingerprint;
|
|
243
249
|
}
|
|
@@ -495,7 +501,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
495
501
|
}),
|
|
496
502
|
getManifest: () => ({
|
|
497
503
|
name: "Chrome",
|
|
498
|
-
version: "
|
|
504
|
+
version: "146.0.0.0",
|
|
499
505
|
manifest_version: 3,
|
|
500
506
|
description: "Chrome Browser"
|
|
501
507
|
}),
|
|
@@ -1609,18 +1615,41 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1609
1615
|
}, 'enhanced mouse/pointer spoofing');
|
|
1610
1616
|
|
|
1611
1617
|
safeExecute(() => {
|
|
1612
|
-
//
|
|
1618
|
+
// Neutralize CDP fingerprinting traps and filter DevTools traces
|
|
1619
|
+
// CDP's Runtime.enable causes the inspector to read properties on console-logged objects.
|
|
1620
|
+
// Detection scripts exploit this via console.debug with Error objects (custom .stack getters)
|
|
1621
|
+
// or objects with Proxy prototypes. Only override console.debug — safest, minimal footprint.
|
|
1622
|
+
|
|
1613
1623
|
const originalConsoleDebug = console.debug;
|
|
1614
1624
|
console.debug = function(...args) {
|
|
1625
|
+
// Filter DevTools-related messages
|
|
1615
1626
|
const message = args.join(' ');
|
|
1616
1627
|
if (typeof message === 'string' && (
|
|
1617
1628
|
message.includes('DevTools') ||
|
|
1618
1629
|
message.includes('Runtime.evaluate') ||
|
|
1619
1630
|
message.includes('Page.addScriptToEvaluateOnNewDocument') ||
|
|
1620
1631
|
message.includes('Protocol error'))) {
|
|
1621
|
-
return;
|
|
1632
|
+
return;
|
|
1622
1633
|
}
|
|
1623
|
-
|
|
1634
|
+
// Sanitize args to neutralize CDP fingerprinting traps
|
|
1635
|
+
const sanitized = args.map(arg => {
|
|
1636
|
+
// Strip Error objects with custom .stack getters (CDP inspector reads .stack)
|
|
1637
|
+
if (arg instanceof Error) {
|
|
1638
|
+
const desc = Object.getOwnPropertyDescriptor(arg, 'stack');
|
|
1639
|
+
if (desc && desc.get) return `${arg.name}: ${arg.message}`;
|
|
1640
|
+
}
|
|
1641
|
+
// Neutralize Proxy prototype traps (CDP inspector walks prototype chain)
|
|
1642
|
+
if (arg !== null && typeof arg === 'object') {
|
|
1643
|
+
try {
|
|
1644
|
+
const proto = Object.getPrototypeOf(arg);
|
|
1645
|
+
if (proto && proto !== Object.prototype && proto !== Array.prototype) {
|
|
1646
|
+
try { Object.keys(proto); } catch { return '[object Object]'; }
|
|
1647
|
+
}
|
|
1648
|
+
} catch { return '[object Object]'; }
|
|
1649
|
+
}
|
|
1650
|
+
return arg;
|
|
1651
|
+
});
|
|
1652
|
+
return originalConsoleDebug.apply(this, sanitized);
|
|
1624
1653
|
};
|
|
1625
1654
|
|
|
1626
1655
|
}, 'console error suppression');
|
|
@@ -1670,6 +1699,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1670
1699
|
if (typeof window.Image === 'function') maskAsNative(window.Image, 'Image');
|
|
1671
1700
|
if (typeof window.fetch === 'function') maskAsNative(window.fetch, 'fetch');
|
|
1672
1701
|
if (typeof window.PointerEvent === 'function') maskAsNative(window.PointerEvent, 'PointerEvent');
|
|
1702
|
+
if (typeof console.debug === 'function') maskAsNative(console.debug, 'debug');
|
|
1673
1703
|
|
|
1674
1704
|
// Mask property getters on navigator
|
|
1675
1705
|
const navProps = ['userAgentData', 'connection', 'pdfViewerEnabled', 'webdriver',
|
package/lib/grep.js
CHANGED
|
@@ -41,23 +41,19 @@ function grepContent(content, searchPatterns, options = {}) {
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
try {
|
|
44
|
-
|
|
45
44
|
const allMatches = [];
|
|
46
45
|
let firstMatch = null;
|
|
47
|
-
|
|
46
|
+
|
|
47
|
+
// Build common args once outside the loop
|
|
48
|
+
const baseArgs = ['--text', '--color=never'];
|
|
49
|
+
if (ignoreCase) baseArgs.push('-i');
|
|
50
|
+
if (wholeWord) baseArgs.push('-w');
|
|
51
|
+
if (!regex) baseArgs.push('-F');
|
|
52
|
+
|
|
48
53
|
for (const pattern of searchPatterns) {
|
|
49
54
|
if (!pattern || pattern.trim().length === 0) continue;
|
|
50
|
-
|
|
51
|
-
const grepArgs = [
|
|
52
|
-
'--text', // Treat file as text
|
|
53
|
-
'--color=never', // Disable color output
|
|
54
|
-
];
|
|
55
|
-
|
|
56
|
-
if (ignoreCase) grepArgs.push('-i');
|
|
57
|
-
if (wholeWord) grepArgs.push('-w');
|
|
58
|
-
if (!regex) grepArgs.push('-F'); // Fixed strings (literal)
|
|
59
|
-
|
|
60
|
-
grepArgs.push(pattern);
|
|
55
|
+
|
|
56
|
+
const grepArgs = [...baseArgs, pattern];
|
|
61
57
|
|
|
62
58
|
try {
|
|
63
59
|
const result = spawnSync('grep', grepArgs, {
|
package/lib/nettools.js
CHANGED
|
@@ -5,8 +5,11 @@
|
|
|
5
5
|
|
|
6
6
|
const { exec, execSync } = require('child_process');
|
|
7
7
|
const util = require('util');
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
8
10
|
const { formatLogMessage, messageColors } = require('./colorize');
|
|
9
11
|
const execPromise = util.promisify(exec);
|
|
12
|
+
const ANSI_REGEX = /\x1b\[[0-9;]*m/g;
|
|
10
13
|
|
|
11
14
|
// Cycling index for whois server rotation
|
|
12
15
|
let whoisServerCycleIndex = 0;
|
|
@@ -16,14 +19,121 @@ let whoisServerCycleIndex = 0;
|
|
|
16
19
|
// DNS records don't change based on what terms you're searching for,
|
|
17
20
|
// so we cache the raw dig output and let each handler check its own terms against it
|
|
18
21
|
const globalDigResultCache = new Map();
|
|
19
|
-
const GLOBAL_DIG_CACHE_TTL =
|
|
20
|
-
const GLOBAL_DIG_CACHE_MAX =
|
|
22
|
+
const GLOBAL_DIG_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
|
|
23
|
+
const GLOBAL_DIG_CACHE_MAX = 1000;
|
|
21
24
|
|
|
22
25
|
// Global whois result cache — shared across ALL handler instances and processUrl calls
|
|
23
26
|
// Whois data is per root domain and doesn't change based on search terms
|
|
24
27
|
const globalWhoisResultCache = new Map();
|
|
25
|
-
const GLOBAL_WHOIS_CACHE_TTL =
|
|
26
|
-
const GLOBAL_WHOIS_CACHE_MAX =
|
|
28
|
+
const GLOBAL_WHOIS_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
|
|
29
|
+
const GLOBAL_WHOIS_CACHE_MAX = 1000;
|
|
30
|
+
|
|
31
|
+
// Persistent disk cache file paths
|
|
32
|
+
const DIG_CACHE_FILE = path.join(__dirname, '..', '.digcache');
|
|
33
|
+
const WHOIS_CACHE_FILE = path.join(__dirname, '..', '.whoiscache');
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Load persistent cache from disk into in-memory Map
|
|
37
|
+
* Skips expired entries and enforces max size
|
|
38
|
+
* @param {string} filePath - Path to cache file
|
|
39
|
+
* @param {Map} cache - In-memory cache Map to populate
|
|
40
|
+
* @param {number} ttl - TTL in milliseconds
|
|
41
|
+
* @param {number} maxSize - Maximum cache entries
|
|
42
|
+
*/
|
|
43
|
+
function loadDiskCache(filePath, cache, ttl, maxSize) {
|
|
44
|
+
try {
|
|
45
|
+
if (!fs.existsSync(filePath)) return;
|
|
46
|
+
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
47
|
+
const now = Date.now();
|
|
48
|
+
let loaded = 0;
|
|
49
|
+
for (const [key, entry] of Object.entries(data)) {
|
|
50
|
+
if (loaded >= maxSize) break;
|
|
51
|
+
if (now - entry.timestamp < ttl) {
|
|
52
|
+
cache.set(key, entry);
|
|
53
|
+
loaded++;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
} catch {
|
|
57
|
+
// Corrupt or unreadable cache file — delete and start fresh
|
|
58
|
+
try { fs.unlinkSync(filePath); } catch {}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Save in-memory cache to disk, evicting oldest entries if over max size
|
|
64
|
+
* @param {string} filePath - Path to cache file
|
|
65
|
+
* @param {Map} cache - In-memory cache Map to persist
|
|
66
|
+
* @param {number} ttl - TTL in milliseconds
|
|
67
|
+
* @param {number} maxSize - Maximum cache entries
|
|
68
|
+
*/
|
|
69
|
+
function saveDiskCache(filePath, cache, ttl, maxSize) {
|
|
70
|
+
try {
|
|
71
|
+
const now = Date.now();
|
|
72
|
+
const entries = {};
|
|
73
|
+
let count = 0;
|
|
74
|
+
|
|
75
|
+
// Collect valid entries, skip expired
|
|
76
|
+
for (const [key, entry] of cache) {
|
|
77
|
+
if (now - entry.timestamp < ttl) {
|
|
78
|
+
entries[key] = entry;
|
|
79
|
+
count++;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// If over max, keep only the newest entries
|
|
84
|
+
if (count > maxSize) {
|
|
85
|
+
const sorted = Object.entries(entries)
|
|
86
|
+
.sort((a, b) => b[1].timestamp - a[1].timestamp)
|
|
87
|
+
.slice(0, maxSize);
|
|
88
|
+
const trimmed = {};
|
|
89
|
+
for (const [key, entry] of sorted) {
|
|
90
|
+
trimmed[key] = entry;
|
|
91
|
+
}
|
|
92
|
+
fs.writeFileSync(filePath, JSON.stringify(trimmed, null, 2));
|
|
93
|
+
} else {
|
|
94
|
+
fs.writeFileSync(filePath, JSON.stringify(entries, null, 2));
|
|
95
|
+
}
|
|
96
|
+
} catch {
|
|
97
|
+
// Disk write failed — non-fatal, in-memory cache still works
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Track in-flight lookups to prevent duplicate concurrent requests
|
|
102
|
+
const pendingDigLookups = new Map();
|
|
103
|
+
const pendingWhoisLookups = new Map();
|
|
104
|
+
|
|
105
|
+
// DNS cache statistics
|
|
106
|
+
const dnsCacheStats = { digHits: 0, digMisses: 0, whoisHits: 0, whoisMisses: 0, freshDig: [], freshWhois: [] };
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Get DNS cache statistics for end-of-scan reporting
|
|
110
|
+
* @returns {Object} Cache hit/miss counts and fresh domain lists
|
|
111
|
+
*/
|
|
112
|
+
function getDnsCacheStats() {
|
|
113
|
+
return { ...dnsCacheStats };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Disk cache is opt-in via --dns-cache flag
|
|
117
|
+
let diskCacheEnabled = false;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Enable persistent disk caching for dig/whois results
|
|
121
|
+
* Call this when --dns-cache flag is set
|
|
122
|
+
*/
|
|
123
|
+
function enableDiskCache() {
|
|
124
|
+
diskCacheEnabled = true;
|
|
125
|
+
loadDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
|
|
126
|
+
loadDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
|
|
127
|
+
|
|
128
|
+
// Save caches to disk once on process exit instead of per-lookup
|
|
129
|
+
const flushCaches = () => {
|
|
130
|
+
saveDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
|
|
131
|
+
saveDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
|
|
132
|
+
};
|
|
133
|
+
process.on('exit', flushCaches);
|
|
134
|
+
process.on('SIGINT', () => { flushCaches(); process.exit(0); });
|
|
135
|
+
process.on('SIGTERM', () => { flushCaches(); process.exit(0); });
|
|
136
|
+
}
|
|
27
137
|
|
|
28
138
|
/**
|
|
29
139
|
* Strips ANSI color codes from a string for clean file logging
|
|
@@ -32,7 +142,8 @@ const GLOBAL_WHOIS_CACHE_MAX = 500;
|
|
|
32
142
|
*/
|
|
33
143
|
function stripAnsiColors(text) {
|
|
34
144
|
// Remove ANSI escape sequences (color codes)
|
|
35
|
-
|
|
145
|
+
ANSI_REGEX.lastIndex = 0;
|
|
146
|
+
return text.replace(ANSI_REGEX, '');
|
|
36
147
|
}
|
|
37
148
|
|
|
38
149
|
/**
|
|
@@ -1008,6 +1119,7 @@ function createNetToolsHandler(config) {
|
|
|
1008
1119
|
cacheAge: now - cachedEntry.timestamp,
|
|
1009
1120
|
originalTimestamp: cachedEntry.timestamp
|
|
1010
1121
|
});
|
|
1122
|
+
dnsCacheStats.whoisHits++;
|
|
1011
1123
|
} else {
|
|
1012
1124
|
// Cache expired, remove it
|
|
1013
1125
|
globalWhoisResultCache.delete(whoisCacheKey);
|
|
@@ -1019,34 +1131,43 @@ function createNetToolsHandler(config) {
|
|
|
1019
1131
|
|
|
1020
1132
|
// Perform fresh lookup if not cached
|
|
1021
1133
|
if (!whoisResult) {
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1134
|
+
// Deduplicate concurrent lookups — wait for in-flight request instead of starting a new one
|
|
1135
|
+
if (pendingWhoisLookups.has(whoisCacheKey)) {
|
|
1136
|
+
whoisResult = await pendingWhoisLookups.get(whoisCacheKey);
|
|
1137
|
+
} else {
|
|
1138
|
+
if (forceDebug) {
|
|
1139
|
+
const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
|
|
1140
|
+
logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
// Configure retry options based on site config or use defaults
|
|
1144
|
+
const retryOptions = {
|
|
1145
|
+
maxRetries: siteConfig.whois_max_retries || 3,
|
|
1146
|
+
timeoutMultiplier: siteConfig.whois_timeout_multiplier || 1.5,
|
|
1147
|
+
useFallbackServers: siteConfig.whois_use_fallback !== false, // Default true
|
|
1148
|
+
retryOnTimeout: siteConfig.whois_retry_on_timeout !== false, // Default true
|
|
1149
|
+
retryOnError: siteConfig.whois_retry_on_error === true // Default false
|
|
1150
|
+
};
|
|
1151
|
+
|
|
1152
|
+
try {
|
|
1153
|
+
const lookupPromise = whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
|
|
1154
|
+
pendingWhoisLookups.set(whoisCacheKey, lookupPromise);
|
|
1155
|
+
whoisResult = await lookupPromise;
|
|
1156
|
+
pendingWhoisLookups.delete(whoisCacheKey);
|
|
1157
|
+
|
|
1158
|
+
// Cache successful results (and certain types of failures)
|
|
1159
|
+
if (whoisResult.success ||
|
|
1160
|
+
(whoisResult.error && !whoisResult.isTimeout &&
|
|
1161
|
+
!whoisResult.error.toLowerCase().includes('connection') &&
|
|
1162
|
+
!whoisResult.error.toLowerCase().includes('network'))) {
|
|
1163
|
+
|
|
1164
|
+
globalWhoisResultCache.set(whoisCacheKey, {
|
|
1165
|
+
result: whoisResult,
|
|
1047
1166
|
timestamp: now
|
|
1048
1167
|
});
|
|
1049
|
-
|
|
1168
|
+
dnsCacheStats.whoisMisses++;
|
|
1169
|
+
dnsCacheStats.freshWhois.push(whoisRootDomain);
|
|
1170
|
+
|
|
1050
1171
|
if (forceDebug) {
|
|
1051
1172
|
const cacheType = whoisResult.success ? 'successful' : 'failed';
|
|
1052
1173
|
const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
|
|
@@ -1070,6 +1191,7 @@ function createNetToolsHandler(config) {
|
|
|
1070
1191
|
// Continue with dig if configured
|
|
1071
1192
|
whoisResult = null; // Ensure we don't process a null result
|
|
1072
1193
|
}
|
|
1194
|
+
}
|
|
1073
1195
|
}
|
|
1074
1196
|
|
|
1075
1197
|
// Process whois result (whether from cache or fresh lookup)
|
|
@@ -1235,6 +1357,7 @@ function createNetToolsHandler(config) {
|
|
|
1235
1357
|
logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Using cached result for ${digDomain} (${digRecordType}) [age: ${Math.round((now - cachedEntry.timestamp) / 1000)}s]`);
|
|
1236
1358
|
}
|
|
1237
1359
|
digResult = cachedEntry.result;
|
|
1360
|
+
dnsCacheStats.digHits++;
|
|
1238
1361
|
} else {
|
|
1239
1362
|
// Cache expired, remove it
|
|
1240
1363
|
globalDigResultCache.delete(digCacheKey);
|
|
@@ -1242,16 +1365,26 @@ function createNetToolsHandler(config) {
|
|
|
1242
1365
|
}
|
|
1243
1366
|
|
|
1244
1367
|
if (!digResult) {
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1368
|
+
// Deduplicate concurrent lookups — wait for in-flight request instead of starting a new one
|
|
1369
|
+
if (pendingDigLookups.has(digCacheKey)) {
|
|
1370
|
+
digResult = await pendingDigLookups.get(digCacheKey);
|
|
1371
|
+
} else {
|
|
1372
|
+
const lookupPromise = digLookup(digDomain, digRecordType, 5000);
|
|
1373
|
+
pendingDigLookups.set(digCacheKey, lookupPromise);
|
|
1374
|
+
digResult = await lookupPromise;
|
|
1375
|
+
pendingDigLookups.delete(digCacheKey);
|
|
1376
|
+
|
|
1377
|
+
// Cache the result for future use
|
|
1378
|
+
globalDigResultCache.set(digCacheKey, {
|
|
1379
|
+
result: digResult,
|
|
1380
|
+
timestamp: now
|
|
1381
|
+
});
|
|
1382
|
+
dnsCacheStats.digMisses++;
|
|
1383
|
+
dnsCacheStats.freshDig.push(`${digDomain} (${digRecordType})`);
|
|
1384
|
+
|
|
1385
|
+
if (forceDebug && digResult.success) {
|
|
1386
|
+
logToConsoleAndFile(`${messageColors.highlight('[dig-cache]')} Cached new result for ${digDomain} (${digRecordType})`);
|
|
1387
|
+
}
|
|
1255
1388
|
}
|
|
1256
1389
|
}
|
|
1257
1390
|
|
|
@@ -1435,5 +1568,7 @@ module.exports = {
|
|
|
1435
1568
|
selectWhoisServer,
|
|
1436
1569
|
getCommonWhoisServers,
|
|
1437
1570
|
suggestWhoisServers,
|
|
1438
|
-
execWithTimeout // Export for testing
|
|
1571
|
+
execWithTimeout, // Export for testing
|
|
1572
|
+
enableDiskCache,
|
|
1573
|
+
getDnsCacheStats
|
|
1439
1574
|
};
|
package/lib/output.js
CHANGED
|
@@ -5,8 +5,17 @@ const { getTotalDomainsSkipped } = require('./domain-cache');
|
|
|
5
5
|
const { loadComparisonRules, filterUniqueRules } = require('./compare');
|
|
6
6
|
const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
|
|
7
7
|
|
|
8
|
-
// Cache for compiled wildcard regex patterns in matchesIgnoreDomain
|
|
8
|
+
// Cache for compiled wildcard regex patterns in matchesIgnoreDomain (capped to prevent memory leak)
|
|
9
9
|
const wildcardRegexCache = new Map();
|
|
10
|
+
const WILDCARD_CACHE_MAX = 500;
|
|
11
|
+
|
|
12
|
+
// Hoisted resource type map — avoid recreating per call
|
|
13
|
+
const RESOURCE_TYPE_TO_ADBLOCK = {
|
|
14
|
+
'script': 'script', 'xhr': 'xmlhttprequest', 'fetch': 'xmlhttprequest',
|
|
15
|
+
'stylesheet': 'stylesheet', 'image': 'image', 'font': 'font',
|
|
16
|
+
'document': 'document', 'subdocument': 'subdocument', 'iframe': 'subdocument',
|
|
17
|
+
'websocket': 'websocket', 'media': 'media', 'ping': 'ping', 'other': null
|
|
18
|
+
};
|
|
10
19
|
|
|
11
20
|
/**
|
|
12
21
|
* Check if domain matches any ignore patterns (supports wildcards)
|
|
@@ -23,18 +32,9 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
|
|
|
23
32
|
if (pattern.includes('*')) {
|
|
24
33
|
// Enhanced wildcard pattern handling
|
|
25
34
|
if (pattern.startsWith('*.')) {
|
|
26
|
-
// Pattern: *.example.com
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
const domainRoot = extractDomainFromRule(`||${domain}^`) || domain;
|
|
30
|
-
|
|
31
|
-
// Use basic root domain comparison for output filtering
|
|
32
|
-
const getSimpleRoot = (d) => {
|
|
33
|
-
const parts = d.split('.');
|
|
34
|
-
return parts.length >= 2 ? parts.slice(-2).join('.') : d;
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
return getSimpleRoot(domainRoot) === getSimpleRoot(wildcardRoot);
|
|
35
|
+
// Pattern: *.example.com — match exact or any subdomain
|
|
36
|
+
const suffix = pattern.substring(2);
|
|
37
|
+
return domain === suffix || domain.endsWith('.' + suffix);
|
|
38
38
|
} else if (pattern.endsWith('.*')) {
|
|
39
39
|
// Pattern: example.*
|
|
40
40
|
const baseDomain = pattern.slice(0, -2); // Remove ".*"
|
|
@@ -42,6 +42,9 @@ function matchesIgnoreDomain(domain, ignorePatterns) {
|
|
|
42
42
|
} else {
|
|
43
43
|
// Complex wildcard pattern (cached)
|
|
44
44
|
if (!wildcardRegexCache.has(pattern)) {
|
|
45
|
+
if (wildcardRegexCache.size >= WILDCARD_CACHE_MAX) {
|
|
46
|
+
wildcardRegexCache.delete(wildcardRegexCache.keys().next().value);
|
|
47
|
+
}
|
|
45
48
|
const regexPattern = pattern
|
|
46
49
|
.replace(/\./g, '\\.') // Escape dots
|
|
47
50
|
.replace(/\*/g, '.*'); // Convert * to .*
|
|
@@ -153,23 +156,7 @@ function formatDomain(domain, options = {}) {
|
|
|
153
156
|
* @returns {string|null} Adblock filter modifier, or null if should be ignored
|
|
154
157
|
*/
|
|
155
158
|
function mapResourceTypeToAdblockModifier(resourceType) {
|
|
156
|
-
|
|
157
|
-
'script': 'script',
|
|
158
|
-
'xhr': 'xmlhttprequest',
|
|
159
|
-
'fetch': 'xmlhttprequest',
|
|
160
|
-
'stylesheet': 'stylesheet',
|
|
161
|
-
'image': 'image',
|
|
162
|
-
'font': 'font',
|
|
163
|
-
'document': 'document',
|
|
164
|
-
'subdocument': 'subdocument',
|
|
165
|
-
'iframe': 'subdocument',
|
|
166
|
-
'websocket': 'websocket',
|
|
167
|
-
'media': 'media',
|
|
168
|
-
'ping': 'ping',
|
|
169
|
-
'other': null // Ignore 'other' type - return null
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
return typeMap[resourceType] || null; // Return null for unknown types too
|
|
159
|
+
return RESOURCE_TYPE_TO_ADBLOCK[resourceType] || null;
|
|
173
160
|
}
|
|
174
161
|
|
|
175
162
|
/**
|