@fanboynz/network-scanner 2.0.48 → 2.0.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adblock.js +60 -43
- package/lib/cdp.js +15 -19
- package/lib/cloudflare.js +1 -21
- package/lib/dry-run.js +1 -1
- package/lib/fingerprint.js +80 -26
- package/lib/interaction.js +2 -28
- package/nwss.js +58 -28
- package/package.json +1 -1
package/lib/adblock.js
CHANGED
|
@@ -312,10 +312,17 @@ function parseRule(rule, isWhitelist) {
|
|
|
312
312
|
|
|
313
313
|
// Domain rules: ||domain.com^ or ||domain.com
|
|
314
314
|
if (pattern.startsWith('||')) {
|
|
315
|
-
parsed.isDomain = true;
|
|
316
315
|
const domain = pattern.substring(2).replace(/\^.*$/, '').replace(/\*$/, '');
|
|
317
|
-
|
|
318
|
-
|
|
316
|
+
const afterDomain = pattern.substring(2 + domain.length);
|
|
317
|
+
if (!afterDomain || afterDomain === '^') {
|
|
318
|
+
// Pure domain rule: ||domain.com^ or ||domain.com
|
|
319
|
+
parsed.isDomain = true;
|
|
320
|
+
parsed.domain = domain;
|
|
321
|
+
parsed.matcher = createDomainMatcher(domain);
|
|
322
|
+
} else {
|
|
323
|
+
// Domain + path rule: ||domain.com^*path or ||domain.com/path
|
|
324
|
+
parsed.matcher = createPatternMatcher(pattern);
|
|
325
|
+
}
|
|
319
326
|
}
|
|
320
327
|
// Regex rules: /pattern/
|
|
321
328
|
else if (pattern.startsWith('/') && pattern.endsWith('/')) {
|
|
@@ -339,11 +346,12 @@ function parseRule(rule, isWhitelist) {
|
|
|
339
346
|
*/
|
|
340
347
|
function createDomainMatcher(domain) {
|
|
341
348
|
const lowerDomain = domain.toLowerCase();
|
|
349
|
+
const dotDomain = '.' + lowerDomain;
|
|
342
350
|
return (url, hostname) => {
|
|
343
351
|
const lowerHostname = hostname.toLowerCase();
|
|
344
352
|
// Exact match or subdomain match
|
|
345
|
-
return lowerHostname === lowerDomain ||
|
|
346
|
-
lowerHostname.endsWith(
|
|
353
|
+
return lowerHostname === lowerDomain ||
|
|
354
|
+
lowerHostname.endsWith(dotDomain);
|
|
347
355
|
};
|
|
348
356
|
}
|
|
349
357
|
|
|
@@ -378,11 +386,11 @@ function createPatternMatcher(pattern) {
|
|
|
378
386
|
function createMatcher(rules, options = {}) {
|
|
379
387
|
const { enableLogging = false, caseSensitive = false } = options;
|
|
380
388
|
|
|
381
|
-
|
|
382
|
-
const urlCache = new URLCache(1000);
|
|
389
|
+
const urlCache = new URLCache(8000);
|
|
383
390
|
let cacheHits = 0;
|
|
384
391
|
let cacheMisses = 0;
|
|
385
392
|
const hasPartyRules = rules.thirdPartyRules.length > 0 || rules.firstPartyRules.length > 0;
|
|
393
|
+
const resultCache = new URLCache(8000); // Cache full shouldBlock results
|
|
386
394
|
|
|
387
395
|
return {
|
|
388
396
|
rules,
|
|
@@ -396,6 +404,15 @@ function createMatcher(rules, options = {}) {
|
|
|
396
404
|
*/
|
|
397
405
|
shouldBlock(url, sourceUrl = '', resourceType = '') {
|
|
398
406
|
try {
|
|
407
|
+
// Check result cache — same URL+source+type always produces same result
|
|
408
|
+
const resultKey = url + '\0' + sourceUrl + '\0' + resourceType;
|
|
409
|
+
const cachedResult = resultCache.get(resultKey);
|
|
410
|
+
if (cachedResult) {
|
|
411
|
+
cacheHits++;
|
|
412
|
+
return cachedResult;
|
|
413
|
+
}
|
|
414
|
+
cacheMisses++;
|
|
415
|
+
|
|
399
416
|
// OPTIMIZATION: Check cache first for URL parsing (60% faster)
|
|
400
417
|
let cachedData = urlCache.get(url);
|
|
401
418
|
let hostname, lowerHostname;
|
|
@@ -468,7 +485,9 @@ function createMatcher(rules, options = {}) {
|
|
|
468
485
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
469
486
|
}
|
|
470
487
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
471
|
-
|
|
488
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
489
|
+
resultCache.set(resultKey, r);
|
|
490
|
+
return r;
|
|
472
491
|
}
|
|
473
492
|
}
|
|
474
493
|
|
|
@@ -480,7 +499,9 @@ function createMatcher(rules, options = {}) {
|
|
|
480
499
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
481
500
|
}
|
|
482
501
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
483
|
-
|
|
502
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
503
|
+
resultCache.set(resultKey, r);
|
|
504
|
+
return r;
|
|
484
505
|
}
|
|
485
506
|
}
|
|
486
507
|
}
|
|
@@ -493,7 +514,9 @@ function createMatcher(rules, options = {}) {
|
|
|
493
514
|
if (enableLogging) {
|
|
494
515
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
495
516
|
}
|
|
496
|
-
|
|
517
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
518
|
+
resultCache.set(resultKey, r);
|
|
519
|
+
return r;
|
|
497
520
|
}
|
|
498
521
|
}
|
|
499
522
|
|
|
@@ -506,7 +529,9 @@ function createMatcher(rules, options = {}) {
|
|
|
506
529
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
507
530
|
}
|
|
508
531
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
509
|
-
|
|
532
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
533
|
+
resultCache.set(resultKey, r);
|
|
534
|
+
return r;
|
|
510
535
|
}
|
|
511
536
|
}
|
|
512
537
|
|
|
@@ -518,7 +543,9 @@ function createMatcher(rules, options = {}) {
|
|
|
518
543
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
519
544
|
}
|
|
520
545
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
521
|
-
|
|
546
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
547
|
+
resultCache.set(resultKey, r);
|
|
548
|
+
return r;
|
|
522
549
|
}
|
|
523
550
|
}
|
|
524
551
|
}
|
|
@@ -531,7 +558,9 @@ function createMatcher(rules, options = {}) {
|
|
|
531
558
|
if (enableLogging) {
|
|
532
559
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
533
560
|
}
|
|
534
|
-
|
|
561
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
562
|
+
resultCache.set(resultKey, r);
|
|
563
|
+
return r;
|
|
535
564
|
}
|
|
536
565
|
}
|
|
537
566
|
|
|
@@ -544,11 +573,9 @@ function createMatcher(rules, options = {}) {
|
|
|
544
573
|
if (enableLogging) {
|
|
545
574
|
console.log(`[Adblock] Blocked third-party: ${url} (${rule.raw})`);
|
|
546
575
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
reason: 'third_party_rule'
|
|
551
|
-
};
|
|
576
|
+
const r = { blocked: true, rule: rule.raw, reason: 'third_party_rule' };
|
|
577
|
+
resultCache.set(resultKey, r);
|
|
578
|
+
return r;
|
|
552
579
|
}
|
|
553
580
|
}
|
|
554
581
|
}
|
|
@@ -562,11 +589,9 @@ function createMatcher(rules, options = {}) {
|
|
|
562
589
|
if (enableLogging) {
|
|
563
590
|
console.log(`[Adblock] Blocked first-party: ${url} (${rule.raw})`);
|
|
564
591
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
reason: 'first_party_rule'
|
|
569
|
-
};
|
|
592
|
+
const r = { blocked: true, rule: rule.raw, reason: 'first_party_rule' };
|
|
593
|
+
resultCache.set(resultKey, r);
|
|
594
|
+
return r;
|
|
570
595
|
}
|
|
571
596
|
}
|
|
572
597
|
}
|
|
@@ -580,11 +605,9 @@ function createMatcher(rules, options = {}) {
|
|
|
580
605
|
if (enableLogging) {
|
|
581
606
|
console.log(`[Adblock] Blocked script: ${url} (${rule.raw})`);
|
|
582
607
|
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
reason: 'script_rule'
|
|
587
|
-
};
|
|
608
|
+
const r = { blocked: true, rule: rule.raw, reason: 'script_rule' };
|
|
609
|
+
resultCache.set(resultKey, r);
|
|
610
|
+
return r;
|
|
588
611
|
}
|
|
589
612
|
}
|
|
590
613
|
}
|
|
@@ -597,11 +620,9 @@ function createMatcher(rules, options = {}) {
|
|
|
597
620
|
if (enableLogging) {
|
|
598
621
|
console.log(`[Adblock] Blocked path: ${url} (${rule.raw})`);
|
|
599
622
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
reason: 'path_rule'
|
|
604
|
-
};
|
|
623
|
+
const r = { blocked: true, rule: rule.raw, reason: 'path_rule' };
|
|
624
|
+
resultCache.set(resultKey, r);
|
|
625
|
+
return r;
|
|
605
626
|
}
|
|
606
627
|
}
|
|
607
628
|
|
|
@@ -613,20 +634,16 @@ function createMatcher(rules, options = {}) {
|
|
|
613
634
|
if (enableLogging) {
|
|
614
635
|
console.log(`[Adblock] Blocked regex: ${url} (${rule.raw})`);
|
|
615
636
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
reason: 'regex_rule'
|
|
620
|
-
};
|
|
637
|
+
const r = { blocked: true, rule: rule.raw, reason: 'regex_rule' };
|
|
638
|
+
resultCache.set(resultKey, r);
|
|
639
|
+
return r;
|
|
621
640
|
}
|
|
622
641
|
}
|
|
623
642
|
|
|
624
643
|
// No match - allow request
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
reason: 'no_match'
|
|
629
|
-
};
|
|
644
|
+
const r = { blocked: false, rule: null, reason: 'no_match' };
|
|
645
|
+
resultCache.set(resultKey, r);
|
|
646
|
+
return r;
|
|
630
647
|
|
|
631
648
|
} catch (err) {
|
|
632
649
|
if (enableLogging) {
|
package/lib/cdp.js
CHANGED
|
@@ -175,30 +175,26 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
175
175
|
|
|
176
176
|
// Enable network domain - required for network event monitoring
|
|
177
177
|
await cdpSession.send('Network.enable');
|
|
178
|
-
|
|
178
|
+
|
|
179
|
+
// Parse current URL hostname once, reused across all request events
|
|
180
|
+
let currentHostname = 'unknown';
|
|
181
|
+
try { currentHostname = new URL(currentUrl).hostname; } catch (_) {}
|
|
182
|
+
|
|
179
183
|
// Set up network request monitoring
|
|
180
184
|
// This captures ALL network requests at the browser engine level
|
|
181
185
|
cdpSession.on('Network.requestWillBeSent', (params) => {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
const hostnameForLog = (() => {
|
|
186
|
+
if (forceDebug) {
|
|
187
|
+
const { url: requestUrl, method } = params.request;
|
|
188
|
+
const initiator = params.initiator ? params.initiator.type : 'unknown';
|
|
189
|
+
let hostnameForLog = currentHostname;
|
|
187
190
|
try {
|
|
188
|
-
const currentHostname = new URL(currentUrl).hostname;
|
|
189
191
|
const requestHostname = new URL(requestUrl).hostname;
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
} catch (_) {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
})();
|
|
197
|
-
|
|
198
|
-
// Log the request with context only if debug mode is enabled
|
|
199
|
-
if (forceDebug) {
|
|
200
|
-
console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
|
|
201
|
-
}
|
|
192
|
+
if (currentHostname !== requestHostname) {
|
|
193
|
+
hostnameForLog = `${currentHostname}?${requestHostname}`;
|
|
194
|
+
}
|
|
195
|
+
} catch (_) {}
|
|
196
|
+
console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
|
|
197
|
+
}
|
|
202
198
|
});
|
|
203
199
|
|
|
204
200
|
if (forceDebug) {
|
package/lib/cloudflare.js
CHANGED
|
@@ -368,22 +368,7 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
|
|
|
368
368
|
}
|
|
369
369
|
|
|
370
370
|
const result = await Promise.race([
|
|
371
|
-
page.evaluate(
|
|
372
|
-
// Additional runtime validation inside evaluation
|
|
373
|
-
try {
|
|
374
|
-
if (typeof window === 'undefined' || !document) {
|
|
375
|
-
throw new Error('Execution context invalid during evaluation');
|
|
376
|
-
}
|
|
377
|
-
return func();
|
|
378
|
-
} catch (evalError) {
|
|
379
|
-
// Return error info instead of throwing to avoid unhandled promise rejections
|
|
380
|
-
return {
|
|
381
|
-
__evaluation_error: true,
|
|
382
|
-
message: evalError.message,
|
|
383
|
-
type: 'evaluation_error'
|
|
384
|
-
};
|
|
385
|
-
}
|
|
386
|
-
}),
|
|
371
|
+
page.evaluate(func),
|
|
387
372
|
new Promise((_, reject) => {
|
|
388
373
|
timeoutId = setTimeout(() => reject(new Error('Page evaluation timeout')), timeout);
|
|
389
374
|
})
|
|
@@ -394,11 +379,6 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
|
|
|
394
379
|
clearTimeout(timeoutId);
|
|
395
380
|
}
|
|
396
381
|
|
|
397
|
-
// Check if evaluation returned an error
|
|
398
|
-
if (result && result.__evaluation_error) {
|
|
399
|
-
throw new Error(`Evaluation failed: ${result.message}`);
|
|
400
|
-
}
|
|
401
|
-
|
|
402
382
|
if (forceDebug && attempt > 1) {
|
|
403
383
|
console.log(formatLogMessage('cloudflare', `Page evaluation succeeded on attempt ${attempt}`));
|
|
404
384
|
}
|
package/lib/dry-run.js
CHANGED
|
@@ -174,7 +174,7 @@ function generateAdblockRule(domain, resourceType = null) {
|
|
|
174
174
|
if (!domain) return '';
|
|
175
175
|
|
|
176
176
|
if (resourceType && resourceType !== 'other') {
|
|
177
|
-
return `||${domain}
|
|
177
|
+
return `||${domain}^$${resourceType}`;
|
|
178
178
|
}
|
|
179
179
|
return `||${domain}^`;
|
|
180
180
|
}
|
package/lib/fingerprint.js
CHANGED
|
@@ -7,6 +7,22 @@
|
|
|
7
7
|
const DEFAULT_PLATFORM = 'Win32';
|
|
8
8
|
const DEFAULT_TIMEZONE = 'America/New_York';
|
|
9
9
|
|
|
10
|
+
// Deterministic random generator seeded by domain string
|
|
11
|
+
// Same domain always produces the same sequence of values
|
|
12
|
+
function seededRandom(seed) {
|
|
13
|
+
let h = 0;
|
|
14
|
+
for (let i = 0; i < seed.length; i++) {
|
|
15
|
+
h = ((h << 5) - h + seed.charCodeAt(i)) | 0;
|
|
16
|
+
}
|
|
17
|
+
return () => {
|
|
18
|
+
h = (h * 1664525 + 1013904223) | 0;
|
|
19
|
+
return (h >>> 0) / 4294967296;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Cache fingerprints per domain so reloads and multi-page visits stay consistent
|
|
24
|
+
const _fingerprintCache = new Map();
|
|
25
|
+
|
|
10
26
|
// Type-specific property spoofing functions for monomorphic optimization
|
|
11
27
|
function spoofNavigatorProperties(navigator, properties, options = {}) {
|
|
12
28
|
if (!navigator || typeof navigator !== 'object') return false;
|
|
@@ -111,8 +127,18 @@ function getRealisticScreenResolution() {
|
|
|
111
127
|
|
|
112
128
|
/**
|
|
113
129
|
* Generates randomized but realistic browser fingerprint values
|
|
130
|
+
* When domain is provided, values are deterministic per-domain (consistent across reloads)
|
|
114
131
|
*/
|
|
115
|
-
function generateRealisticFingerprint(userAgent) {
|
|
132
|
+
function generateRealisticFingerprint(userAgent, domain = '') {
|
|
133
|
+
// Return cached fingerprint if same domain visited again
|
|
134
|
+
if (domain) {
|
|
135
|
+
const cached = _fingerprintCache.get(domain);
|
|
136
|
+
if (cached) return cached;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Use seeded random for consistency, or Math.random if no domain
|
|
140
|
+
const rand = domain ? seededRandom(domain) : Math.random;
|
|
141
|
+
|
|
116
142
|
// Determine OS from user agent
|
|
117
143
|
let osType = 'windows';
|
|
118
144
|
if (userAgent.includes('Macintosh') || userAgent.includes('Mac OS X')) {
|
|
@@ -162,11 +188,11 @@ function generateRealisticFingerprint(userAgent) {
|
|
|
162
188
|
};
|
|
163
189
|
|
|
164
190
|
const profile = profiles[osType];
|
|
165
|
-
const resolution = profile.resolutions[Math.floor(
|
|
191
|
+
const resolution = profile.resolutions[Math.floor(rand() * profile.resolutions.length)];
|
|
166
192
|
|
|
167
|
-
|
|
168
|
-
deviceMemory: profile.deviceMemory[Math.floor(
|
|
169
|
-
hardwareConcurrency: profile.hardwareConcurrency[Math.floor(
|
|
193
|
+
const fingerprint = {
|
|
194
|
+
deviceMemory: profile.deviceMemory[Math.floor(rand() * profile.deviceMemory.length)],
|
|
195
|
+
hardwareConcurrency: profile.hardwareConcurrency[Math.floor(rand() * profile.hardwareConcurrency.length)],
|
|
170
196
|
screen: {
|
|
171
197
|
width: resolution.width,
|
|
172
198
|
height: resolution.height,
|
|
@@ -181,6 +207,11 @@ function generateRealisticFingerprint(userAgent) {
|
|
|
181
207
|
cookieEnabled: true,
|
|
182
208
|
doNotTrack: null // Most users don't enable DNT
|
|
183
209
|
};
|
|
210
|
+
|
|
211
|
+
// Cache for this domain
|
|
212
|
+
if (domain) _fingerprintCache.set(domain, fingerprint);
|
|
213
|
+
|
|
214
|
+
return fingerprint;
|
|
184
215
|
}
|
|
185
216
|
|
|
186
217
|
/**
|
|
@@ -1009,8 +1040,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1009
1040
|
CanvasRenderingContext2D.prototype.measureText = function(text) {
|
|
1010
1041
|
const result = originalMeasureText.call(this, text);
|
|
1011
1042
|
// Add slight noise to text measurements to prevent precise fingerprinting
|
|
1012
|
-
|
|
1013
|
-
return result
|
|
1043
|
+
const originalWidth = result.width;
|
|
1044
|
+
return Object.create(result, {
|
|
1045
|
+
width: { get: () => originalWidth + (Math.random() - 0.5) * 0.1 }
|
|
1046
|
+
});
|
|
1014
1047
|
};
|
|
1015
1048
|
|
|
1016
1049
|
// Comprehensive canvas fingerprinting protection
|
|
@@ -1018,6 +1051,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1018
1051
|
const originalGetImageData = CanvasRenderingContext2D.prototype.getImageData;
|
|
1019
1052
|
CanvasRenderingContext2D.prototype.getImageData = function(sx, sy, sw, sh) {
|
|
1020
1053
|
const imageData = originalGetImageData.call(this, sx, sy, sw, sh);
|
|
1054
|
+
// Skip noise on large canvases (>500K pixels) � too expensive for minimal fingerprint benefit
|
|
1055
|
+
if (imageData.data.length > 2000000) {
|
|
1056
|
+
return imageData;
|
|
1057
|
+
}
|
|
1021
1058
|
// Add subtle noise to pixel data
|
|
1022
1059
|
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
1023
1060
|
if (Math.random() < 0.1) { // 10% chance to modify each pixel
|
|
@@ -1090,14 +1127,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1090
1127
|
safeExecute(() => {
|
|
1091
1128
|
const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
1092
1129
|
HTMLCanvasElement.prototype.toDataURL = function(...args) {
|
|
1093
|
-
|
|
1094
|
-
if (context) {
|
|
1095
|
-
const imageData = context.getImageData(0, 0, this.width, this.height);
|
|
1096
|
-
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
1097
|
-
imageData.data[i] = imageData.data[i] + Math.floor(Math.random() * 3) - 1;
|
|
1098
|
-
}
|
|
1099
|
-
context.putImageData(imageData, 0, 0);
|
|
1100
|
-
}
|
|
1130
|
+
// Noise already applied by getImageData override
|
|
1101
1131
|
return originalToDataURL.apply(this, args);
|
|
1102
1132
|
};
|
|
1103
1133
|
}, 'canvas fingerprinting protection');
|
|
@@ -1129,7 +1159,9 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1129
1159
|
}
|
|
1130
1160
|
|
|
1131
1161
|
// Spoof mouse timing patterns to prevent behavioral fingerprinting
|
|
1162
|
+
const listenerMap = new WeakMap();
|
|
1132
1163
|
const originalAddEventListener = EventTarget.prototype.addEventListener;
|
|
1164
|
+
const originalRemoveEventListener = EventTarget.prototype.removeEventListener;
|
|
1133
1165
|
EventTarget.prototype.addEventListener = function(type, listener, options) {
|
|
1134
1166
|
if (type === 'mousemove' && typeof listener === 'function') {
|
|
1135
1167
|
const wrappedListener = function(event) {
|
|
@@ -1137,10 +1169,21 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1137
1169
|
const delay = Math.random() * 2; // 0-2ms variation
|
|
1138
1170
|
setTimeout(() => listener.call(this, event), delay);
|
|
1139
1171
|
};
|
|
1172
|
+
listenerMap.set(listener, wrappedListener);
|
|
1140
1173
|
return originalAddEventListener.call(this, type, wrappedListener, options);
|
|
1141
1174
|
}
|
|
1142
1175
|
return originalAddEventListener.call(this, type, listener, options);
|
|
1143
1176
|
};
|
|
1177
|
+
EventTarget.prototype.removeEventListener = function(type, listener, options) {
|
|
1178
|
+
if (type === 'mousemove' && typeof listener === 'function') {
|
|
1179
|
+
const wrapped = listenerMap.get(listener);
|
|
1180
|
+
if (wrapped) {
|
|
1181
|
+
listenerMap.delete(listener);
|
|
1182
|
+
return originalRemoveEventListener.call(this, type, wrapped, options);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
return originalRemoveEventListener.call(this, type, listener, options);
|
|
1186
|
+
};
|
|
1144
1187
|
|
|
1145
1188
|
// Spoof PointerEvent if available
|
|
1146
1189
|
//
|
|
@@ -1323,7 +1366,11 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1323
1366
|
return;
|
|
1324
1367
|
}
|
|
1325
1368
|
|
|
1326
|
-
|
|
1369
|
+
// Extract root domain for consistent per-site fingerprinting
|
|
1370
|
+
let siteDomain = '';
|
|
1371
|
+
try { siteDomain = new URL(currentUrl).hostname; } catch (_) {}
|
|
1372
|
+
|
|
1373
|
+
const spoof = fingerprintSetting === 'random' ? generateRealisticFingerprint(currentUserAgent, siteDomain) : {
|
|
1327
1374
|
deviceMemory: 8,
|
|
1328
1375
|
hardwareConcurrency: 4,
|
|
1329
1376
|
screen: { width: 1920, height: 1080, availWidth: 1920, availHeight: 1040, colorDepth: 24, pixelDepth: 24 },
|
|
@@ -1377,15 +1424,17 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1377
1424
|
// Platform, memory, and hardware spoofing combined for better V8 optimization
|
|
1378
1425
|
// (moved into navigatorProps above);
|
|
1379
1426
|
|
|
1427
|
+
const connectionInfo = {
|
|
1428
|
+
effectiveType: ['slow-2g', '2g', '3g', '4g'][Math.floor(Math.random() * 4)],
|
|
1429
|
+
type: Math.random() > 0.5 ? 'cellular' : 'wifi',
|
|
1430
|
+
saveData: Math.random() > 0.8,
|
|
1431
|
+
downlink: 1.5 + Math.random() * 8,
|
|
1432
|
+
rtt: 50 + Math.random() * 200
|
|
1433
|
+
};
|
|
1434
|
+
|
|
1380
1435
|
// Connection type spoofing
|
|
1381
1436
|
safeDefinePropertyLocal(navigator, 'connection', {
|
|
1382
|
-
get: () =>
|
|
1383
|
-
effectiveType: ['slow-2g', '2g', '3g', '4g'][Math.floor(Math.random() * 4)],
|
|
1384
|
-
type: Math.random() > 0.5 ? 'cellular' : 'wifi',
|
|
1385
|
-
saveData: Math.random() > 0.8,
|
|
1386
|
-
downlink: 1.5 + Math.random() * 8,
|
|
1387
|
-
rtt: 50 + Math.random() * 200
|
|
1388
|
-
})
|
|
1437
|
+
get: () => connectionInfo
|
|
1389
1438
|
});
|
|
1390
1439
|
|
|
1391
1440
|
// Screen properties spoofing
|
|
@@ -1509,7 +1558,9 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
1509
1558
|
let currentPattern = patterns[Math.floor(Math.random() * patterns.length)];
|
|
1510
1559
|
let patternChangeCounter = 0;
|
|
1511
1560
|
|
|
1512
|
-
|
|
1561
|
+
let moveTimeout;
|
|
1562
|
+
function scheduleMove() {
|
|
1563
|
+
moveTimeout = setTimeout(() => {
|
|
1513
1564
|
const now = Date.now();
|
|
1514
1565
|
const timeDelta = now - lastMoveTime;
|
|
1515
1566
|
|
|
@@ -1556,12 +1607,15 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
1556
1607
|
lastMoveTime = now;
|
|
1557
1608
|
|
|
1558
1609
|
} catch (e) {}
|
|
1559
|
-
|
|
1610
|
+
scheduleMove();
|
|
1611
|
+
}, 50 + Math.random() * 100);
|
|
1612
|
+
}
|
|
1613
|
+
scheduleMove();
|
|
1560
1614
|
|
|
1561
1615
|
// Stop after 45 seconds with gradual slowdown
|
|
1562
1616
|
setTimeout(() => {
|
|
1563
1617
|
try {
|
|
1564
|
-
|
|
1618
|
+
clearTimeout(moveTimeout);
|
|
1565
1619
|
if (debugEnabled) console.log('[fingerprint] Enhanced mouse simulation completed');
|
|
1566
1620
|
} catch (e) {}
|
|
1567
1621
|
}, 45000);
|
package/lib/interaction.js
CHANGED
|
@@ -94,7 +94,7 @@ const SCROLLING = {
|
|
|
94
94
|
DEFAULT_AMOUNT: 3, // Default number of scroll actions
|
|
95
95
|
DEFAULT_SMOOTHNESS: 5, // Default smoothness (higher = more increments)
|
|
96
96
|
SCROLL_DELTA: 200, // Pixels to scroll per action
|
|
97
|
-
PAUSE_BETWEEN:
|
|
97
|
+
PAUSE_BETWEEN: 50, // Milliseconds between scroll actions
|
|
98
98
|
SMOOTH_INCREMENT_DELAY: 20 // Milliseconds between smooth scroll increments
|
|
99
99
|
};
|
|
100
100
|
|
|
@@ -796,39 +796,13 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
|
|
|
796
796
|
|
|
797
797
|
// Validate page state before starting interaction
|
|
798
798
|
try {
|
|
799
|
-
// Optimized timeout calculation - shorter for better performance
|
|
800
|
-
const siteTimeout = options.siteTimeout || 20000; // Reduced default
|
|
801
|
-
const bodyTimeout = Math.min(Math.max(siteTimeout / 8, 2000), 5000); // 2-5 seconds (reduced)
|
|
802
|
-
|
|
803
|
-
await page.waitForSelector('body', { timeout: bodyTimeout });
|
|
804
|
-
|
|
805
|
-
// Additional check: ensure page is not closed/crashed
|
|
806
799
|
if (page.isClosed()) {
|
|
807
800
|
if (forceDebug) {
|
|
808
801
|
console.log(`[interaction] Page is closed for ${currentUrl}, skipping interaction`);
|
|
809
802
|
}
|
|
810
803
|
return;
|
|
811
804
|
}
|
|
812
|
-
|
|
813
|
-
const bodyExists = await page.$('body');
|
|
814
|
-
if (!bodyExists) {
|
|
815
|
-
if (forceDebug) {
|
|
816
|
-
console.log(`[interaction] Body element not found for ${currentUrl}, skipping interaction`);
|
|
817
|
-
}
|
|
818
|
-
return;
|
|
819
|
-
}
|
|
820
|
-
await bodyExists.dispose();
|
|
821
|
-
} catch (bodyCheckErr) {
|
|
822
|
-
if (forceDebug) {
|
|
823
|
-
console.log(`[interaction] Page not ready for interaction on ${currentUrl} (waited ${Math.min(Math.max((options.siteTimeout || 20000) / 8, 2000), 5000)}ms): ${bodyCheckErr.message}`);
|
|
824
|
-
// For very slow sites, we might want to try a minimal interaction anyway
|
|
825
|
-
if (Math.min(Math.max((options.siteTimeout || 20000) / 8, 2000), 5000) >= 4000 && !bodyCheckErr.message.includes('closed')) {
|
|
826
|
-
console.log(`[interaction] Attempting minimal mouse movement only for slow-loading ${currentUrl}`);
|
|
827
|
-
return await performMinimalInteraction(page, currentUrl, options, forceDebug);
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
return;
|
|
831
|
-
}
|
|
805
|
+
} catch { return; }
|
|
832
806
|
|
|
833
807
|
// Use cached viewport for better performance
|
|
834
808
|
const viewport = await getCachedViewport(page);
|
package/nwss.js
CHANGED
|
@@ -744,6 +744,17 @@ const globalBlockedRegexes = Array.isArray(globalBlocked)
|
|
|
744
744
|
? globalBlocked.map(pattern => new RegExp(pattern))
|
|
745
745
|
: [];
|
|
746
746
|
|
|
747
|
+
// Pre-split ignoreDomains into exact Set (O(1) lookup) and wildcard array
|
|
748
|
+
const _ignoreDomainsExact = new Set();
|
|
749
|
+
const _ignoreDomainsWildcard = [];
|
|
750
|
+
for (const pattern of ignoreDomains) {
|
|
751
|
+
if (pattern.includes('*')) {
|
|
752
|
+
_ignoreDomainsWildcard.push(pattern);
|
|
753
|
+
} else {
|
|
754
|
+
_ignoreDomainsExact.add(pattern);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
|
|
747
758
|
// Apply global configuration overrides with validation
|
|
748
759
|
// Priority: Command line args > config.json > defaults
|
|
749
760
|
const MAX_CONCURRENT_SITES = (() => {
|
|
@@ -1131,20 +1142,32 @@ function shouldBypassCacheForUrl(url, siteConfig) {
|
|
|
1131
1142
|
// Cache compiled wildcard regexes to avoid recompilation on every request
|
|
1132
1143
|
const _wildcardRegexCache = new Map();
|
|
1133
1144
|
function matchesIgnoreDomain(domain, ignorePatterns) {
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
compiled = new RegExp(`^${regexPattern}$`);
|
|
1142
|
-
_wildcardRegexCache.set(pattern, compiled);
|
|
1143
|
-
}
|
|
1144
|
-
return compiled.test(domain);
|
|
1145
|
+
// Fast path: exact match or suffix match against Set (O(n) for parts, but no regex)
|
|
1146
|
+
if (_ignoreDomainsExact.size > 0) {
|
|
1147
|
+
if (_ignoreDomainsExact.has(domain)) return true;
|
|
1148
|
+
// Check parent domains: sub.ads.example.com → ads.example.com → example.com
|
|
1149
|
+
const parts = domain.split('.');
|
|
1150
|
+
for (let i = 1; i < parts.length; i++) {
|
|
1151
|
+
if (_ignoreDomainsExact.has(parts.slice(i).join('.'))) return true;
|
|
1145
1152
|
}
|
|
1146
|
-
|
|
1147
|
-
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// Slow path: wildcard patterns only
|
|
1156
|
+
const wildcards = _ignoreDomainsWildcard;
|
|
1157
|
+
const len = wildcards.length;
|
|
1158
|
+
for (let i = 0; i < len; i++) {
|
|
1159
|
+
const pattern = wildcards[i];
|
|
1160
|
+
let compiled = _wildcardRegexCache.get(pattern);
|
|
1161
|
+
if (!compiled) {
|
|
1162
|
+
const regexPattern = pattern
|
|
1163
|
+
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
1164
|
+
.replace(/\\\*/g, '.*');
|
|
1165
|
+
compiled = new RegExp(`^${regexPattern}$`);
|
|
1166
|
+
_wildcardRegexCache.set(pattern, compiled);
|
|
1167
|
+
}
|
|
1168
|
+
if (compiled.test(domain)) return true;
|
|
1169
|
+
}
|
|
1170
|
+
return false;
|
|
1148
1171
|
}
|
|
1149
1172
|
|
|
1150
1173
|
function setupFrameHandling(page, forceDebug) {
|
|
@@ -2442,8 +2465,16 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2442
2465
|
|
|
2443
2466
|
page.on('request', request => {
|
|
2444
2467
|
const checkedUrl = request.url();
|
|
2445
|
-
|
|
2446
|
-
|
|
2468
|
+
// Parse URL once, derive all domain variants from single parse
|
|
2469
|
+
let fullSubdomain = '';
|
|
2470
|
+
let checkedRootDomain = '';
|
|
2471
|
+
try {
|
|
2472
|
+
const parsedUrl = new URL(checkedUrl);
|
|
2473
|
+
fullSubdomain = parsedUrl.hostname;
|
|
2474
|
+
const pslResult = psl.parse(fullSubdomain);
|
|
2475
|
+
checkedRootDomain = pslResult.domain || fullSubdomain;
|
|
2476
|
+
} catch (e) {}
|
|
2477
|
+
|
|
2447
2478
|
// Check against ALL first-party domains (original + all redirects)
|
|
2448
2479
|
const isFirstParty = checkedRootDomain && firstPartyDomains.has(checkedRootDomain);
|
|
2449
2480
|
|
|
@@ -2516,20 +2547,19 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2516
2547
|
}
|
|
2517
2548
|
const reqUrl = checkedUrl;
|
|
2518
2549
|
|
|
2519
|
-
const reqDomain =
|
|
2550
|
+
const reqDomain = perSiteSubDomains ? fullSubdomain : checkedRootDomain;
|
|
2520
2551
|
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
}
|
|
2552
|
+
let blockedMatchIndex = -1;
|
|
2553
|
+
for (let i = 0; i < allBlockedRegexes.length; i++) {
|
|
2554
|
+
if (allBlockedRegexes[i].test(reqUrl)) {
|
|
2555
|
+
blockedMatchIndex = i;
|
|
2556
|
+
break;
|
|
2557
|
+
}
|
|
2558
|
+
}
|
|
2559
|
+
if (blockedMatchIndex !== -1) {
|
|
2560
|
+
if (forceDebug) {
|
|
2561
|
+
const matchedPattern = allBlockedRegexes[blockedMatchIndex].source;
|
|
2562
|
+
const patternSource = blockedMatchIndex < blockedRegexes.length ? 'site' : 'global';
|
|
2533
2563
|
console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked]')}[${simplifiedCurrentUrl}] ${reqUrl} blocked by ${patternSource} pattern: ${matchedPattern}`));
|
|
2534
2564
|
|
|
2535
2565
|
// Also log to file (buffered)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.50",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|