@fanboynz/network-scanner 2.0.47 → 2.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adblock.js +80 -83
- package/lib/cdp.js +15 -19
- package/lib/dry-run.js +1 -1
- package/lib/fingerprint.js +80 -26
- package/lib/grep.js +5 -76
- package/lib/interaction.js +2 -28
- package/nwss.js +58 -28
- package/package.json +1 -1
package/lib/adblock.js
CHANGED
|
@@ -312,10 +312,17 @@ function parseRule(rule, isWhitelist) {
|
|
|
312
312
|
|
|
313
313
|
// Domain rules: ||domain.com^ or ||domain.com
|
|
314
314
|
if (pattern.startsWith('||')) {
|
|
315
|
-
parsed.isDomain = true;
|
|
316
315
|
const domain = pattern.substring(2).replace(/\^.*$/, '').replace(/\*$/, '');
|
|
317
|
-
|
|
318
|
-
|
|
316
|
+
const afterDomain = pattern.substring(2 + domain.length);
|
|
317
|
+
if (!afterDomain || afterDomain === '^') {
|
|
318
|
+
// Pure domain rule: ||domain.com^ or ||domain.com
|
|
319
|
+
parsed.isDomain = true;
|
|
320
|
+
parsed.domain = domain;
|
|
321
|
+
parsed.matcher = createDomainMatcher(domain);
|
|
322
|
+
} else {
|
|
323
|
+
// Domain + path rule: ||domain.com^*path or ||domain.com/path
|
|
324
|
+
parsed.matcher = createPatternMatcher(pattern);
|
|
325
|
+
}
|
|
319
326
|
}
|
|
320
327
|
// Regex rules: /pattern/
|
|
321
328
|
else if (pattern.startsWith('/') && pattern.endsWith('/')) {
|
|
@@ -339,11 +346,12 @@ function parseRule(rule, isWhitelist) {
|
|
|
339
346
|
*/
|
|
340
347
|
function createDomainMatcher(domain) {
|
|
341
348
|
const lowerDomain = domain.toLowerCase();
|
|
349
|
+
const dotDomain = '.' + lowerDomain;
|
|
342
350
|
return (url, hostname) => {
|
|
343
351
|
const lowerHostname = hostname.toLowerCase();
|
|
344
352
|
// Exact match or subdomain match
|
|
345
|
-
return lowerHostname === lowerDomain ||
|
|
346
|
-
lowerHostname.endsWith(
|
|
353
|
+
return lowerHostname === lowerDomain ||
|
|
354
|
+
lowerHostname.endsWith(dotDomain);
|
|
347
355
|
};
|
|
348
356
|
}
|
|
349
357
|
|
|
@@ -378,10 +386,11 @@ function createPatternMatcher(pattern) {
|
|
|
378
386
|
function createMatcher(rules, options = {}) {
|
|
379
387
|
const { enableLogging = false, caseSensitive = false } = options;
|
|
380
388
|
|
|
381
|
-
|
|
382
|
-
const urlCache = new URLCache(1000);
|
|
389
|
+
const urlCache = new URLCache(8000);
|
|
383
390
|
let cacheHits = 0;
|
|
384
391
|
let cacheMisses = 0;
|
|
392
|
+
const hasPartyRules = rules.thirdPartyRules.length > 0 || rules.firstPartyRules.length > 0;
|
|
393
|
+
const resultCache = new URLCache(8000); // Cache full shouldBlock results
|
|
385
394
|
|
|
386
395
|
return {
|
|
387
396
|
rules,
|
|
@@ -395,6 +404,15 @@ function createMatcher(rules, options = {}) {
|
|
|
395
404
|
*/
|
|
396
405
|
shouldBlock(url, sourceUrl = '', resourceType = '') {
|
|
397
406
|
try {
|
|
407
|
+
// Check result cache — same URL+source+type always produces same result
|
|
408
|
+
const resultKey = url + '\0' + sourceUrl + '\0' + resourceType;
|
|
409
|
+
const cachedResult = resultCache.get(resultKey);
|
|
410
|
+
if (cachedResult) {
|
|
411
|
+
cacheHits++;
|
|
412
|
+
return cachedResult;
|
|
413
|
+
}
|
|
414
|
+
cacheMisses++;
|
|
415
|
+
|
|
398
416
|
// OPTIMIZATION: Check cache first for URL parsing (60% faster)
|
|
399
417
|
let cachedData = urlCache.get(url);
|
|
400
418
|
let hostname, lowerHostname;
|
|
@@ -416,22 +434,21 @@ function createMatcher(rules, options = {}) {
|
|
|
416
434
|
cacheMisses++;
|
|
417
435
|
}
|
|
418
436
|
|
|
419
|
-
//
|
|
420
|
-
const hasPartyRules = rules.thirdPartyRules.length > 0 || rules.firstPartyRules.length > 0;
|
|
421
|
-
const isThirdParty = (sourceUrl && hasPartyRules)
|
|
422
|
-
? isThirdPartyRequest(url, sourceUrl)
|
|
423
|
-
: false;
|
|
424
|
-
|
|
425
|
-
// OPTIMIZATION #2: Calculate hostname parts once and reuse (avoid duplicate split operations)
|
|
437
|
+
// Calculate hostname parts once and reuse
|
|
426
438
|
const hostnameParts = lowerHostname.split('.');
|
|
439
|
+
|
|
440
|
+
// Precompute parent domains once, reused for whitelist and block checks
|
|
441
|
+
const parentDomains = [];
|
|
442
|
+
const partsLen = hostnameParts.length;
|
|
443
|
+
for (let i = 1; i < partsLen; i++) {
|
|
444
|
+
parentDomains.push(hostnameParts.slice(i).join('.'));
|
|
445
|
+
}
|
|
427
446
|
|
|
428
|
-
//
|
|
447
|
+
// Extract and cache source page domain for $domain and third-party checks
|
|
429
448
|
let sourceDomain = null;
|
|
430
|
-
let cachedSourceData = null;
|
|
431
449
|
|
|
432
450
|
if (sourceUrl) {
|
|
433
|
-
|
|
434
|
-
cachedSourceData = urlCache.get(sourceUrl);
|
|
451
|
+
const cachedSourceData = urlCache.get(sourceUrl);
|
|
435
452
|
|
|
436
453
|
if (cachedSourceData) {
|
|
437
454
|
sourceDomain = cachedSourceData.lowerHostname;
|
|
@@ -454,6 +471,11 @@ function createMatcher(rules, options = {}) {
|
|
|
454
471
|
}
|
|
455
472
|
}
|
|
456
473
|
|
|
474
|
+
// Calculate third-party status using already-parsed hostnames
|
|
475
|
+
const isThirdParty = (sourceDomain && hasPartyRules)
|
|
476
|
+
? getBaseDomain(lowerHostname) !== getBaseDomain(sourceDomain)
|
|
477
|
+
: false;
|
|
478
|
+
|
|
457
479
|
// === WHITELIST CHECK (exception rules take precedence) ===
|
|
458
480
|
|
|
459
481
|
// Fast path: Check exact domain in Map (O(1))
|
|
@@ -463,21 +485,23 @@ function createMatcher(rules, options = {}) {
|
|
|
463
485
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
464
486
|
}
|
|
465
487
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
466
|
-
|
|
488
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
489
|
+
resultCache.set(resultKey, r);
|
|
490
|
+
return r;
|
|
467
491
|
}
|
|
468
492
|
}
|
|
469
493
|
|
|
470
494
|
// Check parent domains for subdomain matches (e.g., sub.example.com -> example.com)
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const parentDomain = hostnameParts.slice(i).join('.');
|
|
474
|
-
rule = rules.whitelistMap.get(parentDomain); // V8: Single Map lookup
|
|
495
|
+
for (let i = 0; i < parentDomains.length; i++) {
|
|
496
|
+
rule = rules.whitelistMap.get(parentDomains[i]);
|
|
475
497
|
if (rule) {
|
|
476
498
|
if (enableLogging) {
|
|
477
499
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
478
500
|
}
|
|
479
501
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
480
|
-
|
|
502
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
503
|
+
resultCache.set(resultKey, r);
|
|
504
|
+
return r;
|
|
481
505
|
}
|
|
482
506
|
}
|
|
483
507
|
}
|
|
@@ -490,7 +514,9 @@ function createMatcher(rules, options = {}) {
|
|
|
490
514
|
if (enableLogging) {
|
|
491
515
|
console.log(`[Adblock] Whitelisted: ${url} (${rule.raw})`);
|
|
492
516
|
}
|
|
493
|
-
|
|
517
|
+
const r = { blocked: false, rule: rule.raw, reason: 'whitelisted' };
|
|
518
|
+
resultCache.set(resultKey, r);
|
|
519
|
+
return r;
|
|
494
520
|
}
|
|
495
521
|
}
|
|
496
522
|
|
|
@@ -503,20 +529,23 @@ function createMatcher(rules, options = {}) {
|
|
|
503
529
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
504
530
|
}
|
|
505
531
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
506
|
-
|
|
532
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
533
|
+
resultCache.set(resultKey, r);
|
|
534
|
+
return r;
|
|
507
535
|
}
|
|
508
536
|
}
|
|
509
537
|
|
|
510
538
|
// Check parent domains for subdomain matches (e.g., ads.example.com -> example.com)
|
|
511
|
-
for (let i =
|
|
512
|
-
|
|
513
|
-
rule = rules.domainMap.get(parentDomain); // V8: Single Map lookup
|
|
539
|
+
for (let i = 0; i < parentDomains.length; i++) {
|
|
540
|
+
rule = rules.domainMap.get(parentDomains[i]);
|
|
514
541
|
if (rule) {
|
|
515
542
|
if (enableLogging) {
|
|
516
543
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
517
544
|
}
|
|
518
545
|
if (matchesDomainRestrictions(rule, sourceDomain)) {
|
|
519
|
-
|
|
546
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
547
|
+
resultCache.set(resultKey, r);
|
|
548
|
+
return r;
|
|
520
549
|
}
|
|
521
550
|
}
|
|
522
551
|
}
|
|
@@ -529,7 +558,9 @@ function createMatcher(rules, options = {}) {
|
|
|
529
558
|
if (enableLogging) {
|
|
530
559
|
console.log(`[Adblock] Blocked domain: ${url} (${rule.raw})`);
|
|
531
560
|
}
|
|
532
|
-
|
|
561
|
+
const r = { blocked: true, rule: rule.raw, reason: 'domain_rule' };
|
|
562
|
+
resultCache.set(resultKey, r);
|
|
563
|
+
return r;
|
|
533
564
|
}
|
|
534
565
|
}
|
|
535
566
|
|
|
@@ -542,11 +573,9 @@ function createMatcher(rules, options = {}) {
|
|
|
542
573
|
if (enableLogging) {
|
|
543
574
|
console.log(`[Adblock] Blocked third-party: ${url} (${rule.raw})`);
|
|
544
575
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
reason: 'third_party_rule'
|
|
549
|
-
};
|
|
576
|
+
const r = { blocked: true, rule: rule.raw, reason: 'third_party_rule' };
|
|
577
|
+
resultCache.set(resultKey, r);
|
|
578
|
+
return r;
|
|
550
579
|
}
|
|
551
580
|
}
|
|
552
581
|
}
|
|
@@ -560,11 +589,9 @@ function createMatcher(rules, options = {}) {
|
|
|
560
589
|
if (enableLogging) {
|
|
561
590
|
console.log(`[Adblock] Blocked first-party: ${url} (${rule.raw})`);
|
|
562
591
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
reason: 'first_party_rule'
|
|
567
|
-
};
|
|
592
|
+
const r = { blocked: true, rule: rule.raw, reason: 'first_party_rule' };
|
|
593
|
+
resultCache.set(resultKey, r);
|
|
594
|
+
return r;
|
|
568
595
|
}
|
|
569
596
|
}
|
|
570
597
|
}
|
|
@@ -578,11 +605,9 @@ function createMatcher(rules, options = {}) {
|
|
|
578
605
|
if (enableLogging) {
|
|
579
606
|
console.log(`[Adblock] Blocked script: ${url} (${rule.raw})`);
|
|
580
607
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
reason: 'script_rule'
|
|
585
|
-
};
|
|
608
|
+
const r = { blocked: true, rule: rule.raw, reason: 'script_rule' };
|
|
609
|
+
resultCache.set(resultKey, r);
|
|
610
|
+
return r;
|
|
586
611
|
}
|
|
587
612
|
}
|
|
588
613
|
}
|
|
@@ -595,11 +620,9 @@ function createMatcher(rules, options = {}) {
|
|
|
595
620
|
if (enableLogging) {
|
|
596
621
|
console.log(`[Adblock] Blocked path: ${url} (${rule.raw})`);
|
|
597
622
|
}
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
reason: 'path_rule'
|
|
602
|
-
};
|
|
623
|
+
const r = { blocked: true, rule: rule.raw, reason: 'path_rule' };
|
|
624
|
+
resultCache.set(resultKey, r);
|
|
625
|
+
return r;
|
|
603
626
|
}
|
|
604
627
|
}
|
|
605
628
|
|
|
@@ -611,20 +634,16 @@ function createMatcher(rules, options = {}) {
|
|
|
611
634
|
if (enableLogging) {
|
|
612
635
|
console.log(`[Adblock] Blocked regex: ${url} (${rule.raw})`);
|
|
613
636
|
}
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
reason: 'regex_rule'
|
|
618
|
-
};
|
|
637
|
+
const r = { blocked: true, rule: rule.raw, reason: 'regex_rule' };
|
|
638
|
+
resultCache.set(resultKey, r);
|
|
639
|
+
return r;
|
|
619
640
|
}
|
|
620
641
|
}
|
|
621
642
|
|
|
622
643
|
// No match - allow request
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
reason: 'no_match'
|
|
627
|
-
};
|
|
644
|
+
const r = { blocked: false, rule: null, reason: 'no_match' };
|
|
645
|
+
resultCache.set(resultKey, r);
|
|
646
|
+
return r;
|
|
628
647
|
|
|
629
648
|
} catch (err) {
|
|
630
649
|
if (enableLogging) {
|
|
@@ -796,27 +815,6 @@ function matchesRule(rule, url, hostname, isThirdParty, resourceType, sourceDoma
|
|
|
796
815
|
}
|
|
797
816
|
}
|
|
798
817
|
|
|
799
|
-
/**
|
|
800
|
-
* Determine if request is third-party
|
|
801
|
-
* @param {string} requestUrl - URL being requested
|
|
802
|
-
* @param {string} sourceUrl - URL of the page making the request
|
|
803
|
-
* @returns {boolean} True if third-party request
|
|
804
|
-
*/
|
|
805
|
-
function isThirdPartyRequest(requestUrl, sourceUrl) {
|
|
806
|
-
try {
|
|
807
|
-
const requestHostname = new URL(requestUrl).hostname;
|
|
808
|
-
const sourceHostname = new URL(sourceUrl).hostname;
|
|
809
|
-
|
|
810
|
-
// Extract base domain (handle subdomains)
|
|
811
|
-
const requestDomain = getBaseDomain(requestHostname);
|
|
812
|
-
const sourceDomain = getBaseDomain(sourceHostname);
|
|
813
|
-
|
|
814
|
-
return requestDomain !== sourceDomain;
|
|
815
|
-
} catch (err) {
|
|
816
|
-
return false;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
|
|
820
818
|
/**
|
|
821
819
|
* Extract base domain from hostname
|
|
822
820
|
* @param {string} hostname - Full hostname
|
|
@@ -833,6 +831,5 @@ function getBaseDomain(hostname) {
|
|
|
833
831
|
|
|
834
832
|
module.exports = {
|
|
835
833
|
parseAdblockRules,
|
|
836
|
-
isThirdPartyRequest,
|
|
837
834
|
getBaseDomain
|
|
838
835
|
};
|
package/lib/cdp.js
CHANGED
|
@@ -175,30 +175,26 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
175
175
|
|
|
176
176
|
// Enable network domain - required for network event monitoring
|
|
177
177
|
await cdpSession.send('Network.enable');
|
|
178
|
-
|
|
178
|
+
|
|
179
|
+
// Parse current URL hostname once, reused across all request events
|
|
180
|
+
let currentHostname = 'unknown';
|
|
181
|
+
try { currentHostname = new URL(currentUrl).hostname; } catch (_) {}
|
|
182
|
+
|
|
179
183
|
// Set up network request monitoring
|
|
180
184
|
// This captures ALL network requests at the browser engine level
|
|
181
185
|
cdpSession.on('Network.requestWillBeSent', (params) => {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
const hostnameForLog = (() => {
|
|
186
|
+
if (forceDebug) {
|
|
187
|
+
const { url: requestUrl, method } = params.request;
|
|
188
|
+
const initiator = params.initiator ? params.initiator.type : 'unknown';
|
|
189
|
+
let hostnameForLog = currentHostname;
|
|
187
190
|
try {
|
|
188
|
-
const currentHostname = new URL(currentUrl).hostname;
|
|
189
191
|
const requestHostname = new URL(requestUrl).hostname;
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
} catch (_) {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
})();
|
|
197
|
-
|
|
198
|
-
// Log the request with context only if debug mode is enabled
|
|
199
|
-
if (forceDebug) {
|
|
200
|
-
console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
|
|
201
|
-
}
|
|
192
|
+
if (currentHostname !== requestHostname) {
|
|
193
|
+
hostnameForLog = `${currentHostname}?${requestHostname}`;
|
|
194
|
+
}
|
|
195
|
+
} catch (_) {}
|
|
196
|
+
console.log(formatLogMessage('debug', `[cdp][${hostnameForLog}] ${method} ${requestUrl} (initiator: ${initiator})`));
|
|
197
|
+
}
|
|
202
198
|
});
|
|
203
199
|
|
|
204
200
|
if (forceDebug) {
|
package/lib/dry-run.js
CHANGED
|
@@ -174,7 +174,7 @@ function generateAdblockRule(domain, resourceType = null) {
|
|
|
174
174
|
if (!domain) return '';
|
|
175
175
|
|
|
176
176
|
if (resourceType && resourceType !== 'other') {
|
|
177
|
-
return `||${domain}
|
|
177
|
+
return `||${domain}^$${resourceType}`;
|
|
178
178
|
}
|
|
179
179
|
return `||${domain}^`;
|
|
180
180
|
}
|
package/lib/fingerprint.js
CHANGED
|
@@ -7,6 +7,22 @@
|
|
|
7
7
|
const DEFAULT_PLATFORM = 'Win32';
|
|
8
8
|
const DEFAULT_TIMEZONE = 'America/New_York';
|
|
9
9
|
|
|
10
|
+
// Deterministic random generator seeded by domain string
|
|
11
|
+
// Same domain always produces the same sequence of values
|
|
12
|
+
function seededRandom(seed) {
|
|
13
|
+
let h = 0;
|
|
14
|
+
for (let i = 0; i < seed.length; i++) {
|
|
15
|
+
h = ((h << 5) - h + seed.charCodeAt(i)) | 0;
|
|
16
|
+
}
|
|
17
|
+
return () => {
|
|
18
|
+
h = (h * 1664525 + 1013904223) | 0;
|
|
19
|
+
return (h >>> 0) / 4294967296;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Cache fingerprints per domain so reloads and multi-page visits stay consistent
|
|
24
|
+
const _fingerprintCache = new Map();
|
|
25
|
+
|
|
10
26
|
// Type-specific property spoofing functions for monomorphic optimization
|
|
11
27
|
function spoofNavigatorProperties(navigator, properties, options = {}) {
|
|
12
28
|
if (!navigator || typeof navigator !== 'object') return false;
|
|
@@ -111,8 +127,18 @@ function getRealisticScreenResolution() {
|
|
|
111
127
|
|
|
112
128
|
/**
|
|
113
129
|
* Generates randomized but realistic browser fingerprint values
|
|
130
|
+
* When domain is provided, values are deterministic per-domain (consistent across reloads)
|
|
114
131
|
*/
|
|
115
|
-
function generateRealisticFingerprint(userAgent) {
|
|
132
|
+
function generateRealisticFingerprint(userAgent, domain = '') {
|
|
133
|
+
// Return cached fingerprint if same domain visited again
|
|
134
|
+
if (domain) {
|
|
135
|
+
const cached = _fingerprintCache.get(domain);
|
|
136
|
+
if (cached) return cached;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Use seeded random for consistency, or Math.random if no domain
|
|
140
|
+
const rand = domain ? seededRandom(domain) : Math.random;
|
|
141
|
+
|
|
116
142
|
// Determine OS from user agent
|
|
117
143
|
let osType = 'windows';
|
|
118
144
|
if (userAgent.includes('Macintosh') || userAgent.includes('Mac OS X')) {
|
|
@@ -162,11 +188,11 @@ function generateRealisticFingerprint(userAgent) {
|
|
|
162
188
|
};
|
|
163
189
|
|
|
164
190
|
const profile = profiles[osType];
|
|
165
|
-
const resolution = profile.resolutions[Math.floor(
|
|
191
|
+
const resolution = profile.resolutions[Math.floor(rand() * profile.resolutions.length)];
|
|
166
192
|
|
|
167
|
-
|
|
168
|
-
deviceMemory: profile.deviceMemory[Math.floor(
|
|
169
|
-
hardwareConcurrency: profile.hardwareConcurrency[Math.floor(
|
|
193
|
+
const fingerprint = {
|
|
194
|
+
deviceMemory: profile.deviceMemory[Math.floor(rand() * profile.deviceMemory.length)],
|
|
195
|
+
hardwareConcurrency: profile.hardwareConcurrency[Math.floor(rand() * profile.hardwareConcurrency.length)],
|
|
170
196
|
screen: {
|
|
171
197
|
width: resolution.width,
|
|
172
198
|
height: resolution.height,
|
|
@@ -181,6 +207,11 @@ function generateRealisticFingerprint(userAgent) {
|
|
|
181
207
|
cookieEnabled: true,
|
|
182
208
|
doNotTrack: null // Most users don't enable DNT
|
|
183
209
|
};
|
|
210
|
+
|
|
211
|
+
// Cache for this domain
|
|
212
|
+
if (domain) _fingerprintCache.set(domain, fingerprint);
|
|
213
|
+
|
|
214
|
+
return fingerprint;
|
|
184
215
|
}
|
|
185
216
|
|
|
186
217
|
/**
|
|
@@ -1009,8 +1040,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1009
1040
|
CanvasRenderingContext2D.prototype.measureText = function(text) {
|
|
1010
1041
|
const result = originalMeasureText.call(this, text);
|
|
1011
1042
|
// Add slight noise to text measurements to prevent precise fingerprinting
|
|
1012
|
-
|
|
1013
|
-
return result
|
|
1043
|
+
const originalWidth = result.width;
|
|
1044
|
+
return Object.create(result, {
|
|
1045
|
+
width: { get: () => originalWidth + (Math.random() - 0.5) * 0.1 }
|
|
1046
|
+
});
|
|
1014
1047
|
};
|
|
1015
1048
|
|
|
1016
1049
|
// Comprehensive canvas fingerprinting protection
|
|
@@ -1018,6 +1051,10 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1018
1051
|
const originalGetImageData = CanvasRenderingContext2D.prototype.getImageData;
|
|
1019
1052
|
CanvasRenderingContext2D.prototype.getImageData = function(sx, sy, sw, sh) {
|
|
1020
1053
|
const imageData = originalGetImageData.call(this, sx, sy, sw, sh);
|
|
1054
|
+
// Skip noise on large canvases (>500K pixels) � too expensive for minimal fingerprint benefit
|
|
1055
|
+
if (imageData.data.length > 2000000) {
|
|
1056
|
+
return imageData;
|
|
1057
|
+
}
|
|
1021
1058
|
// Add subtle noise to pixel data
|
|
1022
1059
|
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
1023
1060
|
if (Math.random() < 0.1) { // 10% chance to modify each pixel
|
|
@@ -1090,14 +1127,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1090
1127
|
safeExecute(() => {
|
|
1091
1128
|
const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
1092
1129
|
HTMLCanvasElement.prototype.toDataURL = function(...args) {
|
|
1093
|
-
|
|
1094
|
-
if (context) {
|
|
1095
|
-
const imageData = context.getImageData(0, 0, this.width, this.height);
|
|
1096
|
-
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
1097
|
-
imageData.data[i] = imageData.data[i] + Math.floor(Math.random() * 3) - 1;
|
|
1098
|
-
}
|
|
1099
|
-
context.putImageData(imageData, 0, 0);
|
|
1100
|
-
}
|
|
1130
|
+
// Noise already applied by getImageData override
|
|
1101
1131
|
return originalToDataURL.apply(this, args);
|
|
1102
1132
|
};
|
|
1103
1133
|
}, 'canvas fingerprinting protection');
|
|
@@ -1129,7 +1159,9 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1129
1159
|
}
|
|
1130
1160
|
|
|
1131
1161
|
// Spoof mouse timing patterns to prevent behavioral fingerprinting
|
|
1162
|
+
const listenerMap = new WeakMap();
|
|
1132
1163
|
const originalAddEventListener = EventTarget.prototype.addEventListener;
|
|
1164
|
+
const originalRemoveEventListener = EventTarget.prototype.removeEventListener;
|
|
1133
1165
|
EventTarget.prototype.addEventListener = function(type, listener, options) {
|
|
1134
1166
|
if (type === 'mousemove' && typeof listener === 'function') {
|
|
1135
1167
|
const wrappedListener = function(event) {
|
|
@@ -1137,10 +1169,21 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1137
1169
|
const delay = Math.random() * 2; // 0-2ms variation
|
|
1138
1170
|
setTimeout(() => listener.call(this, event), delay);
|
|
1139
1171
|
};
|
|
1172
|
+
listenerMap.set(listener, wrappedListener);
|
|
1140
1173
|
return originalAddEventListener.call(this, type, wrappedListener, options);
|
|
1141
1174
|
}
|
|
1142
1175
|
return originalAddEventListener.call(this, type, listener, options);
|
|
1143
1176
|
};
|
|
1177
|
+
EventTarget.prototype.removeEventListener = function(type, listener, options) {
|
|
1178
|
+
if (type === 'mousemove' && typeof listener === 'function') {
|
|
1179
|
+
const wrapped = listenerMap.get(listener);
|
|
1180
|
+
if (wrapped) {
|
|
1181
|
+
listenerMap.delete(listener);
|
|
1182
|
+
return originalRemoveEventListener.call(this, type, wrapped, options);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
return originalRemoveEventListener.call(this, type, listener, options);
|
|
1186
|
+
};
|
|
1144
1187
|
|
|
1145
1188
|
// Spoof PointerEvent if available
|
|
1146
1189
|
//
|
|
@@ -1323,7 +1366,11 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1323
1366
|
return;
|
|
1324
1367
|
}
|
|
1325
1368
|
|
|
1326
|
-
|
|
1369
|
+
// Extract root domain for consistent per-site fingerprinting
|
|
1370
|
+
let siteDomain = '';
|
|
1371
|
+
try { siteDomain = new URL(currentUrl).hostname; } catch (_) {}
|
|
1372
|
+
|
|
1373
|
+
const spoof = fingerprintSetting === 'random' ? generateRealisticFingerprint(currentUserAgent, siteDomain) : {
|
|
1327
1374
|
deviceMemory: 8,
|
|
1328
1375
|
hardwareConcurrency: 4,
|
|
1329
1376
|
screen: { width: 1920, height: 1080, availWidth: 1920, availHeight: 1040, colorDepth: 24, pixelDepth: 24 },
|
|
@@ -1377,15 +1424,17 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1377
1424
|
// Platform, memory, and hardware spoofing combined for better V8 optimization
|
|
1378
1425
|
// (moved into navigatorProps above);
|
|
1379
1426
|
|
|
1427
|
+
const connectionInfo = {
|
|
1428
|
+
effectiveType: ['slow-2g', '2g', '3g', '4g'][Math.floor(Math.random() * 4)],
|
|
1429
|
+
type: Math.random() > 0.5 ? 'cellular' : 'wifi',
|
|
1430
|
+
saveData: Math.random() > 0.8,
|
|
1431
|
+
downlink: 1.5 + Math.random() * 8,
|
|
1432
|
+
rtt: 50 + Math.random() * 200
|
|
1433
|
+
};
|
|
1434
|
+
|
|
1380
1435
|
// Connection type spoofing
|
|
1381
1436
|
safeDefinePropertyLocal(navigator, 'connection', {
|
|
1382
|
-
get: () =>
|
|
1383
|
-
effectiveType: ['slow-2g', '2g', '3g', '4g'][Math.floor(Math.random() * 4)],
|
|
1384
|
-
type: Math.random() > 0.5 ? 'cellular' : 'wifi',
|
|
1385
|
-
saveData: Math.random() > 0.8,
|
|
1386
|
-
downlink: 1.5 + Math.random() * 8,
|
|
1387
|
-
rtt: 50 + Math.random() * 200
|
|
1388
|
-
})
|
|
1437
|
+
get: () => connectionInfo
|
|
1389
1438
|
});
|
|
1390
1439
|
|
|
1391
1440
|
// Screen properties spoofing
|
|
@@ -1509,7 +1558,9 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
1509
1558
|
let currentPattern = patterns[Math.floor(Math.random() * patterns.length)];
|
|
1510
1559
|
let patternChangeCounter = 0;
|
|
1511
1560
|
|
|
1512
|
-
|
|
1561
|
+
let moveTimeout;
|
|
1562
|
+
function scheduleMove() {
|
|
1563
|
+
moveTimeout = setTimeout(() => {
|
|
1513
1564
|
const now = Date.now();
|
|
1514
1565
|
const timeDelta = now - lastMoveTime;
|
|
1515
1566
|
|
|
@@ -1556,12 +1607,15 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
1556
1607
|
lastMoveTime = now;
|
|
1557
1608
|
|
|
1558
1609
|
} catch (e) {}
|
|
1559
|
-
|
|
1610
|
+
scheduleMove();
|
|
1611
|
+
}, 50 + Math.random() * 100);
|
|
1612
|
+
}
|
|
1613
|
+
scheduleMove();
|
|
1560
1614
|
|
|
1561
1615
|
// Stop after 45 seconds with gradual slowdown
|
|
1562
1616
|
setTimeout(() => {
|
|
1563
1617
|
try {
|
|
1564
|
-
|
|
1618
|
+
clearTimeout(moveTimeout);
|
|
1565
1619
|
if (debugEnabled) console.log('[fingerprint] Enhanced mouse simulation completed');
|
|
1566
1620
|
} catch (e) {}
|
|
1567
1621
|
}, 45000);
|
package/lib/grep.js
CHANGED
|
@@ -3,9 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const { spawnSync } = require('child_process');
|
|
6
|
-
const crypto = require('crypto');
|
|
7
|
-
const path = require('path');
|
|
8
|
-
const os = require('os');
|
|
9
6
|
const { colorize, colors, messageColors, tags, formatLogMessage } = require('./colorize');
|
|
10
7
|
|
|
11
8
|
// === Constants ===
|
|
@@ -21,53 +18,9 @@ const GREP_DEFAULTS = {
|
|
|
21
18
|
GREP_SUCCESS_STATUS: 0,
|
|
22
19
|
GREP_NOT_FOUND_STATUS: 1,
|
|
23
20
|
CURL_SUCCESS_STATUS: 0,
|
|
24
|
-
VERSION_LINE_INDEX: 0
|
|
25
|
-
RANDOM_STRING_LENGTH: 9,
|
|
26
|
-
TEMP_DIR_PREFIX: 'grep_search_'
|
|
21
|
+
VERSION_LINE_INDEX: 0
|
|
27
22
|
};
|
|
28
23
|
|
|
29
|
-
/**
|
|
30
|
-
* Creates a temporary directory and file with content for grep processing
|
|
31
|
-
* Uses mkdtempSync to avoid race conditions from filename collisions
|
|
32
|
-
* @param {string} content - The content to write to temp file
|
|
33
|
-
* @param {string} prefix - Prefix for temp filename
|
|
34
|
-
* @returns {object} Object containing tempDir and tempFile paths
|
|
35
|
-
*/
|
|
36
|
-
function createTempFile(content, prefix = 'scanner_grep') {
|
|
37
|
-
const tempDir = os.tmpdir();
|
|
38
|
-
|
|
39
|
-
// Create a unique temporary directory to avoid race conditions
|
|
40
|
-
const uniqueTempDir = fs.mkdtempSync(path.join(tempDir, GREP_DEFAULTS.TEMP_DIR_PREFIX));
|
|
41
|
-
|
|
42
|
-
// Use cryptographically secure random ID for additional uniqueness
|
|
43
|
-
const uniqueId = crypto.randomBytes(8).toString('hex');
|
|
44
|
-
const tempFile = path.join(uniqueTempDir, `${prefix}_${uniqueId}.tmp`);
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
// Write atomically with error handling
|
|
48
|
-
fs.writeFileSync(tempFile, content, {
|
|
49
|
-
encoding: 'utf8',
|
|
50
|
-
mode: 0o600 // Restrict permissions for security
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
return { tempDir: uniqueTempDir, tempFile };
|
|
54
|
-
} catch (error) {
|
|
55
|
-
// Clean up temp directory on write failure
|
|
56
|
-
try {
|
|
57
|
-
if (fs.existsSync(tempFile)) {
|
|
58
|
-
fs.unlinkSync(tempFile);
|
|
59
|
-
}
|
|
60
|
-
if (fs.existsSync(uniqueTempDir)) {
|
|
61
|
-
fs.rmdirSync(uniqueTempDir);
|
|
62
|
-
}
|
|
63
|
-
} catch (cleanupErr) {
|
|
64
|
-
// Ignore cleanup errors, report original error
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
throw new Error(`Failed to create temp file: ${error.message}`);
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
24
|
/**
|
|
72
25
|
* Searches content using grep with the provided patterns
|
|
73
26
|
* @param {string} content - The content to search
|
|
@@ -86,15 +39,8 @@ async function grepContent(content, searchPatterns, options = {}) {
|
|
|
86
39
|
if (!content || searchPatterns.length === 0) {
|
|
87
40
|
return { found: false, matchedPattern: null, allMatches: [] };
|
|
88
41
|
}
|
|
89
|
-
|
|
90
|
-
let tempFile = null;
|
|
91
|
-
let tempDir = null;
|
|
92
|
-
|
|
42
|
+
|
|
93
43
|
try {
|
|
94
|
-
// Create temporary directory and file with content
|
|
95
|
-
const tempResult = createTempFile(content, 'grep_search');
|
|
96
|
-
tempDir = tempResult.tempDir;
|
|
97
|
-
tempFile = tempResult.tempFile;
|
|
98
44
|
|
|
99
45
|
const allMatches = [];
|
|
100
46
|
let firstMatch = null;
|
|
@@ -111,12 +57,12 @@ async function grepContent(content, searchPatterns, options = {}) {
|
|
|
111
57
|
if (wholeWord) grepArgs.push('-w');
|
|
112
58
|
if (!regex) grepArgs.push('-F'); // Fixed strings (literal)
|
|
113
59
|
|
|
114
|
-
|
|
115
|
-
grepArgs.push(pattern, tempFile);
|
|
60
|
+
grepArgs.push(pattern);
|
|
116
61
|
|
|
117
62
|
try {
|
|
118
63
|
const result = spawnSync('grep', grepArgs, {
|
|
119
64
|
encoding: 'utf8',
|
|
65
|
+
input: content,
|
|
120
66
|
timeout: GREP_DEFAULTS.GREP_TIMEOUT,
|
|
121
67
|
maxBuffer: GREP_DEFAULTS.MAX_BUFFER_SIZE
|
|
122
68
|
});
|
|
@@ -147,22 +93,6 @@ async function grepContent(content, searchPatterns, options = {}) {
|
|
|
147
93
|
|
|
148
94
|
} catch (error) {
|
|
149
95
|
throw new Error(`Grep search failed: ${error.message}`);
|
|
150
|
-
} finally {
|
|
151
|
-
// Clean up temporary file and directory
|
|
152
|
-
if (tempFile) {
|
|
153
|
-
try {
|
|
154
|
-
fs.unlinkSync(tempFile);
|
|
155
|
-
} catch (cleanupErr) {
|
|
156
|
-
console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp file ${tempFile}: ${cleanupErr.message}`));
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
if (tempDir) {
|
|
160
|
-
try {
|
|
161
|
-
fs.rmdirSync(tempDir);
|
|
162
|
-
} catch (cleanupErr) {
|
|
163
|
-
console.warn(formatLogMessage('warn', `[grep] Failed to cleanup temp directory ${tempDir}: ${cleanupErr.message}`));
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
96
|
}
|
|
167
97
|
}
|
|
168
98
|
|
|
@@ -418,6 +348,5 @@ module.exports = {
|
|
|
418
348
|
grepContent,
|
|
419
349
|
downloadAndGrep,
|
|
420
350
|
createGrepHandler,
|
|
421
|
-
validateGrepAvailability
|
|
422
|
-
createTempFile
|
|
351
|
+
validateGrepAvailability
|
|
423
352
|
};
|
package/lib/interaction.js
CHANGED
|
@@ -94,7 +94,7 @@ const SCROLLING = {
|
|
|
94
94
|
DEFAULT_AMOUNT: 3, // Default number of scroll actions
|
|
95
95
|
DEFAULT_SMOOTHNESS: 5, // Default smoothness (higher = more increments)
|
|
96
96
|
SCROLL_DELTA: 200, // Pixels to scroll per action
|
|
97
|
-
PAUSE_BETWEEN:
|
|
97
|
+
PAUSE_BETWEEN: 50, // Milliseconds between scroll actions
|
|
98
98
|
SMOOTH_INCREMENT_DELAY: 20 // Milliseconds between smooth scroll increments
|
|
99
99
|
};
|
|
100
100
|
|
|
@@ -796,39 +796,13 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
|
|
|
796
796
|
|
|
797
797
|
// Validate page state before starting interaction
|
|
798
798
|
try {
|
|
799
|
-
// Optimized timeout calculation - shorter for better performance
|
|
800
|
-
const siteTimeout = options.siteTimeout || 20000; // Reduced default
|
|
801
|
-
const bodyTimeout = Math.min(Math.max(siteTimeout / 8, 2000), 5000); // 2-5 seconds (reduced)
|
|
802
|
-
|
|
803
|
-
await page.waitForSelector('body', { timeout: bodyTimeout });
|
|
804
|
-
|
|
805
|
-
// Additional check: ensure page is not closed/crashed
|
|
806
799
|
if (page.isClosed()) {
|
|
807
800
|
if (forceDebug) {
|
|
808
801
|
console.log(`[interaction] Page is closed for ${currentUrl}, skipping interaction`);
|
|
809
802
|
}
|
|
810
803
|
return;
|
|
811
804
|
}
|
|
812
|
-
|
|
813
|
-
const bodyExists = await page.$('body');
|
|
814
|
-
if (!bodyExists) {
|
|
815
|
-
if (forceDebug) {
|
|
816
|
-
console.log(`[interaction] Body element not found for ${currentUrl}, skipping interaction`);
|
|
817
|
-
}
|
|
818
|
-
return;
|
|
819
|
-
}
|
|
820
|
-
await bodyExists.dispose();
|
|
821
|
-
} catch (bodyCheckErr) {
|
|
822
|
-
if (forceDebug) {
|
|
823
|
-
console.log(`[interaction] Page not ready for interaction on ${currentUrl} (waited ${Math.min(Math.max((options.siteTimeout || 20000) / 8, 2000), 5000)}ms): ${bodyCheckErr.message}`);
|
|
824
|
-
// For very slow sites, we might want to try a minimal interaction anyway
|
|
825
|
-
if (Math.min(Math.max((options.siteTimeout || 20000) / 8, 2000), 5000) >= 4000 && !bodyCheckErr.message.includes('closed')) {
|
|
826
|
-
console.log(`[interaction] Attempting minimal mouse movement only for slow-loading ${currentUrl}`);
|
|
827
|
-
return await performMinimalInteraction(page, currentUrl, options, forceDebug);
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
return;
|
|
831
|
-
}
|
|
805
|
+
} catch { return; }
|
|
832
806
|
|
|
833
807
|
// Use cached viewport for better performance
|
|
834
808
|
const viewport = await getCachedViewport(page);
|
package/nwss.js
CHANGED
|
@@ -744,6 +744,17 @@ const globalBlockedRegexes = Array.isArray(globalBlocked)
|
|
|
744
744
|
? globalBlocked.map(pattern => new RegExp(pattern))
|
|
745
745
|
: [];
|
|
746
746
|
|
|
747
|
+
// Pre-split ignoreDomains into exact Set (O(1) lookup) and wildcard array
|
|
748
|
+
const _ignoreDomainsExact = new Set();
|
|
749
|
+
const _ignoreDomainsWildcard = [];
|
|
750
|
+
for (const pattern of ignoreDomains) {
|
|
751
|
+
if (pattern.includes('*')) {
|
|
752
|
+
_ignoreDomainsWildcard.push(pattern);
|
|
753
|
+
} else {
|
|
754
|
+
_ignoreDomainsExact.add(pattern);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
|
|
747
758
|
// Apply global configuration overrides with validation
|
|
748
759
|
// Priority: Command line args > config.json > defaults
|
|
749
760
|
const MAX_CONCURRENT_SITES = (() => {
|
|
@@ -1131,20 +1142,32 @@ function shouldBypassCacheForUrl(url, siteConfig) {
|
|
|
1131
1142
|
// Cache compiled wildcard regexes to avoid recompilation on every request
|
|
1132
1143
|
const _wildcardRegexCache = new Map();
|
|
1133
1144
|
function matchesIgnoreDomain(domain, ignorePatterns) {
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
compiled = new RegExp(`^${regexPattern}$`);
|
|
1142
|
-
_wildcardRegexCache.set(pattern, compiled);
|
|
1143
|
-
}
|
|
1144
|
-
return compiled.test(domain);
|
|
1145
|
+
// Fast path: exact match or suffix match against Set (O(n) for parts, but no regex)
|
|
1146
|
+
if (_ignoreDomainsExact.size > 0) {
|
|
1147
|
+
if (_ignoreDomainsExact.has(domain)) return true;
|
|
1148
|
+
// Check parent domains: sub.ads.example.com → ads.example.com → example.com
|
|
1149
|
+
const parts = domain.split('.');
|
|
1150
|
+
for (let i = 1; i < parts.length; i++) {
|
|
1151
|
+
if (_ignoreDomainsExact.has(parts.slice(i).join('.'))) return true;
|
|
1145
1152
|
}
|
|
1146
|
-
|
|
1147
|
-
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// Slow path: wildcard patterns only
|
|
1156
|
+
const wildcards = _ignoreDomainsWildcard;
|
|
1157
|
+
const len = wildcards.length;
|
|
1158
|
+
for (let i = 0; i < len; i++) {
|
|
1159
|
+
const pattern = wildcards[i];
|
|
1160
|
+
let compiled = _wildcardRegexCache.get(pattern);
|
|
1161
|
+
if (!compiled) {
|
|
1162
|
+
const regexPattern = pattern
|
|
1163
|
+
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
1164
|
+
.replace(/\\\*/g, '.*');
|
|
1165
|
+
compiled = new RegExp(`^${regexPattern}$`);
|
|
1166
|
+
_wildcardRegexCache.set(pattern, compiled);
|
|
1167
|
+
}
|
|
1168
|
+
if (compiled.test(domain)) return true;
|
|
1169
|
+
}
|
|
1170
|
+
return false;
|
|
1148
1171
|
}
|
|
1149
1172
|
|
|
1150
1173
|
function setupFrameHandling(page, forceDebug) {
|
|
@@ -2442,8 +2465,16 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2442
2465
|
|
|
2443
2466
|
page.on('request', request => {
|
|
2444
2467
|
const checkedUrl = request.url();
|
|
2445
|
-
|
|
2446
|
-
|
|
2468
|
+
// Parse URL once, derive all domain variants from single parse
|
|
2469
|
+
let fullSubdomain = '';
|
|
2470
|
+
let checkedRootDomain = '';
|
|
2471
|
+
try {
|
|
2472
|
+
const parsedUrl = new URL(checkedUrl);
|
|
2473
|
+
fullSubdomain = parsedUrl.hostname;
|
|
2474
|
+
const pslResult = psl.parse(fullSubdomain);
|
|
2475
|
+
checkedRootDomain = pslResult.domain || fullSubdomain;
|
|
2476
|
+
} catch (e) {}
|
|
2477
|
+
|
|
2447
2478
|
// Check against ALL first-party domains (original + all redirects)
|
|
2448
2479
|
const isFirstParty = checkedRootDomain && firstPartyDomains.has(checkedRootDomain);
|
|
2449
2480
|
|
|
@@ -2516,20 +2547,19 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2516
2547
|
}
|
|
2517
2548
|
const reqUrl = checkedUrl;
|
|
2518
2549
|
|
|
2519
|
-
const reqDomain =
|
|
2550
|
+
const reqDomain = perSiteSubDomains ? fullSubdomain : checkedRootDomain;
|
|
2520
2551
|
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
}
|
|
2552
|
+
let blockedMatchIndex = -1;
|
|
2553
|
+
for (let i = 0; i < allBlockedRegexes.length; i++) {
|
|
2554
|
+
if (allBlockedRegexes[i].test(reqUrl)) {
|
|
2555
|
+
blockedMatchIndex = i;
|
|
2556
|
+
break;
|
|
2557
|
+
}
|
|
2558
|
+
}
|
|
2559
|
+
if (blockedMatchIndex !== -1) {
|
|
2560
|
+
if (forceDebug) {
|
|
2561
|
+
const matchedPattern = allBlockedRegexes[blockedMatchIndex].source;
|
|
2562
|
+
const patternSource = blockedMatchIndex < blockedRegexes.length ? 'site' : 'global';
|
|
2533
2563
|
console.log(formatLogMessage('debug', `${messageColors.blocked('[blocked]')}[${simplifiedCurrentUrl}] ${reqUrl} blocked by ${patternSource} pattern: ${matchedPattern}`));
|
|
2534
2564
|
|
|
2535
2565
|
// Also log to file (buffered)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.49",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|