@fanboynz/network-scanner 1.0.70 → 1.0.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cloudflare.js +314 -34
- package/nwss.js +52 -6
- package/package.json +1 -1
- package/regex-tool/index.html +64 -15
package/lib/cloudflare.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Cloudflare bypass and challenge handling module - Optimized with smart detection and adaptive timeouts
|
|
3
|
-
* Version: 2.
|
|
3
|
+
* Version: 2.2.0 - Enhanced with retry logic, caching, and improved error handling
|
|
4
4
|
* Handles phishing warnings, Turnstile challenges, and modern Cloudflare protections
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Module version information
|
|
9
9
|
*/
|
|
10
|
-
const CLOUDFLARE_MODULE_VERSION = '2.
|
|
10
|
+
const CLOUDFLARE_MODULE_VERSION = '2.2.0';
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Timeout constants for various operations (in milliseconds)
|
|
@@ -29,7 +29,12 @@ const TIMEOUTS = {
|
|
|
29
29
|
ADAPTIVE_TIMEOUT_WITH_INDICATORS: 25000, // Adaptive timeout when indicators found + explicit config
|
|
30
30
|
ADAPTIVE_TIMEOUT_WITHOUT_INDICATORS: 20000, // Adaptive timeout with explicit config only
|
|
31
31
|
ADAPTIVE_TIMEOUT_AUTO_WITH_INDICATORS: 15000, // Adaptive timeout for auto-detected with indicators
|
|
32
|
-
ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000 // Adaptive timeout for auto-detected without indicators
|
|
32
|
+
ADAPTIVE_TIMEOUT_AUTO_WITHOUT_INDICATORS: 10000, // Adaptive timeout for auto-detected without indicators
|
|
33
|
+
// New timeouts for enhanced functionality
|
|
34
|
+
RETRY_DELAY: 1000, // Delay between retry attempts
|
|
35
|
+
MAX_RETRIES: 3, // Maximum retry attempts for operations
|
|
36
|
+
CHALLENGE_POLL_INTERVAL: 500, // Interval for polling challenge completion
|
|
37
|
+
CHALLENGE_MAX_POLLS: 20 // Maximum polling attempts
|
|
33
38
|
};
|
|
34
39
|
|
|
35
40
|
// Fast timeout constants - optimized for speed
|
|
@@ -45,6 +50,101 @@ const FAST_TIMEOUTS = {
|
|
|
45
50
|
CHALLENGE_COMPLETION: 3000 // Fast completion check
|
|
46
51
|
};
|
|
47
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Error categories for better handling
|
|
55
|
+
*/
|
|
56
|
+
const ERROR_TYPES = {
|
|
57
|
+
NETWORK: 'network',
|
|
58
|
+
TIMEOUT: 'timeout',
|
|
59
|
+
ELEMENT_NOT_FOUND: 'element_not_found',
|
|
60
|
+
EVALUATION_FAILED: 'evaluation_failed',
|
|
61
|
+
NAVIGATION_FAILED: 'navigation_failed',
|
|
62
|
+
UNKNOWN: 'unknown'
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Retry configuration with exponential backoff
|
|
67
|
+
*/
|
|
68
|
+
const RETRY_CONFIG = {
|
|
69
|
+
maxAttempts: 3,
|
|
70
|
+
baseDelay: 1000,
|
|
71
|
+
maxDelay: 8000,
|
|
72
|
+
backoffMultiplier: 2,
|
|
73
|
+
retryableErrors: [ERROR_TYPES.NETWORK, ERROR_TYPES.TIMEOUT, ERROR_TYPES.ELEMENT_NOT_FOUND]
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Performance cache for detection results
|
|
78
|
+
* Stores detection results per domain to avoid redundant checks
|
|
79
|
+
*/
|
|
80
|
+
class CloudflareDetectionCache {
|
|
81
|
+
constructor(ttl = 300000) { // 5 minutes TTL by default
|
|
82
|
+
this.cache = new Map();
|
|
83
|
+
this.ttl = ttl;
|
|
84
|
+
this.hits = 0;
|
|
85
|
+
this.misses = 0;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
getCacheKey(url) {
|
|
89
|
+
try {
|
|
90
|
+
const urlObj = new URL(url);
|
|
91
|
+
return `${urlObj.hostname}${urlObj.pathname}`;
|
|
92
|
+
} catch {
|
|
93
|
+
return url;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
get(url) {
|
|
98
|
+
const key = this.getCacheKey(url);
|
|
99
|
+
const cached = this.cache.get(key);
|
|
100
|
+
|
|
101
|
+
if (cached && Date.now() - cached.timestamp < this.ttl) {
|
|
102
|
+
this.hits++;
|
|
103
|
+
return cached.data;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (cached) {
|
|
107
|
+
this.cache.delete(key); // Remove expired entry
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
this.misses++;
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
set(url, data) {
|
|
115
|
+
const key = this.getCacheKey(url);
|
|
116
|
+
this.cache.set(key, {
|
|
117
|
+
data,
|
|
118
|
+
timestamp: Date.now()
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// Prevent cache from growing too large
|
|
122
|
+
if (this.cache.size > 1000) {
|
|
123
|
+
const firstKey = this.cache.keys().next().value;
|
|
124
|
+
this.cache.delete(firstKey);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
clear() {
|
|
129
|
+
this.cache.clear();
|
|
130
|
+
this.hits = 0;
|
|
131
|
+
this.misses = 0;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
getStats() {
|
|
135
|
+
const total = this.hits + this.misses;
|
|
136
|
+
return {
|
|
137
|
+
hits: this.hits,
|
|
138
|
+
misses: this.misses,
|
|
139
|
+
hitRate: total > 0 ? (this.hits / total * 100).toFixed(2) + '%' : '0%',
|
|
140
|
+
size: this.cache.size
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Initialize cache singleton
|
|
146
|
+
const detectionCache = new CloudflareDetectionCache();
|
|
147
|
+
|
|
48
148
|
/**
|
|
49
149
|
* Gets module version information
|
|
50
150
|
* @returns {object} Version information object
|
|
@@ -108,27 +208,96 @@ async function waitForTimeout(page, timeout) {
|
|
|
108
208
|
}
|
|
109
209
|
|
|
110
210
|
/**
|
|
111
|
-
*
|
|
211
|
+
* Categorizes errors for better handling
|
|
112
212
|
*/
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
setTimeout(() => reject(new Error('Page evaluation timeout')), timeout)
|
|
119
|
-
)
|
|
120
|
-
]);
|
|
121
|
-
} catch (error) {
|
|
122
|
-
console.warn(`[cloudflare] Page evaluation failed: ${error.message}`);
|
|
123
|
-
return {
|
|
124
|
-
isChallengePresent: false,
|
|
125
|
-
isPhishingWarning: false,
|
|
126
|
-
isTurnstile: false,
|
|
127
|
-
isJSChallenge: false,
|
|
128
|
-
isChallengeCompleted: false,
|
|
129
|
-
error: error.message
|
|
130
|
-
};
|
|
213
|
+
function categorizeError(error) {
|
|
214
|
+
const errorMessage = error.message || '';
|
|
215
|
+
|
|
216
|
+
if (errorMessage.includes('timeout') || errorMessage.includes('Timeout')) {
|
|
217
|
+
return ERROR_TYPES.TIMEOUT;
|
|
131
218
|
}
|
|
219
|
+
if (errorMessage.includes('Protocol error') || errorMessage.includes('Target closed')) {
|
|
220
|
+
return ERROR_TYPES.NETWORK;
|
|
221
|
+
}
|
|
222
|
+
if (errorMessage.includes('evaluation') || errorMessage.includes('Evaluation')) {
|
|
223
|
+
return ERROR_TYPES.EVALUATION_FAILED;
|
|
224
|
+
}
|
|
225
|
+
if (errorMessage.includes('navigation') || errorMessage.includes('Navigation')) {
|
|
226
|
+
return ERROR_TYPES.NAVIGATION_FAILED;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
return ERROR_TYPES.UNKNOWN;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Implements exponential backoff delay
|
|
234
|
+
*/
|
|
235
|
+
async function getRetryDelay(attempt) {
|
|
236
|
+
const delay = Math.min(
|
|
237
|
+
RETRY_CONFIG.baseDelay * Math.pow(RETRY_CONFIG.backoffMultiplier, attempt - 1),
|
|
238
|
+
RETRY_CONFIG.maxDelay
|
|
239
|
+
);
|
|
240
|
+
return new Promise(resolve => setTimeout(resolve, delay));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Enhanced safe page evaluation with retry logic and better error handling
|
|
245
|
+
*/
|
|
246
|
+
async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE, options = {}) {
|
|
247
|
+
const { maxRetries = RETRY_CONFIG.maxAttempts, forceDebug = false } = options;
|
|
248
|
+
let lastError = null;
|
|
249
|
+
|
|
250
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
251
|
+
try {
|
|
252
|
+
const result = await Promise.race([
|
|
253
|
+
page.evaluate(func),
|
|
254
|
+
new Promise((_, reject) =>
|
|
255
|
+
setTimeout(() => reject(new Error('Page evaluation timeout')), timeout)
|
|
256
|
+
)
|
|
257
|
+
]);
|
|
258
|
+
|
|
259
|
+
if (forceDebug && attempt > 1) {
|
|
260
|
+
console.log(`[cloudflare] Page evaluation succeeded on attempt ${attempt}`);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
return result;
|
|
264
|
+
} catch (error) {
|
|
265
|
+
lastError = error;
|
|
266
|
+
const errorType = categorizeError(error);
|
|
267
|
+
|
|
268
|
+
if (forceDebug) {
|
|
269
|
+
console.warn(`[cloudflare] Page evaluation failed (attempt ${attempt}/${maxRetries}): ${error.message} [${errorType}]`);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Don't retry if error type is not retryable or if it's the last attempt
|
|
273
|
+
if (!RETRY_CONFIG.retryableErrors.includes(errorType) || attempt === maxRetries) {
|
|
274
|
+
return {
|
|
275
|
+
isChallengePresent: false,
|
|
276
|
+
isPhishingWarning: false,
|
|
277
|
+
isTurnstile: false,
|
|
278
|
+
isJSChallenge: false,
|
|
279
|
+
isChallengeCompleted: false,
|
|
280
|
+
error: error.message,
|
|
281
|
+
errorType: errorType,
|
|
282
|
+
attempts: attempt
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// Wait before retrying with exponential backoff
|
|
287
|
+
await getRetryDelay(attempt);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return {
|
|
292
|
+
isChallengePresent: false,
|
|
293
|
+
isPhishingWarning: false,
|
|
294
|
+
isTurnstile: false,
|
|
295
|
+
isJSChallenge: false,
|
|
296
|
+
isChallengeCompleted: false,
|
|
297
|
+
error: lastError?.message || 'Unknown error',
|
|
298
|
+
errorType: categorizeError(lastError),
|
|
299
|
+
attempts: maxRetries
|
|
300
|
+
};
|
|
132
301
|
}
|
|
133
302
|
|
|
134
303
|
/**
|
|
@@ -165,7 +334,7 @@ async function safeWaitForNavigation(page, timeout = TIMEOUTS.NAVIGATION_TIMEOUT
|
|
|
165
334
|
}
|
|
166
335
|
|
|
167
336
|
/**
|
|
168
|
-
* Quick Cloudflare detection
|
|
337
|
+
* Quick Cloudflare detection with caching for performance
|
|
169
338
|
*/
|
|
170
339
|
async function quickCloudflareDetection(page, forceDebug = false) {
|
|
171
340
|
try {
|
|
@@ -179,23 +348,34 @@ async function quickCloudflareDetection(page, forceDebug = false) {
|
|
|
179
348
|
return { hasIndicators: false, skippedInvalidUrl: true };
|
|
180
349
|
}
|
|
181
350
|
|
|
182
|
-
//
|
|
183
|
-
|
|
351
|
+
// Check cache first
|
|
352
|
+
const cachedResult = detectionCache.get(currentPageUrl);
|
|
353
|
+
if (cachedResult !== null) {
|
|
354
|
+
if (forceDebug) {
|
|
355
|
+
const stats = detectionCache.getStats();
|
|
356
|
+
console.log(`[debug][cloudflare] Using cached detection result (cache hit rate: ${stats.hitRate})`);
|
|
357
|
+
}
|
|
358
|
+
return cachedResult;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Perform actual detection with enhanced error handling
|
|
184
362
|
const quickCheck = await safePageEvaluate(page, () => {
|
|
185
363
|
const title = document.title || '';
|
|
186
364
|
const bodyText = document.body ? document.body.textContent.substring(0, 500) : '';
|
|
187
365
|
const url = window.location.href;
|
|
188
366
|
|
|
189
|
-
//
|
|
367
|
+
// Enhanced indicators with 2025 patterns
|
|
190
368
|
const hasCloudflareIndicators =
|
|
191
369
|
title.includes('Just a moment') ||
|
|
192
370
|
title.includes('Checking your browser') ||
|
|
193
371
|
title.includes('Attention Required') ||
|
|
372
|
+
title.includes('Security check') || // New pattern
|
|
194
373
|
bodyText.includes('Cloudflare') ||
|
|
195
374
|
bodyText.includes('cf-ray') ||
|
|
196
375
|
bodyText.includes('Verify you are human') ||
|
|
197
376
|
bodyText.includes('This website has been reported for potential phishing') ||
|
|
198
377
|
bodyText.includes('Please wait while we verify') ||
|
|
378
|
+
bodyText.includes('Checking if the site connection is secure') || // New pattern
|
|
199
379
|
url.includes('/cdn-cgi/challenge-platform/') ||
|
|
200
380
|
url.includes('cloudflare.com') ||
|
|
201
381
|
document.querySelector('[data-ray]') ||
|
|
@@ -207,7 +387,10 @@ async function quickCloudflareDetection(page, forceDebug = false) {
|
|
|
207
387
|
document.querySelector('iframe[src*="challenges.cloudflare.com"]') ||
|
|
208
388
|
document.querySelector('iframe[title*="Cloudflare security challenge"]') ||
|
|
209
389
|
document.querySelector('script[src*="/cdn-cgi/challenge-platform/"]') ||
|
|
210
|
-
document.querySelector('a[href*="continue"]')
|
|
390
|
+
document.querySelector('a[href*="continue"]') ||
|
|
391
|
+
// New selectors for 2025
|
|
392
|
+
document.querySelector('.cf-managed-challenge') ||
|
|
393
|
+
document.querySelector('[data-cf-managed]');
|
|
211
394
|
|
|
212
395
|
return {
|
|
213
396
|
hasIndicators: hasCloudflareIndicators,
|
|
@@ -215,12 +398,21 @@ async function quickCloudflareDetection(page, forceDebug = false) {
|
|
|
215
398
|
url,
|
|
216
399
|
bodySnippet: bodyText.substring(0, 200)
|
|
217
400
|
};
|
|
218
|
-
}, FAST_TIMEOUTS.QUICK_DETECTION);
|
|
401
|
+
}, FAST_TIMEOUTS.QUICK_DETECTION, { maxRetries: 2, forceDebug });
|
|
402
|
+
|
|
403
|
+
// Cache the result
|
|
404
|
+
detectionCache.set(currentPageUrl, quickCheck);
|
|
219
405
|
|
|
220
|
-
if (forceDebug
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
406
|
+
if (forceDebug) {
|
|
407
|
+
if (quickCheck.hasIndicators) {
|
|
408
|
+
console.log(`[debug][cloudflare] Quick detection found Cloudflare indicators on ${quickCheck.url}`);
|
|
409
|
+
} else {
|
|
410
|
+
console.log(`[debug][cloudflare] Quick detection found no Cloudflare indicators on ${quickCheck.url}`);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (quickCheck.attempts && quickCheck.attempts > 1) {
|
|
414
|
+
console.log(`[debug][cloudflare] Detection required ${quickCheck.attempts} attempts`);
|
|
415
|
+
}
|
|
224
416
|
}
|
|
225
417
|
|
|
226
418
|
return quickCheck;
|
|
@@ -1017,6 +1209,87 @@ async function performCloudflareHandling(page, currentUrl, siteConfig, forceDebu
|
|
|
1017
1209
|
return result;
|
|
1018
1210
|
}
|
|
1019
1211
|
|
|
1212
|
+
/**
|
|
1213
|
+
* Performs parallel detection of multiple challenge types for better performance
|
|
1214
|
+
*/
|
|
1215
|
+
async function parallelChallengeDetection(page, forceDebug = false) {
|
|
1216
|
+
const detectionPromises = [];
|
|
1217
|
+
|
|
1218
|
+
// Check for JS challenge
|
|
1219
|
+
detectionPromises.push(
|
|
1220
|
+
page.evaluate(() => {
|
|
1221
|
+
return {
|
|
1222
|
+
type: 'js',
|
|
1223
|
+
detected: document.querySelector('script[src*="/cdn-cgi/challenge-platform/"]') !== null ||
|
|
1224
|
+
document.body?.textContent?.includes('Checking your browser') ||
|
|
1225
|
+
document.body?.textContent?.includes('Please wait while we verify')
|
|
1226
|
+
};
|
|
1227
|
+
}).catch(err => ({ type: 'js', detected: false, error: err.message }))
|
|
1228
|
+
);
|
|
1229
|
+
|
|
1230
|
+
// Check for Turnstile
|
|
1231
|
+
detectionPromises.push(
|
|
1232
|
+
page.evaluate(() => {
|
|
1233
|
+
return {
|
|
1234
|
+
type: 'turnstile',
|
|
1235
|
+
detected: document.querySelector('.cf-turnstile') !== null ||
|
|
1236
|
+
document.querySelector('iframe[src*="challenges.cloudflare.com"]') !== null ||
|
|
1237
|
+
document.querySelector('.ctp-checkbox-container') !== null
|
|
1238
|
+
};
|
|
1239
|
+
}).catch(err => ({ type: 'turnstile', detected: false, error: err.message }))
|
|
1240
|
+
);
|
|
1241
|
+
|
|
1242
|
+
// Check for phishing warning
|
|
1243
|
+
detectionPromises.push(
|
|
1244
|
+
page.evaluate(() => {
|
|
1245
|
+
return {
|
|
1246
|
+
type: 'phishing',
|
|
1247
|
+
detected: document.body?.textContent?.includes('This website has been reported for potential phishing') ||
|
|
1248
|
+
document.querySelector('a[href*="continue"]') !== null
|
|
1249
|
+
};
|
|
1250
|
+
}).catch(err => ({ type: 'phishing', detected: false, error: err.message }))
|
|
1251
|
+
);
|
|
1252
|
+
|
|
1253
|
+
// Check for managed challenge
|
|
1254
|
+
detectionPromises.push(
|
|
1255
|
+
page.evaluate(() => {
|
|
1256
|
+
return {
|
|
1257
|
+
type: 'managed',
|
|
1258
|
+
detected: document.querySelector('.cf-managed-challenge') !== null ||
|
|
1259
|
+
document.querySelector('[data-cf-managed]') !== null
|
|
1260
|
+
};
|
|
1261
|
+
}).catch(err => ({ type: 'managed', detected: false, error: err.message }))
|
|
1262
|
+
);
|
|
1263
|
+
|
|
1264
|
+
const results = await Promise.all(detectionPromises);
|
|
1265
|
+
|
|
1266
|
+
const detectedChallenges = results.filter(r => r.detected).map(r => r.type);
|
|
1267
|
+
|
|
1268
|
+
if (forceDebug && detectedChallenges.length > 0) {
|
|
1269
|
+
console.log(`[debug][cloudflare] Parallel detection found challenges: ${detectedChallenges.join(', ')}`);
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
return {
|
|
1273
|
+
challenges: detectedChallenges,
|
|
1274
|
+
hasAnyChallenge: detectedChallenges.length > 0,
|
|
1275
|
+
details: results
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/**
|
|
1280
|
+
* Gets cache statistics for performance monitoring
|
|
1281
|
+
*/
|
|
1282
|
+
function getCacheStats() {
|
|
1283
|
+
return detectionCache.getStats();
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
/**
|
|
1287
|
+
* Clears the detection cache
|
|
1288
|
+
*/
|
|
1289
|
+
function clearDetectionCache() {
|
|
1290
|
+
detectionCache.clear();
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1020
1293
|
module.exports = {
|
|
1021
1294
|
analyzeCloudflareChallenge,
|
|
1022
1295
|
handlePhishingWarning,
|
|
@@ -1029,5 +1302,12 @@ module.exports = {
|
|
|
1029
1302
|
checkChallengeCompletion,
|
|
1030
1303
|
quickCloudflareDetection,
|
|
1031
1304
|
getModuleInfo,
|
|
1032
|
-
CLOUDFLARE_MODULE_VERSION
|
|
1033
|
-
|
|
1305
|
+
CLOUDFLARE_MODULE_VERSION,
|
|
1306
|
+
// New exports
|
|
1307
|
+
parallelChallengeDetection,
|
|
1308
|
+
getCacheStats,
|
|
1309
|
+
clearDetectionCache,
|
|
1310
|
+
categorizeError,
|
|
1311
|
+
ERROR_TYPES,
|
|
1312
|
+
RETRY_CONFIG
|
|
1313
|
+
};
|
package/nwss.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// === Network scanner script (nwss.js) v1.0.
|
|
1
|
+
// === Network scanner script (nwss.js) v1.0.72 ===
|
|
2
2
|
|
|
3
3
|
// puppeteer for browser automation, fs for file system operations, psl for domain parsing.
|
|
4
4
|
// const pLimit = require('p-limit'); // Will be dynamically imported
|
|
@@ -14,7 +14,12 @@ const { formatRules, handleOutput, getFormatDescription } = require('./lib/outpu
|
|
|
14
14
|
// Rule validation
|
|
15
15
|
const { validateRulesetFile, validateFullConfig, testDomainValidation, cleanRulesetFile } = require('./lib/validate_rules');
|
|
16
16
|
// CF Bypass
|
|
17
|
-
const {
|
|
17
|
+
const {
|
|
18
|
+
handleCloudflareProtection,
|
|
19
|
+
getCacheStats,
|
|
20
|
+
clearDetectionCache,
|
|
21
|
+
parallelChallengeDetection
|
|
22
|
+
} = require('./lib/cloudflare');
|
|
18
23
|
// FP Bypass
|
|
19
24
|
const { handleFlowProxyProtection, getFlowProxyTimeouts } = require('./lib/flowproxy');
|
|
20
25
|
// ignore_similar rules
|
|
@@ -83,7 +88,7 @@ function detectPuppeteerVersion() {
|
|
|
83
88
|
try {
|
|
84
89
|
const puppeteer = require('puppeteer');
|
|
85
90
|
let versionString = null;
|
|
86
|
-
|
|
91
|
+
|
|
87
92
|
// Try multiple methods to get version
|
|
88
93
|
if (puppeteer.version) {
|
|
89
94
|
versionString = puppeteer.version;
|
|
@@ -118,7 +123,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
|
|
|
118
123
|
const { monitorBrowserHealth, isBrowserHealthy } = require('./lib/browserhealth');
|
|
119
124
|
|
|
120
125
|
// --- Script Configuration & Constants ---
|
|
121
|
-
const VERSION = '1.0.
|
|
126
|
+
const VERSION = '1.0.72'; // Script version
|
|
122
127
|
|
|
123
128
|
// get startTime
|
|
124
129
|
const startTime = Date.now();
|
|
@@ -311,6 +316,10 @@ if (clearCache && !dryRunMode) {
|
|
|
311
316
|
forceDebug,
|
|
312
317
|
cachePath: CACHE_LIMITS.DEFAULT_CACHE_PATH // Default path, will be updated after config loads if needed
|
|
313
318
|
});
|
|
319
|
+
|
|
320
|
+
// Also clear Cloudflare detection cache
|
|
321
|
+
clearDetectionCache();
|
|
322
|
+
if (forceDebug) console.log(formatLogMessage('debug', 'Cleared Cloudflare detection cache'));
|
|
314
323
|
}
|
|
315
324
|
|
|
316
325
|
// Handle validation-only operations before main help
|
|
@@ -517,6 +526,10 @@ Redirect Handling Options:
|
|
|
517
526
|
Cloudflare Protection Options:
|
|
518
527
|
cloudflare_phish: true/false Auto-click through Cloudflare phishing warnings (default: false)
|
|
519
528
|
cloudflare_bypass: true/false Auto-solve Cloudflare "Verify you are human" challenges (default: false)
|
|
529
|
+
cloudflare_parallel_detection: true/false Use parallel detection for faster Cloudflare checks (default: true)
|
|
530
|
+
cloudflare_max_retries: <number> Maximum retry attempts for Cloudflare operations (default: 3)
|
|
531
|
+
cloudflare_cache_ttl: <milliseconds> TTL for Cloudflare detection cache (default: 300000 - 5 minutes)
|
|
532
|
+
cloudflare_retry_on_error: true/false Enable retry logic for Cloudflare operations (default: true)
|
|
520
533
|
|
|
521
534
|
FlowProxy Protection Options:
|
|
522
535
|
flowproxy_detection: true/false Enable flowProxy protection detection and handling (default: false)
|
|
@@ -2409,15 +2422,41 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2409
2422
|
|
|
2410
2423
|
siteCounter++;
|
|
2411
2424
|
|
|
2412
|
-
//
|
|
2425
|
+
// Enhanced Cloudflare handling with parallel detection
|
|
2426
|
+
if (siteConfig.cloudflare_parallel_detection !== false) { // Enable by default
|
|
2427
|
+
try {
|
|
2428
|
+
const parallelResult = await parallelChallengeDetection(page, forceDebug);
|
|
2429
|
+
if (parallelResult.hasAnyChallenge && forceDebug) {
|
|
2430
|
+
console.log(formatLogMessage('debug', `[cloudflare] Parallel detection found: ${parallelResult.challenges.join(', ')}`));
|
|
2431
|
+
}
|
|
2432
|
+
} catch (parallelErr) {
|
|
2433
|
+
if (forceDebug) {
|
|
2434
|
+
console.log(formatLogMessage('debug', `[cloudflare] Parallel detection failed: ${parallelErr.message}`));
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
// Handle all Cloudflare protections using the enhanced module
|
|
2413
2440
|
const cloudflareResult = await handleCloudflareProtection(page, currentUrl, siteConfig, forceDebug);
|
|
2414
|
-
|
|
2441
|
+
// Check for retry recommendations
|
|
2442
|
+
if (cloudflareResult.errors && cloudflareResult.errors.length > 0) {
|
|
2443
|
+
const hasRetryableErrors = cloudflareResult.errors.some(err =>
|
|
2444
|
+
err.includes('timeout') || err.includes('network')
|
|
2445
|
+
);
|
|
2446
|
+
|
|
2447
|
+
if (hasRetryableErrors && forceDebug) {
|
|
2448
|
+
console.log(formatLogMessage('debug', '[cloudflare] Errors may be retryable - consider enabling retry logic'));
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
|
|
2415
2452
|
if (!cloudflareResult.overallSuccess) {
|
|
2416
2453
|
console.warn(`⚠ [cloudflare] Protection handling failed for ${currentUrl}:`);
|
|
2417
2454
|
cloudflareResult.errors.forEach(error => {
|
|
2418
2455
|
console.warn(` - ${error}`);
|
|
2419
2456
|
});
|
|
2420
2457
|
// Continue with scan despite Cloudflare issues
|
|
2458
|
+
} else if (cloudflareResult.verificationChallenge?.success && forceDebug) {
|
|
2459
|
+
console.log(formatLogMessage('debug', `[cloudflare] Challenge solved using: ${cloudflareResult.verificationChallenge.method}`));
|
|
2421
2460
|
}
|
|
2422
2461
|
|
|
2423
2462
|
// Handle flowProxy protection if enabled
|
|
@@ -2959,6 +2998,13 @@ function setupFrameHandling(page, forceDebug) {
|
|
|
2959
2998
|
console.log(formatLogMessage('debug', `Output format: ${getFormatDescription(globalOptions)}`));
|
|
2960
2999
|
console.log(formatLogMessage('debug', `Generated ${outputResult.totalRules} rules from ${outputResult.successfulPageLoads} successful page loads`));
|
|
2961
3000
|
console.log(formatLogMessage('debug', `Performance: ${totalDomainsSkipped} domains skipped (already detected), ${detectedDomainsCount} unique domains cached`));
|
|
3001
|
+
// Cloudflare cache statistics
|
|
3002
|
+
const cloudflareStats = getCacheStats();
|
|
3003
|
+
if (cloudflareStats.size > 0) {
|
|
3004
|
+
console.log(formatLogMessage('debug', '=== Cloudflare Cache Statistics ==='));
|
|
3005
|
+
console.log(formatLogMessage('debug', `Cache hit rate: ${cloudflareStats.hitRate}, Total hits: ${cloudflareStats.hits}, Misses: ${cloudflareStats.misses}`));
|
|
3006
|
+
console.log(formatLogMessage('debug', `Cached detections: ${cloudflareStats.size}`));
|
|
3007
|
+
}
|
|
2962
3008
|
// Log smart cache statistics (if cache is enabled)
|
|
2963
3009
|
if (smartCache) {
|
|
2964
3010
|
const cacheStats = smartCache.getStats();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.72",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|
package/regex-tool/index.html
CHANGED
|
@@ -291,16 +291,16 @@
|
|
|
291
291
|
<body>
|
|
292
292
|
<div class="container">
|
|
293
293
|
<div class="header">
|
|
294
|
-
<h1
|
|
294
|
+
<h1>?? Regex Tools</h1>
|
|
295
295
|
<p>Convert and Validate Regular Expressions with Ease</p>
|
|
296
296
|
</div>
|
|
297
297
|
|
|
298
298
|
<div class="tabs">
|
|
299
299
|
<button class="tab active" data-tab="convert">
|
|
300
|
-
|
|
300
|
+
?? Convert to JSON
|
|
301
301
|
</button>
|
|
302
302
|
<button class="tab" data-tab="validate">
|
|
303
|
-
|
|
303
|
+
? Validate Regex
|
|
304
304
|
</button>
|
|
305
305
|
</div>
|
|
306
306
|
|
|
@@ -321,12 +321,12 @@
|
|
|
321
321
|
<div id="validate" class="tab-content">
|
|
322
322
|
<div class="checkbox-group" style="background: #f0f9ff; padding: 15px; border-radius: 8px; margin-bottom: 20px; border: 2px solid #0ea5e9;">
|
|
323
323
|
<input type="checkbox" id="useStandardRegex">
|
|
324
|
-
<label for="useStandardRegex"><strong
|
|
324
|
+
<label for="useStandardRegex"><strong>?? Check this box to validate Standard Regex</strong> (uncheck for JSON format)</label>
|
|
325
325
|
</div>
|
|
326
326
|
|
|
327
327
|
<div class="input-group">
|
|
328
328
|
<label for="regexToValidate">Enter Regex Pattern:</label>
|
|
329
|
-
<textarea id="regexToValidate" placeholder='
|
|
329
|
+
<textarea id="regexToValidate" placeholder='?? IMPORTANT: Check the box above if entering standard regex! For JSON (box unchecked): {"filterRegex": ["^https?:\\\\/\\\\/.*"]} For Standard (box checked): ^https?:\\/\\/.*'></textarea>
|
|
330
330
|
</div>
|
|
331
331
|
|
|
332
332
|
<button class="button" id="validateBtn">Validate Regex</button>
|
|
@@ -637,7 +637,7 @@
|
|
|
637
637
|
try {
|
|
638
638
|
regex.lastIndex = 0;
|
|
639
639
|
return regex.test(ex);
|
|
640
|
-
} catch {
|
|
640
|
+
} catch (e) {
|
|
641
641
|
return false;
|
|
642
642
|
}
|
|
643
643
|
});
|
|
@@ -680,7 +680,56 @@
|
|
|
680
680
|
const jsonOutput = {
|
|
681
681
|
filterRegex: [cleanPattern]
|
|
682
682
|
};
|
|
683
|
-
|
|
683
|
+
|
|
684
|
+
// Generate examples
|
|
685
|
+
const examples = generateExamples(testRegex);
|
|
686
|
+
|
|
687
|
+
outputDiv.innerHTML = `
|
|
688
|
+
<div class="output-section">
|
|
689
|
+
<h3>?? JSON Output:</h3>
|
|
690
|
+
<div class="output-box">${JSON.stringify(jsonOutput, null, 2)}</div>
|
|
691
|
+
|
|
692
|
+
<h3>? Example Matches:</h3>
|
|
693
|
+
<ul class="examples-list">
|
|
694
|
+
${examples.map(ex => `<li>${escapeHtml(ex)}</li>`).join('')}
|
|
695
|
+
</ul>
|
|
696
|
+
|
|
697
|
+
<div class="success">? Regex successfully converted to JSON format!</div>
|
|
698
|
+
</div>
|
|
699
|
+
`;
|
|
700
|
+
} catch (error) {
|
|
701
|
+
outputDiv.innerHTML = `<div class="error">? Invalid regex pattern: ${escapeHtml(error.message)}</div>`;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function validateRegex() {
|
|
706
|
+
const useStandard = document.getElementById('useStandardRegex').checked;
|
|
707
|
+
const regexInput = document.getElementById('regexToValidate').value.trim();
|
|
708
|
+
const outputDiv = document.getElementById('validateOutput');
|
|
709
|
+
|
|
710
|
+
if (!regexInput) {
|
|
711
|
+
outputDiv.innerHTML = '<div class="error">Please enter a regex pattern to validate</div>';
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
try {
|
|
716
|
+
let regex;
|
|
717
|
+
let pattern;
|
|
718
|
+
|
|
719
|
+
if (useStandard) {
|
|
720
|
+
// Use standard regex directly
|
|
721
|
+
pattern = regexInput;
|
|
722
|
+
regex = new RegExp(pattern);
|
|
723
|
+
} else {
|
|
724
|
+
// JSON mode - handle multiple formats
|
|
725
|
+
if (regexInput.startsWith('{')) {
|
|
726
|
+
// Full JSON format
|
|
727
|
+
const jsonRegex = JSON.parse(regexInput);
|
|
728
|
+
|
|
729
|
+
if (jsonRegex.filterRegex && Array.isArray(jsonRegex.filterRegex)) {
|
|
730
|
+
pattern = jsonRegex.filterRegex[0];
|
|
731
|
+
} else if (jsonRegex.pattern) {
|
|
732
|
+
pattern = jsonRegex.pattern;
|
|
684
733
|
} else {
|
|
685
734
|
throw new Error('Invalid JSON format. Expected {"filterRegex": ["pattern"]}');
|
|
686
735
|
}
|
|
@@ -691,14 +740,14 @@
|
|
|
691
740
|
// Raw pattern without quotes or JSON wrapper
|
|
692
741
|
outputDiv.innerHTML = `
|
|
693
742
|
<div class="error">
|
|
694
|
-
|
|
743
|
+
? Invalid format for JSON mode.<br><br>
|
|
695
744
|
You can use one of these formats:<br>
|
|
696
745
|
<div class="output-box" style="margin-top: 10px; background: #fef3c7;">
|
|
697
746
|
1. Full JSON: {"filterRegex": ["${escapeHtml(regexInput)}"]}<br><br>
|
|
698
747
|
2. Quoted string: "${escapeHtml(regexInput)}"
|
|
699
748
|
</div>
|
|
700
749
|
<br>
|
|
701
|
-
|
|
750
|
+
?? <strong>Tip:</strong> Check the "Use Standard Regex" box above to validate regex patterns directly without quotes.
|
|
702
751
|
</div>
|
|
703
752
|
`;
|
|
704
753
|
return;
|
|
@@ -712,25 +761,25 @@
|
|
|
712
761
|
|
|
713
762
|
outputDiv.innerHTML = `
|
|
714
763
|
<div class="output-section">
|
|
715
|
-
<h3
|
|
764
|
+
<h3>?? Validation Result:</h3>
|
|
716
765
|
<div class="output-box">
|
|
717
766
|
Pattern: ${escapeHtml(pattern)}<br>
|
|
718
767
|
Type: ${useStandard ? 'Standard Regex' : 'JSON Regex'}
|
|
719
768
|
</div>
|
|
720
769
|
|
|
721
|
-
<h3
|
|
770
|
+
<h3>? Example Matches:</h3>
|
|
722
771
|
<ul class="examples-list">
|
|
723
772
|
${examples.map(ex => `<li>${escapeHtml(ex)}</li>`).join('')}
|
|
724
773
|
</ul>
|
|
725
774
|
|
|
726
|
-
<div class="success"
|
|
775
|
+
<div class="success">? Regex is valid!</div>
|
|
727
776
|
</div>
|
|
728
777
|
`;
|
|
729
778
|
} catch (error) {
|
|
730
779
|
if (error instanceof SyntaxError && !useStandard) {
|
|
731
|
-
outputDiv.innerHTML = `<div class="error"
|
|
780
|
+
outputDiv.innerHTML = `<div class="error">? Invalid JSON syntax: ${escapeHtml(error.message)}<br><br>Expected format: {"filterRegex": ["your-regex-pattern"]} or "your-regex-pattern"</div>`;
|
|
732
781
|
} else {
|
|
733
|
-
outputDiv.innerHTML = `<div class="error"
|
|
782
|
+
outputDiv.innerHTML = `<div class="error">? Invalid regex: ${escapeHtml(error.message)}</div>`;
|
|
734
783
|
}
|
|
735
784
|
}
|
|
736
785
|
}
|
|
@@ -742,4 +791,4 @@
|
|
|
742
791
|
}
|
|
743
792
|
</script>
|
|
744
793
|
</body>
|
|
745
|
-
</html>
|
|
794
|
+
</html>
|