@fanboynz/network-scanner 2.0.26 → 2.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/lib/browserhealth.js +139 -57
- package/lib/cdp.js +60 -38
- package/lib/nettools.js +172 -117
- package/nwss.js +2 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -281,6 +281,36 @@ These options go at the root level of your config.json:
|
|
|
281
281
|
| `cache_requests` | Boolean | `false` | Enable HTTP request response caching |
|
|
282
282
|
|
|
283
283
|
---
|
|
284
|
+
#### Special Characters in searchstring
|
|
285
|
+
|
|
286
|
+
The `searchstring` parameter supports all characters including special symbols. Only double quotes need JSON escaping:
|
|
287
|
+
|
|
288
|
+
```json
|
|
289
|
+
{
|
|
290
|
+
"searchstring": [
|
|
291
|
+
")}return n}function N(n,e,r){try{\"function\"==typeof",
|
|
292
|
+
"addEventListener(\"click\",function(){",
|
|
293
|
+
"{\"status\":\"success\",\"data\":[",
|
|
294
|
+
"console.log('Debug: ' + value);",
|
|
295
|
+
"`API endpoint: ${baseUrl}/users`",
|
|
296
|
+
"@media screen and (max-width: 768px)",
|
|
297
|
+
"if(e&&e.preventDefault){e.preventDefault()}",
|
|
298
|
+
"__webpack_require__(/*! ./module */ \"./src/module.js\")",
|
|
299
|
+
"console.log('Hello world')",
|
|
300
|
+
"#header { background-color: #ff0000; }",
|
|
301
|
+
"$(document).ready(function() {",
|
|
302
|
+
"completion: 85% @ $1,500 budget",
|
|
303
|
+
"SELECT * FROM users WHERE id = *",
|
|
304
|
+
"regex: ^[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}$",
|
|
305
|
+
"typeof window !== 'undefined'"
|
|
306
|
+
]
|
|
307
|
+
}
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
**Character escaping rules:**
|
|
311
|
+
- `"` becomes `\"` (required in JSON)
|
|
312
|
+
- `\` becomes `\\` (if searching for literal backslashes)
|
|
313
|
+
- All other characters are used literally: `'` `` ` `` `@` `#` `$` `%` `*` `^` `[` `]` `{` `}` `(` `)` `;` `=` `!` `?` `:`
|
|
284
314
|
|
|
285
315
|
## Usage Examples
|
|
286
316
|
|
package/lib/browserhealth.js
CHANGED
|
@@ -33,6 +33,18 @@ const PAGE_IDLE_THRESHOLD = 25000; // 25 seconds of inactivity before considerin
|
|
|
33
33
|
async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug, cleanupMode = true) {
|
|
34
34
|
try {
|
|
35
35
|
// Wait before cleanup to allow any final operations to complete
|
|
36
|
+
// Initialize result object with ALL possible properties upfront for V8 optimization
|
|
37
|
+
const result = {
|
|
38
|
+
success: false,
|
|
39
|
+
closedCount: 0,
|
|
40
|
+
totalPages: 0,
|
|
41
|
+
mainPagePreserved: false,
|
|
42
|
+
delayUsed: 0,
|
|
43
|
+
estimatedMemoryFreed: 0,
|
|
44
|
+
estimatedMemoryFreedFormatted: '',
|
|
45
|
+
cleanupMode: '',
|
|
46
|
+
error: null
|
|
47
|
+
};
|
|
36
48
|
const modeText = cleanupMode === "all" ? "aggressive cleanup of old windows" : "conservative cleanup of extra windows"
|
|
37
49
|
if (forceDebug) {
|
|
38
50
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before ${modeText} for group: ${groupDescription}`));
|
|
@@ -47,8 +59,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
47
59
|
|
|
48
60
|
// Find the main page - typically the first page that's about:blank or has been there longest
|
|
49
61
|
for (const page of allPages) {
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
// Cache page.url() call to avoid repeated DOM/browser communication
|
|
63
|
+
const pageUrl = page.url();
|
|
64
|
+
if (pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://')) {
|
|
52
65
|
if (!mainPuppeteerPage) {
|
|
53
66
|
mainPuppeteerPage = page; // First blank page is likely the main window
|
|
54
67
|
} else {
|
|
@@ -56,6 +69,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
56
69
|
}
|
|
57
70
|
} else {
|
|
58
71
|
// Any page with actual content should be evaluated for closure
|
|
72
|
+
// Cache page state checks to avoid multiple browser calls
|
|
73
|
+
const isPageClosed = page.isClosed();
|
|
74
|
+
|
|
59
75
|
if (cleanupMode === "all") {
|
|
60
76
|
// Aggressive mode: close all content pages
|
|
61
77
|
pagesToClose.push(page);
|
|
@@ -75,7 +91,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
75
91
|
mainPuppeteerPage = allPages[0]; // Fallback to first page
|
|
76
92
|
pagesToClose = allPages.slice(1);
|
|
77
93
|
if (forceDebug) {
|
|
78
|
-
|
|
94
|
+
// Cache URL call for logging
|
|
95
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
96
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] No blank page found, using first page as main: ${mainPageUrl}`));
|
|
79
97
|
}
|
|
80
98
|
}
|
|
81
99
|
|
|
@@ -83,14 +101,11 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
83
101
|
if (forceDebug) {
|
|
84
102
|
console.log(formatLogMessage('debug', `[group_window_cleanup] No windows to close for group: ${groupDescription}`));
|
|
85
103
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
mainPagePreserved: true,
|
|
92
|
-
cleanupMode: cleanupMode === "all" ? "all" : "default"
|
|
93
|
-
};
|
|
104
|
+
result.success = true;
|
|
105
|
+
result.totalPages = allPages.length;
|
|
106
|
+
result.mainPagePreserved = true;
|
|
107
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
108
|
+
return result;
|
|
94
109
|
}
|
|
95
110
|
|
|
96
111
|
// Estimate memory usage before closing
|
|
@@ -102,7 +117,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
102
117
|
let pageMemoryEstimate = 0;
|
|
103
118
|
|
|
104
119
|
try {
|
|
105
|
-
|
|
120
|
+
// Cache page.isClosed() check to avoid repeated browser calls
|
|
121
|
+
const isPageClosed = page.isClosed();
|
|
122
|
+
if (!isPageClosed) {
|
|
106
123
|
// Get page metrics if available
|
|
107
124
|
const metrics = await Promise.race([
|
|
108
125
|
page.metrics(),
|
|
@@ -138,12 +155,15 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
138
155
|
// Close identified old/unused pages
|
|
139
156
|
const closePromises = pagesToClose.map(async (page, index) => {
|
|
140
157
|
try {
|
|
141
|
-
|
|
158
|
+
// Cache page state and URL for this operation
|
|
159
|
+
const isPageClosed = page.isClosed();
|
|
160
|
+
const pageUrl = page.url();
|
|
161
|
+
if (!isPageClosed) {
|
|
142
162
|
if (forceDebug) {
|
|
143
|
-
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${
|
|
163
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${pageUrl}`));
|
|
144
164
|
}
|
|
145
165
|
await page.close();
|
|
146
|
-
return { success: true, url:
|
|
166
|
+
return { success: true, url: pageUrl || `page-${index}`, estimatedMemory: pageMemoryEstimates[index] };
|
|
147
167
|
}
|
|
148
168
|
return { success: false, reason: 'already_closed', estimatedMemory: 0 };
|
|
149
169
|
} catch (closeErr) {
|
|
@@ -177,25 +197,39 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
177
197
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${pagesToClose.length} old windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
|
|
178
198
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Estimated memory freed: ${formatMemory(actualMemoryFreed)}`));
|
|
179
199
|
if (mainPuppeteerPage) {
|
|
180
|
-
|
|
200
|
+
// Cache URL for final logging
|
|
201
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
202
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Main Puppeteer window preserved: ${mainPageUrl}`));
|
|
181
203
|
}
|
|
182
204
|
}
|
|
183
205
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
206
|
+
// Update result object instead of creating new one
|
|
207
|
+
result.success = true;
|
|
208
|
+
result.closedCount = successfulCloses;
|
|
209
|
+
result.totalPages = allPages.length;
|
|
210
|
+
result.mainPagePreserved = mainPuppeteerPage && !mainPuppeteerPage.isClosed();
|
|
211
|
+
result.delayUsed = WINDOW_CLEANUP_DELAY_MS;
|
|
212
|
+
result.estimatedMemoryFreed = actualMemoryFreed;
|
|
213
|
+
result.estimatedMemoryFreedFormatted = formatMemory(actualMemoryFreed);
|
|
214
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
215
|
+
return result;
|
|
194
216
|
} catch (cleanupErr) {
|
|
195
217
|
if (forceDebug) {
|
|
196
218
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Group cleanup failed for ${groupDescription}: ${cleanupErr.message}`));
|
|
197
219
|
}
|
|
198
|
-
|
|
220
|
+
// Initialize result object with consistent shape for error case
|
|
221
|
+
const result = {
|
|
222
|
+
success: false,
|
|
223
|
+
closedCount: 0,
|
|
224
|
+
totalPages: 0,
|
|
225
|
+
mainPagePreserved: false,
|
|
226
|
+
delayUsed: 0,
|
|
227
|
+
estimatedMemoryFreed: 0,
|
|
228
|
+
estimatedMemoryFreedFormatted: '',
|
|
229
|
+
cleanupMode: '',
|
|
230
|
+
error: cleanupErr.message
|
|
231
|
+
};
|
|
232
|
+
return result;
|
|
199
233
|
}
|
|
200
234
|
}
|
|
201
235
|
|
|
@@ -207,14 +241,17 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
207
241
|
*/
|
|
208
242
|
async function isPageSafeToClose(page, forceDebug) {
|
|
209
243
|
try {
|
|
210
|
-
|
|
244
|
+
// Cache page.isClosed() to avoid repeated browser communication
|
|
245
|
+
const isPageClosed = page.isClosed();
|
|
246
|
+
if (isPageClosed) {
|
|
211
247
|
return true; // Already closed
|
|
212
248
|
}
|
|
213
249
|
|
|
214
250
|
// EXTRA SAFETY: Never close pages that might be in injection process
|
|
215
251
|
try {
|
|
216
|
-
|
|
217
|
-
|
|
252
|
+
// Cache page.url() for safety checks
|
|
253
|
+
const pageUrl = page.url();
|
|
254
|
+
if (pageUrl && pageUrl !== 'about:blank' && Date.now() - (pageCreationTracker.get(page) || 0) < 30000) {
|
|
218
255
|
return false; // Don't close recently created pages (within 30 seconds)
|
|
219
256
|
}
|
|
220
257
|
} catch (err) { /* ignore */ }
|
|
@@ -228,7 +265,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
228
265
|
// Check if page is actively processing
|
|
229
266
|
if (usage.isProcessing) {
|
|
230
267
|
if (forceDebug) {
|
|
231
|
-
|
|
268
|
+
// Cache URL for debug logging
|
|
269
|
+
const pageUrl = page.url();
|
|
270
|
+
console.log(formatLogMessage('debug', `[realtime_cleanup] Page still processing: ${pageUrl.substring(0, 50)}...`));
|
|
232
271
|
}
|
|
233
272
|
return false;
|
|
234
273
|
}
|
|
@@ -257,7 +296,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
257
296
|
*/
|
|
258
297
|
function updatePageUsage(page, isProcessing = false) {
|
|
259
298
|
try {
|
|
260
|
-
|
|
299
|
+
// Cache page.isClosed() to avoid repeated calls
|
|
300
|
+
const isPageClosed = page.isClosed();
|
|
301
|
+
if (!isPageClosed) {
|
|
261
302
|
pageUsageTracker.set(page, {
|
|
262
303
|
lastActivity: Date.now(),
|
|
263
304
|
isProcessing: isProcessing
|
|
@@ -281,12 +322,27 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
281
322
|
try {
|
|
282
323
|
const allPages = await browserInstance.pages();
|
|
283
324
|
|
|
325
|
+
// Initialize result object with consistent shape
|
|
326
|
+
const result = {
|
|
327
|
+
success: false,
|
|
328
|
+
closedCount: 0,
|
|
329
|
+
totalPages: 0,
|
|
330
|
+
remainingPages: 0,
|
|
331
|
+
threshold: 0,
|
|
332
|
+
cleanupDelay: 0,
|
|
333
|
+
reason: '',
|
|
334
|
+
error: null
|
|
335
|
+
};
|
|
336
|
+
|
|
284
337
|
// Skip cleanup if we don't have enough pages to warrant it
|
|
285
338
|
if (allPages.length <= Math.max(threshold, REALTIME_CLEANUP_MIN_PAGES)) {
|
|
286
339
|
if (forceDebug) {
|
|
287
340
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Only ${allPages.length} pages open, threshold is ${threshold} - no cleanup needed`));
|
|
288
341
|
}
|
|
289
|
-
|
|
342
|
+
result.success = true;
|
|
343
|
+
result.totalPages = allPages.length;
|
|
344
|
+
result.reason = 'below_threshold';
|
|
345
|
+
return result;
|
|
290
346
|
}
|
|
291
347
|
|
|
292
348
|
// Use the provided total delay (already includes appropriate buffer)
|
|
@@ -324,13 +380,18 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
324
380
|
|
|
325
381
|
// Find main Puppeteer page (usually about:blank)
|
|
326
382
|
let mainPage = allPagesAfterDelay.find(page => {
|
|
327
|
-
|
|
328
|
-
|
|
383
|
+
// Cache page.url() for main page detection
|
|
384
|
+
const pageUrl = page.url();
|
|
385
|
+
return pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://');
|
|
329
386
|
}) || allPagesAfterDelay[0]; // Fallback to first page
|
|
330
387
|
|
|
331
388
|
// Get pages sorted by creation time (oldest first)
|
|
332
389
|
const sortedPages = allPagesAfterDelay
|
|
333
|
-
.filter(page =>
|
|
390
|
+
.filter(page => {
|
|
391
|
+
// Cache page.isClosed() for filtering
|
|
392
|
+
const isPageClosed = page.isClosed();
|
|
393
|
+
return page !== mainPage && !isPageClosed;
|
|
394
|
+
})
|
|
334
395
|
.sort((a, b) => {
|
|
335
396
|
const timeA = pageCreationTracker.get(a) || 0;
|
|
336
397
|
const timeB = pageCreationTracker.get(b) || 0;
|
|
@@ -360,15 +421,21 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
360
421
|
`${pagesToClose.length} pages still active`;
|
|
361
422
|
console.log(formatLogMessage('debug', `[realtime_cleanup] No pages need closing (${reason})`));
|
|
362
423
|
}
|
|
363
|
-
|
|
424
|
+
result.success = true;
|
|
425
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
426
|
+
result.reason = 'no_cleanup_needed';
|
|
427
|
+
return result;
|
|
364
428
|
}
|
|
365
429
|
|
|
366
430
|
// Close oldest pages
|
|
367
431
|
let closedCount = 0;
|
|
368
432
|
for (const page of safePagesToClose) {
|
|
369
433
|
try {
|
|
370
|
-
|
|
371
|
-
|
|
434
|
+
// Cache both page state and URL for this iteration
|
|
435
|
+
const isPageClosed = page.isClosed();
|
|
436
|
+
const pageUrl = page.url();
|
|
437
|
+
|
|
438
|
+
if (!isPageClosed) {
|
|
372
439
|
await page.close();
|
|
373
440
|
pageCreationTracker.delete(page); // Remove from tracker
|
|
374
441
|
closedCount++;
|
|
@@ -390,20 +457,30 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
390
457
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Closed ${closedCount}/${pagesToClose.length} oldest pages (${unsafePagesCount} skipped for safety), ${remainingPages} pages remaining`));
|
|
391
458
|
}
|
|
392
459
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
};
|
|
460
|
+
result.success = true;
|
|
461
|
+
result.closedCount = closedCount;
|
|
462
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
463
|
+
result.remainingPages = remainingPages;
|
|
464
|
+
result.threshold = threshold;
|
|
465
|
+
result.cleanupDelay = cleanupDelay;
|
|
466
|
+
result.reason = 'cleanup_completed';
|
|
467
|
+
return result;
|
|
402
468
|
} catch (cleanupErr) {
|
|
403
469
|
if (forceDebug) {
|
|
404
470
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Cleanup failed: ${cleanupErr.message}`));
|
|
405
471
|
}
|
|
406
|
-
|
|
472
|
+
// Initialize result object with consistent shape for error case
|
|
473
|
+
const result = {
|
|
474
|
+
success: false,
|
|
475
|
+
closedCount: 0,
|
|
476
|
+
totalPages: 0,
|
|
477
|
+
remainingPages: 0,
|
|
478
|
+
threshold: 0,
|
|
479
|
+
cleanupDelay: 0,
|
|
480
|
+
reason: '',
|
|
481
|
+
error: cleanupErr.message
|
|
482
|
+
};
|
|
483
|
+
return result;
|
|
407
484
|
}
|
|
408
485
|
}
|
|
409
486
|
|
|
@@ -415,14 +492,15 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
415
492
|
*/
|
|
416
493
|
async function isPageFromPreviousScan(page, forceDebug) {
|
|
417
494
|
try {
|
|
418
|
-
|
|
495
|
+
// Cache page.url() for all checks in this function
|
|
496
|
+
const pageUrl = page.url();
|
|
419
497
|
|
|
420
498
|
// Always consider these as old/closeable
|
|
421
|
-
if (
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
499
|
+
if (pageUrl === 'about:blank' ||
|
|
500
|
+
pageUrl === '' ||
|
|
501
|
+
pageUrl.startsWith('chrome://') ||
|
|
502
|
+
pageUrl.startsWith('chrome-error://') ||
|
|
503
|
+
pageUrl.startsWith('data:')) {
|
|
426
504
|
return false; // Don't close blank pages here, handled separately
|
|
427
505
|
}
|
|
428
506
|
|
|
@@ -446,7 +524,9 @@ async function isPageFromPreviousScan(page, forceDebug) {
|
|
|
446
524
|
return false; // Conservative - don't close unless we're sure
|
|
447
525
|
} catch (err) {
|
|
448
526
|
if (forceDebug) {
|
|
449
|
-
|
|
527
|
+
// Cache URL for error logging
|
|
528
|
+
const pageUrl = page.url();
|
|
529
|
+
console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${pageUrl}: ${err.message}`));
|
|
450
530
|
}
|
|
451
531
|
return false; // Conservative - don't close if we can't evaluate
|
|
452
532
|
}
|
|
@@ -1003,7 +1083,9 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
|
|
|
1003
1083
|
*/
|
|
1004
1084
|
async function cleanupPageBeforeReload(page, forceDebug = false) {
|
|
1005
1085
|
try {
|
|
1006
|
-
|
|
1086
|
+
// Cache page.isClosed() to avoid repeated browser calls
|
|
1087
|
+
const isPageClosed = page.isClosed();
|
|
1088
|
+
if (isPageClosed) {
|
|
1007
1089
|
return false;
|
|
1008
1090
|
}
|
|
1009
1091
|
|
package/lib/cdp.js
CHANGED
|
@@ -27,6 +27,31 @@
|
|
|
27
27
|
|
|
28
28
|
const { formatLogMessage } = require('./colorize');
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Creates a reusable timeout promise to reduce function allocation overhead
|
|
32
|
+
* @param {number} ms - Timeout in milliseconds
|
|
33
|
+
* @param {string} message - Error message for timeout
|
|
34
|
+
* @returns {Promise} Promise that rejects after timeout
|
|
35
|
+
*/
|
|
36
|
+
function createTimeoutPromise(ms, message) {
|
|
37
|
+
return new Promise((_, reject) =>
|
|
38
|
+
setTimeout(() => reject(new Error(message)), ms)
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Creates a standardized session result object for consistent V8 optimization
|
|
44
|
+
* @param {object|null} session - CDP session or null
|
|
45
|
+
* @param {Function} cleanup - Cleanup function
|
|
46
|
+
* @param {boolean} isEnhanced - Whether enhanced features are active
|
|
47
|
+
* @returns {object} Standardized session object
|
|
48
|
+
*/
|
|
49
|
+
const createSessionResult = (session = null, cleanup = async () => {}, isEnhanced = false) => ({
|
|
50
|
+
session,
|
|
51
|
+
cleanup,
|
|
52
|
+
isEnhanced
|
|
53
|
+
});
|
|
54
|
+
|
|
30
55
|
/**
|
|
31
56
|
* Creates a new page with timeout protection to prevent CDP hangs
|
|
32
57
|
* @param {import('puppeteer').Browser} browser - Browser instance
|
|
@@ -36,9 +61,7 @@ const { formatLogMessage } = require('./colorize');
|
|
|
36
61
|
async function createPageWithTimeout(browser, timeout = 30000) {
|
|
37
62
|
return Promise.race([
|
|
38
63
|
browser.newPage(),
|
|
39
|
-
|
|
40
|
-
setTimeout(() => reject(new Error('Page creation timeout - browser may be unresponsive')), timeout)
|
|
41
|
-
)
|
|
64
|
+
createTimeoutPromise(timeout, 'Page creation timeout - browser may be unresponsive')
|
|
42
65
|
]);
|
|
43
66
|
}
|
|
44
67
|
|
|
@@ -52,9 +75,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
52
75
|
try {
|
|
53
76
|
await Promise.race([
|
|
54
77
|
page.setRequestInterception(true),
|
|
55
|
-
|
|
56
|
-
setTimeout(() => reject(new Error('Request interception timeout - first attempt')), timeout)
|
|
57
|
-
)
|
|
78
|
+
createTimeoutPromise(timeout, 'Request interception timeout - first attempt')
|
|
58
79
|
]);
|
|
59
80
|
} catch (firstError) {
|
|
60
81
|
// Check for immediate critical failures
|
|
@@ -68,9 +89,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
68
89
|
try {
|
|
69
90
|
await Promise.race([
|
|
70
91
|
page.setRequestInterception(true),
|
|
71
|
-
|
|
72
|
-
setTimeout(() => reject(new Error('Request interception timeout - retry failed')), timeout * 2)
|
|
73
|
-
)
|
|
92
|
+
createTimeoutPromise(timeout * 2, 'Request interception timeout - retry failed')
|
|
74
93
|
]);
|
|
75
94
|
} catch (retryError) {
|
|
76
95
|
if (retryError.message.includes('Network.enable timed out') ||
|
|
@@ -128,7 +147,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
128
147
|
|
|
129
148
|
if (!cdpLoggingNeeded) {
|
|
130
149
|
// Return a null session with no-op cleanup for consistent API
|
|
131
|
-
return
|
|
150
|
+
return createSessionResult();
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
// Log which CDP mode is being used
|
|
@@ -151,9 +170,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
151
170
|
// Add timeout protection for CDP session creation
|
|
152
171
|
cdpSession = await Promise.race([
|
|
153
172
|
page.createCDPSession(),
|
|
154
|
-
|
|
155
|
-
setTimeout(() => reject(new Error('CDP session creation timeout')), 20000)
|
|
156
|
-
)
|
|
173
|
+
createTimeoutPromise(20000, 'CDP session creation timeout')
|
|
157
174
|
]);
|
|
158
175
|
|
|
159
176
|
// Enable network domain - required for network event monitoring
|
|
@@ -166,19 +183,17 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
166
183
|
const initiator = params.initiator ? params.initiator.type : 'unknown';
|
|
167
184
|
|
|
168
185
|
// Extract hostname for logging context (handles URL parsing errors gracefully)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
|
|
186
|
+
const hostnameForLog = (() => {
|
|
187
|
+
try {
|
|
188
|
+
const currentHostname = new URL(currentUrl).hostname;
|
|
189
|
+
const requestHostname = new URL(requestUrl).hostname;
|
|
190
|
+
return currentHostname !== requestHostname
|
|
191
|
+
? `${currentHostname}?${requestHostname}`
|
|
192
|
+
: currentHostname;
|
|
193
|
+
} catch (_) {
|
|
194
|
+
return 'unknown-host';
|
|
178
195
|
}
|
|
179
|
-
}
|
|
180
|
-
// Ignore URL parsing errors for logging context
|
|
181
|
-
}
|
|
196
|
+
})();
|
|
182
197
|
|
|
183
198
|
// Log the request with context only if debug mode is enabled
|
|
184
199
|
if (forceDebug) {
|
|
@@ -207,7 +222,8 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
207
222
|
}
|
|
208
223
|
}
|
|
209
224
|
}
|
|
210
|
-
}
|
|
225
|
+
},
|
|
226
|
+
isEnhanced: false
|
|
211
227
|
};
|
|
212
228
|
|
|
213
229
|
} catch (cdpErr) {
|
|
@@ -218,7 +234,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
218
234
|
try {
|
|
219
235
|
return new URL(currentUrl).hostname;
|
|
220
236
|
} catch {
|
|
221
|
-
return currentUrl.substring(0, 50)
|
|
237
|
+
return `${currentUrl.substring(0, 50)}...`;
|
|
222
238
|
}
|
|
223
239
|
})();
|
|
224
240
|
|
|
@@ -239,10 +255,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
239
255
|
console.warn(formatLogMessage('warn', `[cdp] Failed to attach CDP session for ${currentUrl}: ${cdpErr.message}`));
|
|
240
256
|
|
|
241
257
|
// Return null session with no-op cleanup for consistent API
|
|
242
|
-
return
|
|
243
|
-
session: null,
|
|
244
|
-
cleanup: async () => {}
|
|
245
|
-
};
|
|
258
|
+
return createSessionResult();
|
|
246
259
|
}
|
|
247
260
|
}
|
|
248
261
|
|
|
@@ -278,17 +291,25 @@ function validateCDPConfig(siteConfig, globalCDP, cdpSpecificDomains = []) {
|
|
|
278
291
|
warnings.push('cdp_specific is empty - no domains will have CDP enabled');
|
|
279
292
|
} else {
|
|
280
293
|
// Validate domain format
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
294
|
+
const hasInvalidDomains = siteConfig.cdp_specific.some(domain =>
|
|
295
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
296
|
+
);
|
|
297
|
+
|
|
298
|
+
if (hasInvalidDomains) {
|
|
299
|
+
// Only filter invalid domains if we need to show them
|
|
300
|
+
const invalidDomains = siteConfig.cdp_specific.filter(domain =>
|
|
301
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
302
|
+
);
|
|
285
303
|
warnings.push(`cdp_specific contains invalid domains: ${invalidDomains.join(', ')}`);
|
|
286
304
|
}
|
|
287
305
|
}
|
|
288
306
|
}
|
|
289
307
|
|
|
290
308
|
// Performance recommendations
|
|
291
|
-
|
|
309
|
+
const cdpEnabled = globalCDP || siteConfig.cdp === true ||
|
|
310
|
+
(Array.isArray(siteConfig.cdp_specific) && siteConfig.cdp_specific.length > 0);
|
|
311
|
+
|
|
312
|
+
if (cdpEnabled) {
|
|
292
313
|
recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
|
|
293
314
|
|
|
294
315
|
if (siteConfig.timeout && siteConfig.timeout < 30000) {
|
|
@@ -332,7 +353,8 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
332
353
|
const basicSession = await createCDPSession(page, currentUrl, options);
|
|
333
354
|
|
|
334
355
|
if (!basicSession.session) {
|
|
335
|
-
|
|
356
|
+
// Ensure enhanced flag is set even for null sessions
|
|
357
|
+
return { ...basicSession, isEnhanced: false };
|
|
336
358
|
}
|
|
337
359
|
|
|
338
360
|
const { session } = basicSession;
|
|
@@ -377,7 +399,7 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
377
399
|
|
|
378
400
|
// Graceful degradation: return basic session if enhanced features fail
|
|
379
401
|
// This ensures your application continues working even if advanced features break
|
|
380
|
-
return basicSession;
|
|
402
|
+
return { ...basicSession, isEnhanced: false };
|
|
381
403
|
}
|
|
382
404
|
}
|
|
383
405
|
|
package/lib/nettools.js
CHANGED
|
@@ -111,7 +111,7 @@ function execWithTimeout(command, timeout = 10000) {
|
|
|
111
111
|
* @param {string} mode - Selection mode: 'random' (default) or 'cycle'
|
|
112
112
|
* @returns {string|null} Selected whois server or null if none specified
|
|
113
113
|
*/
|
|
114
|
-
function selectWhoisServer(whoisServer, mode = 'random'){
|
|
114
|
+
function selectWhoisServer(whoisServer = '', mode = 'random'){
|
|
115
115
|
if (!whoisServer) {
|
|
116
116
|
return null; // Use default whois behavior
|
|
117
117
|
}
|
|
@@ -201,7 +201,7 @@ function suggestWhoisServers(domain, failedServer = null) {
|
|
|
201
201
|
* @param {boolean} debugMode - Enable debug logging (default: false)
|
|
202
202
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
203
203
|
*/
|
|
204
|
-
async function whoisLookup(domain, timeout = 10000, whoisServer =
|
|
204
|
+
async function whoisLookup(domain = '', timeout = 10000, whoisServer = '', debugMode = false, logFunc = null) {
|
|
205
205
|
const startTime = Date.now();
|
|
206
206
|
let cleanDomain, selectedServer, whoisCommand;
|
|
207
207
|
|
|
@@ -357,7 +357,7 @@ async function whoisLookup(domain, timeout = 10000, whoisServer = null, debugMod
|
|
|
357
357
|
* @param {number} whoisDelay - Delay in milliseconds before whois requests (default: 2000)
|
|
358
358
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
359
359
|
*/
|
|
360
|
-
async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer =
|
|
360
|
+
async function whoisLookupWithRetry(domain = '', timeout = 10000, whoisServer = '', debugMode = false, retryOptions = {}, whoisDelay = 8000, logFunc = null) {
|
|
361
361
|
const {
|
|
362
362
|
maxRetries = 3,
|
|
363
363
|
timeoutMultiplier = 1.5,
|
|
@@ -367,184 +367,240 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
367
367
|
} = retryOptions;
|
|
368
368
|
|
|
369
369
|
let serversToTry = [];
|
|
370
|
-
let currentTimeout = timeout;
|
|
371
370
|
|
|
372
371
|
// Build list of servers to try
|
|
373
|
-
if (whoisServer) {
|
|
372
|
+
if (whoisServer && whoisServer !== '') {
|
|
374
373
|
if (Array.isArray(whoisServer)) {
|
|
375
374
|
serversToTry = [...whoisServer]; // Copy array to avoid modifying original
|
|
376
375
|
} else {
|
|
377
376
|
serversToTry = [whoisServer];
|
|
378
377
|
}
|
|
379
378
|
} else {
|
|
380
|
-
serversToTry = [
|
|
379
|
+
serversToTry = ['']; // Default server (empty string instead of null)
|
|
381
380
|
}
|
|
382
381
|
|
|
383
382
|
// Add fallback servers if enabled and we have custom servers
|
|
384
|
-
if (useFallbackServers && whoisServer) {
|
|
383
|
+
if (useFallbackServers && whoisServer && whoisServer !== '') {
|
|
385
384
|
const fallbacks = suggestWhoisServers(domain).slice(0, 3);
|
|
386
385
|
// Only add fallbacks that aren't already in our list
|
|
387
|
-
const existingServers = serversToTry.filter(s => s !==
|
|
388
|
-
|
|
386
|
+
const existingServers = serversToTry.filter(s => s !== '');
|
|
387
|
+
const existingServerCount = existingServers.length;
|
|
388
|
+
const newFallbacks = fallbacks.filter(fb => {
|
|
389
|
+
for (let i = 0; i < existingServerCount; i++) {
|
|
390
|
+
if (existingServers[i] === fb) return false;
|
|
391
|
+
}
|
|
392
|
+
return true;
|
|
393
|
+
});
|
|
389
394
|
serversToTry.push(...newFallbacks);
|
|
390
395
|
}
|
|
391
396
|
|
|
392
397
|
let lastError = null;
|
|
393
|
-
let
|
|
398
|
+
let totalAttempts = 0;
|
|
399
|
+
let serversAttempted = [];
|
|
394
400
|
|
|
395
401
|
if (debugMode) {
|
|
402
|
+
const totalServers = serversToTry.length;
|
|
396
403
|
if (logFunc) {
|
|
397
|
-
|
|
404
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`);
|
|
398
405
|
logFunc(`${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`);
|
|
399
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries}, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`);
|
|
406
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`);
|
|
400
407
|
} else {
|
|
401
|
-
|
|
408
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`));
|
|
402
409
|
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`));
|
|
403
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries}, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`));
|
|
410
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`));
|
|
404
411
|
}
|
|
405
412
|
}
|
|
406
413
|
|
|
407
|
-
|
|
408
|
-
|
|
414
|
+
// Try each server with retry logic
|
|
415
|
+
const serverCount = serversToTry.length;
|
|
416
|
+
for (let serverIndex = 0; serverIndex < serverCount; serverIndex++) {
|
|
417
|
+
const server = serversToTry[serverIndex];
|
|
418
|
+
let currentTimeout = timeout;
|
|
419
|
+
let retryCount = 0;
|
|
420
|
+
serversAttempted.push(server);
|
|
409
421
|
|
|
410
422
|
if (debugMode) {
|
|
411
|
-
const serverName = server
|
|
423
|
+
const serverName = (server && server !== '') ? server : 'default';
|
|
412
424
|
if (logFunc) {
|
|
413
|
-
logFunc(`${messageColors.highlight('[whois-retry]')}
|
|
425
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`);
|
|
414
426
|
} else {
|
|
415
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')}
|
|
427
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`));
|
|
416
428
|
}
|
|
417
429
|
}
|
|
418
430
|
|
|
419
|
-
//
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
if (logFunc) {
|
|
424
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`);
|
|
425
|
-
} else {
|
|
426
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before retry attempt...`));
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
await new Promise(resolve => setTimeout(resolve, whoisDelay));
|
|
430
|
-
}
|
|
431
|
+
// Retry this server up to maxRetries times
|
|
432
|
+
while (retryCount < maxRetries) {
|
|
433
|
+
totalAttempts++;
|
|
434
|
+
const attemptNum = retryCount + 1;
|
|
431
435
|
|
|
432
|
-
} else if (whoisDelay > 0) {
|
|
433
|
-
// Add initial delay on first attempt if configured
|
|
434
436
|
if (debugMode) {
|
|
437
|
+
const serverName = (server && server !== '') ? server : 'default';
|
|
435
438
|
if (logFunc) {
|
|
436
|
-
logFunc(`${messageColors.highlight('[whois-retry]')}
|
|
439
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Attempt ${attemptNum}/${maxRetries} on server ${serverName} (timeout: ${currentTimeout}ms)`);
|
|
437
440
|
} else {
|
|
438
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')}
|
|
441
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Attempt ${attemptNum}/${maxRetries} on server ${serverName} (timeout: ${currentTimeout}ms)`));
|
|
439
442
|
}
|
|
440
443
|
}
|
|
441
|
-
await new Promise(resolve => setTimeout(resolve, whoisDelay));
|
|
442
|
-
} else if (debugMode) {
|
|
443
|
-
// Log when delay is skipped due to whoisDelay being 0
|
|
444
|
-
if (logFunc) {
|
|
445
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`);
|
|
446
|
-
} else {
|
|
447
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`));
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
try {
|
|
452
|
-
const result = await whoisLookup(domain, currentTimeout, server, debugMode, logFunc);
|
|
453
444
|
|
|
454
|
-
|
|
445
|
+
// Add progressive delay between retries (but not before first attempt on any server)
|
|
446
|
+
if (retryCount > 0 && whoisDelay > 0) {
|
|
447
|
+
// Progressive delay: base delay * retry attempt number + extra delay
|
|
448
|
+
// Attempt 2: base delay * 1 + 4000ms = 8000ms + 4000ms = 12000ms
|
|
449
|
+
// Attempt 3: base delay * 2 + 6000ms = 16000ms + 6000ms = 22000ms
|
|
450
|
+
// Attempt 4+: base delay * 3 + 6000ms = 24000ms + 6000ms = 30000ms (if maxRetries > 3)
|
|
451
|
+
const delayMultiplier = Math.min(retryCount, 3);
|
|
452
|
+
const baseDelay = whoisDelay * delayMultiplier;
|
|
453
|
+
|
|
454
|
+
// Add extra delay based on retry attempt
|
|
455
|
+
let extraDelay = 0;
|
|
456
|
+
if (retryCount === 1) {
|
|
457
|
+
extraDelay = 4000; // Extra 4 seconds for 2nd attempt
|
|
458
|
+
} else if (retryCount >= 2) {
|
|
459
|
+
extraDelay = 6000; // Extra 6 seconds for 3rd+ attempts
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
const actualDelay = baseDelay + extraDelay;
|
|
463
|
+
|
|
464
|
+
if (debugMode) {
|
|
465
|
+
if (logFunc) {
|
|
466
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${actualDelay}ms progressive delay before retry ${retryCount + 1} (base: ${baseDelay}ms + extra: ${extraDelay}ms)...`);
|
|
467
|
+
} else {
|
|
468
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${actualDelay}ms progressive delay before retry ${retryCount + 1} (base: ${baseDelay}ms + extra: ${extraDelay}ms)...`));
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
await new Promise(resolve => setTimeout(resolve, actualDelay));
|
|
472
|
+
} else if (serverIndex > 0 && retryCount === 0 && whoisDelay > 0) {
|
|
473
|
+
// Add delay before trying a new server (but not the very first server)
|
|
455
474
|
if (debugMode) {
|
|
456
475
|
if (logFunc) {
|
|
457
|
-
logFunc(`${messageColors.highlight('[whois-retry]')}
|
|
476
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before trying new server...`);
|
|
458
477
|
} else {
|
|
459
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')}
|
|
478
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before trying new server...`));
|
|
460
479
|
}
|
|
461
480
|
}
|
|
481
|
+
await new Promise(resolve => setTimeout(resolve, whoisDelay));
|
|
482
|
+
} else if (debugMode && whoisDelay === 0) {
|
|
483
|
+
// Log when delay is skipped due to whoisDelay being 0
|
|
484
|
+
if (logFunc) {
|
|
485
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`);
|
|
486
|
+
} else {
|
|
487
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`));
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
try {
|
|
492
|
+
const result = await whoisLookup(domain, currentTimeout, server || '', debugMode, logFunc);
|
|
462
493
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
finalServer: result.whoisServer,
|
|
471
|
-
retriedAfterFailure: attemptCount > 1
|
|
494
|
+
if (result.success) {
|
|
495
|
+
if (debugMode) {
|
|
496
|
+
if (logFunc) {
|
|
497
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} SUCCESS on attempt ${attemptNum}/${maxRetries} for server ${result.whoisServer || 'default'} (total attempts: ${totalAttempts})`);
|
|
498
|
+
} else {
|
|
499
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} SUCCESS on attempt ${attemptNum}/${maxRetries} for server ${result.whoisServer || 'default'} (total attempts: ${totalAttempts})`));
|
|
500
|
+
}
|
|
472
501
|
}
|
|
473
|
-
|
|
474
|
-
|
|
502
|
+
|
|
503
|
+
// Add retry info to result
|
|
504
|
+
// V8 Optimized: Object.assign performs better than spread
|
|
505
|
+
return Object.assign({}, result, {
|
|
506
|
+
retryInfo: {
|
|
507
|
+
totalAttempts: totalAttempts,
|
|
508
|
+
maxAttempts: serverCount * maxRetries,
|
|
509
|
+
serversAttempted: serversAttempted,
|
|
510
|
+
finalServer: result.whoisServer,
|
|
511
|
+
retriedAfterFailure: totalAttempts > 1,
|
|
512
|
+
serverRetries: retryCount,
|
|
513
|
+
serverIndex: serverIndex
|
|
514
|
+
}
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
475
518
|
// Determine if we should retry based on error type
|
|
476
519
|
const shouldRetry = (result.isTimeout && retryOnTimeout) || (!result.isTimeout && retryOnError);
|
|
477
520
|
|
|
478
521
|
if (debugMode) {
|
|
479
|
-
const serverName = result.whoisServer
|
|
522
|
+
const serverName = (result.whoisServer && result.whoisServer !== '') ? result.whoisServer : 'default';
|
|
480
523
|
const errorType = result.isTimeout ? 'TIMEOUT' : 'ERROR';
|
|
481
524
|
if (logFunc) {
|
|
482
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${
|
|
525
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${result.error}`);
|
|
483
526
|
} else {
|
|
484
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${
|
|
527
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${result.error}`));
|
|
485
528
|
}
|
|
486
529
|
|
|
487
|
-
if (
|
|
530
|
+
if (retryCount < maxRetries - 1) {
|
|
488
531
|
if (shouldRetry) {
|
|
489
532
|
if (logFunc) {
|
|
490
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Will retry
|
|
533
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Will retry attempt ${attemptNum + 1}/${maxRetries} on same server...`);
|
|
491
534
|
} else {
|
|
492
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Will retry
|
|
535
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Will retry attempt ${attemptNum + 1}/${maxRetries} on same server...`));
|
|
493
536
|
}
|
|
494
537
|
} else {
|
|
495
538
|
if (logFunc) {
|
|
496
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Skipping retry (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`);
|
|
539
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Skipping retry on same server (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`);
|
|
497
540
|
} else {
|
|
498
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping retry (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`));
|
|
541
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping retry on same server (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`));
|
|
499
542
|
}
|
|
500
543
|
}
|
|
544
|
+
} else if (serverIndex < serverCount - 1) {
|
|
545
|
+
if (logFunc) {
|
|
546
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`);
|
|
547
|
+
} else {
|
|
548
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`)); }
|
|
501
549
|
}
|
|
502
550
|
}
|
|
503
551
|
|
|
504
552
|
lastError = result;
|
|
505
553
|
|
|
506
|
-
// If this is the last server or we shouldn't retry this error type, break
|
|
507
|
-
if (
|
|
554
|
+
// If this is the last retry for this server or we shouldn't retry this error type, break to next server
|
|
555
|
+
if (retryCount >= maxRetries - 1 || !shouldRetry) {
|
|
508
556
|
break;
|
|
509
557
|
}
|
|
510
558
|
|
|
511
|
-
// Increase timeout for next attempt
|
|
559
|
+
// Increase timeout for next retry attempt on same server
|
|
560
|
+
retryCount++;
|
|
512
561
|
currentTimeout = Math.round(currentTimeout * timeoutMultiplier);
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
562
|
+
|
|
563
|
+
} catch (error) {
|
|
564
|
+
if (debugMode) {
|
|
565
|
+
const serverName = (server && server !== '') ? server : 'default'
|
|
566
|
+
if (logFunc) {
|
|
567
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} EXCEPTION on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${error.message}`);
|
|
568
|
+
} else {
|
|
569
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} EXCEPTION on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${error.message}`));
|
|
570
|
+
}
|
|
521
571
|
}
|
|
572
|
+
|
|
573
|
+
lastError = {
|
|
574
|
+
success: false,
|
|
575
|
+
error: error.message,
|
|
576
|
+
domain: domain,
|
|
577
|
+
whoisServer: server || '',
|
|
578
|
+
isTimeout: error.message.includes('timeout'),
|
|
579
|
+
duration: 0
|
|
580
|
+
};
|
|
581
|
+
|
|
582
|
+
// For exceptions, only retry if it's a retryable error type
|
|
583
|
+
const isRetryableException = error.message.includes('timeout') ||
|
|
584
|
+
error.message.includes('ECONNRESET') ||
|
|
585
|
+
error.message.includes('ENOTFOUND');
|
|
586
|
+
|
|
587
|
+
if (retryCount >= maxRetries - 1 || !isRetryableException) {
|
|
588
|
+
break;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
retryCount++;
|
|
592
|
+
currentTimeout = Math.round(currentTimeout * timeoutMultiplier);
|
|
522
593
|
}
|
|
523
|
-
|
|
524
|
-
lastError = {
|
|
525
|
-
success: false,
|
|
526
|
-
error: error.message,
|
|
527
|
-
domain: domain,
|
|
528
|
-
whoisServer: server,
|
|
529
|
-
isTimeout: error.message.includes('timeout'),
|
|
530
|
-
duration: 0
|
|
531
|
-
};
|
|
532
|
-
|
|
533
|
-
// Continue to next server unless this is the last one
|
|
534
|
-
if (attemptCount >= serversToTry.length) {
|
|
535
|
-
break;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
currentTimeout = Math.round(currentTimeout * timeoutMultiplier);
|
|
539
594
|
}
|
|
540
595
|
}
|
|
541
596
|
|
|
542
597
|
// All attempts failed
|
|
543
598
|
if (debugMode) {
|
|
599
|
+
const attemptedServerCount = serversAttempted.length;
|
|
544
600
|
if (logFunc) {
|
|
545
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${
|
|
601
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`);
|
|
546
602
|
} else {
|
|
547
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${
|
|
603
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`));
|
|
548
604
|
}
|
|
549
605
|
if (lastError) {
|
|
550
606
|
if (logFunc) {
|
|
@@ -556,17 +612,17 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
556
612
|
}
|
|
557
613
|
|
|
558
614
|
// Return the last error with retry info
|
|
559
|
-
|
|
560
|
-
|
|
615
|
+
// V8 Optimized: Object.assign instead of spread operator
|
|
616
|
+
return Object.assign({}, lastError, {
|
|
561
617
|
retryInfo: {
|
|
562
|
-
totalAttempts:
|
|
563
|
-
maxAttempts:
|
|
564
|
-
serversAttempted:
|
|
565
|
-
finalServer: lastError?.whoisServer ||
|
|
566
|
-
retriedAfterFailure:
|
|
618
|
+
totalAttempts: totalAttempts,
|
|
619
|
+
maxAttempts: serverCount * maxRetries,
|
|
620
|
+
serversAttempted: serversAttempted,
|
|
621
|
+
finalServer: lastError?.whoisServer || '',
|
|
622
|
+
retriedAfterFailure: totalAttempts > 1,
|
|
567
623
|
allAttemptsFailed: true
|
|
568
624
|
}
|
|
569
|
-
};
|
|
625
|
+
});
|
|
570
626
|
}
|
|
571
627
|
|
|
572
628
|
/**
|
|
@@ -576,7 +632,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
576
632
|
* @param {number} timeout - Timeout in milliseconds (default: 5000)
|
|
577
633
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
578
634
|
*/
|
|
579
|
-
async function digLookup(domain, recordType = 'A', timeout = 5000) {
|
|
635
|
+
async function digLookup(domain = '', recordType = 'A', timeout = 5000) {
|
|
580
636
|
try {
|
|
581
637
|
// Clean domain
|
|
582
638
|
const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
|
|
@@ -826,7 +882,7 @@ function createNetToolsHandler(config) {
|
|
|
826
882
|
if (forceDebug) {
|
|
827
883
|
logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Overall timeout for domain ${domain}, continuing with next...`);
|
|
828
884
|
}
|
|
829
|
-
},
|
|
885
|
+
}, 65000); // 65 second overall timeout
|
|
830
886
|
|
|
831
887
|
// Wrap entire function in timeout protection
|
|
832
888
|
return Promise.race([
|
|
@@ -837,7 +893,7 @@ function createNetToolsHandler(config) {
|
|
|
837
893
|
clearTimeout(netlookupTimeout);
|
|
838
894
|
}
|
|
839
895
|
})(),
|
|
840
|
-
new Promise((_, reject) => setTimeout(() => reject(new Error('NetTools overall timeout')),
|
|
896
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('NetTools overall timeout')), 65000))
|
|
841
897
|
]).catch(err => {
|
|
842
898
|
if (forceDebug) {
|
|
843
899
|
logToConsoleAndFile(`${messageColors.highlight('[nettools]')} ${err.message} for ${domain}, continuing...`);
|
|
@@ -904,7 +960,7 @@ function createNetToolsHandler(config) {
|
|
|
904
960
|
|
|
905
961
|
// Check whois cache first - cache key includes server for accuracy
|
|
906
962
|
const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
|
|
907
|
-
const whoisCacheKey = `${domain}-${selectedServer
|
|
963
|
+
const whoisCacheKey = `${domain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
|
|
908
964
|
const now = Date.now();
|
|
909
965
|
let whoisResult = null;
|
|
910
966
|
|
|
@@ -913,16 +969,15 @@ function createNetToolsHandler(config) {
|
|
|
913
969
|
if (now - cachedEntry.timestamp < WHOIS_CACHE_TTL) {
|
|
914
970
|
if (forceDebug) {
|
|
915
971
|
const age = Math.round((now - cachedEntry.timestamp) / 1000);
|
|
916
|
-
const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
|
|
972
|
+
const serverInfo = (selectedServer && selectedServer !== '') ? ` (server: ${selectedServer})` : ' (default server)';
|
|
917
973
|
logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
|
|
918
974
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
// Add cache metadata
|
|
975
|
+
// V8 Optimized: Object.assign is faster than spread for object merging
|
|
976
|
+
whoisResult = Object.assign({}, cachedEntry.result, {
|
|
922
977
|
fromCache: true,
|
|
923
978
|
cacheAge: now - cachedEntry.timestamp,
|
|
924
979
|
originalTimestamp: cachedEntry.timestamp
|
|
925
|
-
};
|
|
980
|
+
});
|
|
926
981
|
} else {
|
|
927
982
|
// Cache expired, remove it
|
|
928
983
|
whoisResultCache.delete(whoisCacheKey);
|
|
@@ -935,7 +990,7 @@ function createNetToolsHandler(config) {
|
|
|
935
990
|
// Perform fresh lookup if not cached
|
|
936
991
|
if (!whoisResult) {
|
|
937
992
|
if (forceDebug) {
|
|
938
|
-
const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
|
|
993
|
+
const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
|
|
939
994
|
logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
|
|
940
995
|
}
|
|
941
996
|
|
|
@@ -1357,4 +1412,4 @@ module.exports = {
|
|
|
1357
1412
|
getCommonWhoisServers,
|
|
1358
1413
|
suggestWhoisServers,
|
|
1359
1414
|
execWithTimeout // Export for testing
|
|
1360
|
-
};
|
|
1415
|
+
};
|
package/nwss.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// === Network scanner script (nwss.js) v2.0.
|
|
1
|
+
// === Network scanner script (nwss.js) v2.0.27 ===
|
|
2
2
|
|
|
3
3
|
// puppeteer for browser automation, fs for file system operations, psl for domain parsing.
|
|
4
4
|
// const pLimit = require('p-limit'); // Will be dynamically imported
|
|
@@ -143,7 +143,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
|
|
|
143
143
|
const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive, performGroupWindowCleanup, performRealtimeWindowCleanup, trackPageForRealtime, updatePageUsage, cleanupPageBeforeReload } = require('./lib/browserhealth');
|
|
144
144
|
|
|
145
145
|
// --- Script Configuration & Constants ---
|
|
146
|
-
const VERSION = '2.0.
|
|
146
|
+
const VERSION = '2.0.27'; // Script version
|
|
147
147
|
|
|
148
148
|
// get startTime
|
|
149
149
|
const startTime = Date.now();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.28",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|