@fanboynz/network-scanner 2.0.27 → 2.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/lib/browserhealth.js +139 -57
- package/lib/cdp.js +60 -38
- package/lib/nettools.js +51 -44
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -281,6 +281,36 @@ These options go at the root level of your config.json:
|
|
|
281
281
|
| `cache_requests` | Boolean | `false` | Enable HTTP request response caching |
|
|
282
282
|
|
|
283
283
|
---
|
|
284
|
+
#### Special Characters in searchstring
|
|
285
|
+
|
|
286
|
+
The `searchstring` parameter supports all characters including special symbols. Only double quotes need JSON escaping:
|
|
287
|
+
|
|
288
|
+
```json
|
|
289
|
+
{
|
|
290
|
+
"searchstring": [
|
|
291
|
+
")}return n}function N(n,e,r){try{\"function\"==typeof",
|
|
292
|
+
"addEventListener(\"click\",function(){",
|
|
293
|
+
"{\"status\":\"success\",\"data\":[",
|
|
294
|
+
"console.log('Debug: ' + value);",
|
|
295
|
+
"`API endpoint: ${baseUrl}/users`",
|
|
296
|
+
"@media screen and (max-width: 768px)",
|
|
297
|
+
"if(e&&e.preventDefault){e.preventDefault()}",
|
|
298
|
+
"__webpack_require__(/*! ./module */ \"./src/module.js\")",
|
|
299
|
+
"console.log('Hello world')",
|
|
300
|
+
"#header { background-color: #ff0000; }",
|
|
301
|
+
"$(document).ready(function() {",
|
|
302
|
+
"completion: 85% @ $1,500 budget",
|
|
303
|
+
"SELECT * FROM users WHERE id = *",
|
|
304
|
+
"regex: ^[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}$",
|
|
305
|
+
"typeof window !== 'undefined'"
|
|
306
|
+
]
|
|
307
|
+
}
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
**Character escaping rules:**
|
|
311
|
+
- `"` becomes `\"` (required in JSON)
|
|
312
|
+
- `\` becomes `\\` (if searching for literal backslashes)
|
|
313
|
+
- All other characters are used literally: `'` `` ` `` `@` `#` `$` `%` `*` `^` `[` `]` `{` `}` `(` `)` `;` `=` `!` `?` `:`
|
|
284
314
|
|
|
285
315
|
## Usage Examples
|
|
286
316
|
|
package/lib/browserhealth.js
CHANGED
|
@@ -33,6 +33,18 @@ const PAGE_IDLE_THRESHOLD = 25000; // 25 seconds of inactivity before considerin
|
|
|
33
33
|
async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug, cleanupMode = true) {
|
|
34
34
|
try {
|
|
35
35
|
// Wait before cleanup to allow any final operations to complete
|
|
36
|
+
// Initialize result object with ALL possible properties upfront for V8 optimization
|
|
37
|
+
const result = {
|
|
38
|
+
success: false,
|
|
39
|
+
closedCount: 0,
|
|
40
|
+
totalPages: 0,
|
|
41
|
+
mainPagePreserved: false,
|
|
42
|
+
delayUsed: 0,
|
|
43
|
+
estimatedMemoryFreed: 0,
|
|
44
|
+
estimatedMemoryFreedFormatted: '',
|
|
45
|
+
cleanupMode: '',
|
|
46
|
+
error: null
|
|
47
|
+
};
|
|
36
48
|
const modeText = cleanupMode === "all" ? "aggressive cleanup of old windows" : "conservative cleanup of extra windows"
|
|
37
49
|
if (forceDebug) {
|
|
38
50
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before ${modeText} for group: ${groupDescription}`));
|
|
@@ -47,8 +59,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
47
59
|
|
|
48
60
|
// Find the main page - typically the first page that's about:blank or has been there longest
|
|
49
61
|
for (const page of allPages) {
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
// Cache page.url() call to avoid repeated DOM/browser communication
|
|
63
|
+
const pageUrl = page.url();
|
|
64
|
+
if (pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://')) {
|
|
52
65
|
if (!mainPuppeteerPage) {
|
|
53
66
|
mainPuppeteerPage = page; // First blank page is likely the main window
|
|
54
67
|
} else {
|
|
@@ -56,6 +69,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
56
69
|
}
|
|
57
70
|
} else {
|
|
58
71
|
// Any page with actual content should be evaluated for closure
|
|
72
|
+
// Cache page state checks to avoid multiple browser calls
|
|
73
|
+
const isPageClosed = page.isClosed();
|
|
74
|
+
|
|
59
75
|
if (cleanupMode === "all") {
|
|
60
76
|
// Aggressive mode: close all content pages
|
|
61
77
|
pagesToClose.push(page);
|
|
@@ -75,7 +91,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
75
91
|
mainPuppeteerPage = allPages[0]; // Fallback to first page
|
|
76
92
|
pagesToClose = allPages.slice(1);
|
|
77
93
|
if (forceDebug) {
|
|
78
|
-
|
|
94
|
+
// Cache URL call for logging
|
|
95
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
96
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] No blank page found, using first page as main: ${mainPageUrl}`));
|
|
79
97
|
}
|
|
80
98
|
}
|
|
81
99
|
|
|
@@ -83,14 +101,11 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
83
101
|
if (forceDebug) {
|
|
84
102
|
console.log(formatLogMessage('debug', `[group_window_cleanup] No windows to close for group: ${groupDescription}`));
|
|
85
103
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
mainPagePreserved: true,
|
|
92
|
-
cleanupMode: cleanupMode === "all" ? "all" : "default"
|
|
93
|
-
};
|
|
104
|
+
result.success = true;
|
|
105
|
+
result.totalPages = allPages.length;
|
|
106
|
+
result.mainPagePreserved = true;
|
|
107
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
108
|
+
return result;
|
|
94
109
|
}
|
|
95
110
|
|
|
96
111
|
// Estimate memory usage before closing
|
|
@@ -102,7 +117,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
102
117
|
let pageMemoryEstimate = 0;
|
|
103
118
|
|
|
104
119
|
try {
|
|
105
|
-
|
|
120
|
+
// Cache page.isClosed() check to avoid repeated browser calls
|
|
121
|
+
const isPageClosed = page.isClosed();
|
|
122
|
+
if (!isPageClosed) {
|
|
106
123
|
// Get page metrics if available
|
|
107
124
|
const metrics = await Promise.race([
|
|
108
125
|
page.metrics(),
|
|
@@ -138,12 +155,15 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
138
155
|
// Close identified old/unused pages
|
|
139
156
|
const closePromises = pagesToClose.map(async (page, index) => {
|
|
140
157
|
try {
|
|
141
|
-
|
|
158
|
+
// Cache page state and URL for this operation
|
|
159
|
+
const isPageClosed = page.isClosed();
|
|
160
|
+
const pageUrl = page.url();
|
|
161
|
+
if (!isPageClosed) {
|
|
142
162
|
if (forceDebug) {
|
|
143
|
-
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${
|
|
163
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${pageUrl}`));
|
|
144
164
|
}
|
|
145
165
|
await page.close();
|
|
146
|
-
return { success: true, url:
|
|
166
|
+
return { success: true, url: pageUrl || `page-${index}`, estimatedMemory: pageMemoryEstimates[index] };
|
|
147
167
|
}
|
|
148
168
|
return { success: false, reason: 'already_closed', estimatedMemory: 0 };
|
|
149
169
|
} catch (closeErr) {
|
|
@@ -177,25 +197,39 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
177
197
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${pagesToClose.length} old windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
|
|
178
198
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Estimated memory freed: ${formatMemory(actualMemoryFreed)}`));
|
|
179
199
|
if (mainPuppeteerPage) {
|
|
180
|
-
|
|
200
|
+
// Cache URL for final logging
|
|
201
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
202
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Main Puppeteer window preserved: ${mainPageUrl}`));
|
|
181
203
|
}
|
|
182
204
|
}
|
|
183
205
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
206
|
+
// Update result object instead of creating new one
|
|
207
|
+
result.success = true;
|
|
208
|
+
result.closedCount = successfulCloses;
|
|
209
|
+
result.totalPages = allPages.length;
|
|
210
|
+
result.mainPagePreserved = mainPuppeteerPage && !mainPuppeteerPage.isClosed();
|
|
211
|
+
result.delayUsed = WINDOW_CLEANUP_DELAY_MS;
|
|
212
|
+
result.estimatedMemoryFreed = actualMemoryFreed;
|
|
213
|
+
result.estimatedMemoryFreedFormatted = formatMemory(actualMemoryFreed);
|
|
214
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
215
|
+
return result;
|
|
194
216
|
} catch (cleanupErr) {
|
|
195
217
|
if (forceDebug) {
|
|
196
218
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Group cleanup failed for ${groupDescription}: ${cleanupErr.message}`));
|
|
197
219
|
}
|
|
198
|
-
|
|
220
|
+
// Initialize result object with consistent shape for error case
|
|
221
|
+
const result = {
|
|
222
|
+
success: false,
|
|
223
|
+
closedCount: 0,
|
|
224
|
+
totalPages: 0,
|
|
225
|
+
mainPagePreserved: false,
|
|
226
|
+
delayUsed: 0,
|
|
227
|
+
estimatedMemoryFreed: 0,
|
|
228
|
+
estimatedMemoryFreedFormatted: '',
|
|
229
|
+
cleanupMode: '',
|
|
230
|
+
error: cleanupErr.message
|
|
231
|
+
};
|
|
232
|
+
return result;
|
|
199
233
|
}
|
|
200
234
|
}
|
|
201
235
|
|
|
@@ -207,14 +241,17 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
207
241
|
*/
|
|
208
242
|
async function isPageSafeToClose(page, forceDebug) {
|
|
209
243
|
try {
|
|
210
|
-
|
|
244
|
+
// Cache page.isClosed() to avoid repeated browser communication
|
|
245
|
+
const isPageClosed = page.isClosed();
|
|
246
|
+
if (isPageClosed) {
|
|
211
247
|
return true; // Already closed
|
|
212
248
|
}
|
|
213
249
|
|
|
214
250
|
// EXTRA SAFETY: Never close pages that might be in injection process
|
|
215
251
|
try {
|
|
216
|
-
|
|
217
|
-
|
|
252
|
+
// Cache page.url() for safety checks
|
|
253
|
+
const pageUrl = page.url();
|
|
254
|
+
if (pageUrl && pageUrl !== 'about:blank' && Date.now() - (pageCreationTracker.get(page) || 0) < 30000) {
|
|
218
255
|
return false; // Don't close recently created pages (within 30 seconds)
|
|
219
256
|
}
|
|
220
257
|
} catch (err) { /* ignore */ }
|
|
@@ -228,7 +265,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
228
265
|
// Check if page is actively processing
|
|
229
266
|
if (usage.isProcessing) {
|
|
230
267
|
if (forceDebug) {
|
|
231
|
-
|
|
268
|
+
// Cache URL for debug logging
|
|
269
|
+
const pageUrl = page.url();
|
|
270
|
+
console.log(formatLogMessage('debug', `[realtime_cleanup] Page still processing: ${pageUrl.substring(0, 50)}...`));
|
|
232
271
|
}
|
|
233
272
|
return false;
|
|
234
273
|
}
|
|
@@ -257,7 +296,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
257
296
|
*/
|
|
258
297
|
function updatePageUsage(page, isProcessing = false) {
|
|
259
298
|
try {
|
|
260
|
-
|
|
299
|
+
// Cache page.isClosed() to avoid repeated calls
|
|
300
|
+
const isPageClosed = page.isClosed();
|
|
301
|
+
if (!isPageClosed) {
|
|
261
302
|
pageUsageTracker.set(page, {
|
|
262
303
|
lastActivity: Date.now(),
|
|
263
304
|
isProcessing: isProcessing
|
|
@@ -281,12 +322,27 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
281
322
|
try {
|
|
282
323
|
const allPages = await browserInstance.pages();
|
|
283
324
|
|
|
325
|
+
// Initialize result object with consistent shape
|
|
326
|
+
const result = {
|
|
327
|
+
success: false,
|
|
328
|
+
closedCount: 0,
|
|
329
|
+
totalPages: 0,
|
|
330
|
+
remainingPages: 0,
|
|
331
|
+
threshold: 0,
|
|
332
|
+
cleanupDelay: 0,
|
|
333
|
+
reason: '',
|
|
334
|
+
error: null
|
|
335
|
+
};
|
|
336
|
+
|
|
284
337
|
// Skip cleanup if we don't have enough pages to warrant it
|
|
285
338
|
if (allPages.length <= Math.max(threshold, REALTIME_CLEANUP_MIN_PAGES)) {
|
|
286
339
|
if (forceDebug) {
|
|
287
340
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Only ${allPages.length} pages open, threshold is ${threshold} - no cleanup needed`));
|
|
288
341
|
}
|
|
289
|
-
|
|
342
|
+
result.success = true;
|
|
343
|
+
result.totalPages = allPages.length;
|
|
344
|
+
result.reason = 'below_threshold';
|
|
345
|
+
return result;
|
|
290
346
|
}
|
|
291
347
|
|
|
292
348
|
// Use the provided total delay (already includes appropriate buffer)
|
|
@@ -324,13 +380,18 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
324
380
|
|
|
325
381
|
// Find main Puppeteer page (usually about:blank)
|
|
326
382
|
let mainPage = allPagesAfterDelay.find(page => {
|
|
327
|
-
|
|
328
|
-
|
|
383
|
+
// Cache page.url() for main page detection
|
|
384
|
+
const pageUrl = page.url();
|
|
385
|
+
return pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://');
|
|
329
386
|
}) || allPagesAfterDelay[0]; // Fallback to first page
|
|
330
387
|
|
|
331
388
|
// Get pages sorted by creation time (oldest first)
|
|
332
389
|
const sortedPages = allPagesAfterDelay
|
|
333
|
-
.filter(page =>
|
|
390
|
+
.filter(page => {
|
|
391
|
+
// Cache page.isClosed() for filtering
|
|
392
|
+
const isPageClosed = page.isClosed();
|
|
393
|
+
return page !== mainPage && !isPageClosed;
|
|
394
|
+
})
|
|
334
395
|
.sort((a, b) => {
|
|
335
396
|
const timeA = pageCreationTracker.get(a) || 0;
|
|
336
397
|
const timeB = pageCreationTracker.get(b) || 0;
|
|
@@ -360,15 +421,21 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
360
421
|
`${pagesToClose.length} pages still active`;
|
|
361
422
|
console.log(formatLogMessage('debug', `[realtime_cleanup] No pages need closing (${reason})`));
|
|
362
423
|
}
|
|
363
|
-
|
|
424
|
+
result.success = true;
|
|
425
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
426
|
+
result.reason = 'no_cleanup_needed';
|
|
427
|
+
return result;
|
|
364
428
|
}
|
|
365
429
|
|
|
366
430
|
// Close oldest pages
|
|
367
431
|
let closedCount = 0;
|
|
368
432
|
for (const page of safePagesToClose) {
|
|
369
433
|
try {
|
|
370
|
-
|
|
371
|
-
|
|
434
|
+
// Cache both page state and URL for this iteration
|
|
435
|
+
const isPageClosed = page.isClosed();
|
|
436
|
+
const pageUrl = page.url();
|
|
437
|
+
|
|
438
|
+
if (!isPageClosed) {
|
|
372
439
|
await page.close();
|
|
373
440
|
pageCreationTracker.delete(page); // Remove from tracker
|
|
374
441
|
closedCount++;
|
|
@@ -390,20 +457,30 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
390
457
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Closed ${closedCount}/${pagesToClose.length} oldest pages (${unsafePagesCount} skipped for safety), ${remainingPages} pages remaining`));
|
|
391
458
|
}
|
|
392
459
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
};
|
|
460
|
+
result.success = true;
|
|
461
|
+
result.closedCount = closedCount;
|
|
462
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
463
|
+
result.remainingPages = remainingPages;
|
|
464
|
+
result.threshold = threshold;
|
|
465
|
+
result.cleanupDelay = cleanupDelay;
|
|
466
|
+
result.reason = 'cleanup_completed';
|
|
467
|
+
return result;
|
|
402
468
|
} catch (cleanupErr) {
|
|
403
469
|
if (forceDebug) {
|
|
404
470
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Cleanup failed: ${cleanupErr.message}`));
|
|
405
471
|
}
|
|
406
|
-
|
|
472
|
+
// Initialize result object with consistent shape for error case
|
|
473
|
+
const result = {
|
|
474
|
+
success: false,
|
|
475
|
+
closedCount: 0,
|
|
476
|
+
totalPages: 0,
|
|
477
|
+
remainingPages: 0,
|
|
478
|
+
threshold: 0,
|
|
479
|
+
cleanupDelay: 0,
|
|
480
|
+
reason: '',
|
|
481
|
+
error: cleanupErr.message
|
|
482
|
+
};
|
|
483
|
+
return result;
|
|
407
484
|
}
|
|
408
485
|
}
|
|
409
486
|
|
|
@@ -415,14 +492,15 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
415
492
|
*/
|
|
416
493
|
async function isPageFromPreviousScan(page, forceDebug) {
|
|
417
494
|
try {
|
|
418
|
-
|
|
495
|
+
// Cache page.url() for all checks in this function
|
|
496
|
+
const pageUrl = page.url();
|
|
419
497
|
|
|
420
498
|
// Always consider these as old/closeable
|
|
421
|
-
if (
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
499
|
+
if (pageUrl === 'about:blank' ||
|
|
500
|
+
pageUrl === '' ||
|
|
501
|
+
pageUrl.startsWith('chrome://') ||
|
|
502
|
+
pageUrl.startsWith('chrome-error://') ||
|
|
503
|
+
pageUrl.startsWith('data:')) {
|
|
426
504
|
return false; // Don't close blank pages here, handled separately
|
|
427
505
|
}
|
|
428
506
|
|
|
@@ -446,7 +524,9 @@ async function isPageFromPreviousScan(page, forceDebug) {
|
|
|
446
524
|
return false; // Conservative - don't close unless we're sure
|
|
447
525
|
} catch (err) {
|
|
448
526
|
if (forceDebug) {
|
|
449
|
-
|
|
527
|
+
// Cache URL for error logging
|
|
528
|
+
const pageUrl = page.url();
|
|
529
|
+
console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${pageUrl}: ${err.message}`));
|
|
450
530
|
}
|
|
451
531
|
return false; // Conservative - don't close if we can't evaluate
|
|
452
532
|
}
|
|
@@ -1003,7 +1083,9 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
|
|
|
1003
1083
|
*/
|
|
1004
1084
|
async function cleanupPageBeforeReload(page, forceDebug = false) {
|
|
1005
1085
|
try {
|
|
1006
|
-
|
|
1086
|
+
// Cache page.isClosed() to avoid repeated browser calls
|
|
1087
|
+
const isPageClosed = page.isClosed();
|
|
1088
|
+
if (isPageClosed) {
|
|
1007
1089
|
return false;
|
|
1008
1090
|
}
|
|
1009
1091
|
|
package/lib/cdp.js
CHANGED
|
@@ -27,6 +27,31 @@
|
|
|
27
27
|
|
|
28
28
|
const { formatLogMessage } = require('./colorize');
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Creates a reusable timeout promise to reduce function allocation overhead
|
|
32
|
+
* @param {number} ms - Timeout in milliseconds
|
|
33
|
+
* @param {string} message - Error message for timeout
|
|
34
|
+
* @returns {Promise} Promise that rejects after timeout
|
|
35
|
+
*/
|
|
36
|
+
function createTimeoutPromise(ms, message) {
|
|
37
|
+
return new Promise((_, reject) =>
|
|
38
|
+
setTimeout(() => reject(new Error(message)), ms)
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Creates a standardized session result object for consistent V8 optimization
|
|
44
|
+
* @param {object|null} session - CDP session or null
|
|
45
|
+
* @param {Function} cleanup - Cleanup function
|
|
46
|
+
* @param {boolean} isEnhanced - Whether enhanced features are active
|
|
47
|
+
* @returns {object} Standardized session object
|
|
48
|
+
*/
|
|
49
|
+
const createSessionResult = (session = null, cleanup = async () => {}, isEnhanced = false) => ({
|
|
50
|
+
session,
|
|
51
|
+
cleanup,
|
|
52
|
+
isEnhanced
|
|
53
|
+
});
|
|
54
|
+
|
|
30
55
|
/**
|
|
31
56
|
* Creates a new page with timeout protection to prevent CDP hangs
|
|
32
57
|
* @param {import('puppeteer').Browser} browser - Browser instance
|
|
@@ -36,9 +61,7 @@ const { formatLogMessage } = require('./colorize');
|
|
|
36
61
|
async function createPageWithTimeout(browser, timeout = 30000) {
|
|
37
62
|
return Promise.race([
|
|
38
63
|
browser.newPage(),
|
|
39
|
-
|
|
40
|
-
setTimeout(() => reject(new Error('Page creation timeout - browser may be unresponsive')), timeout)
|
|
41
|
-
)
|
|
64
|
+
createTimeoutPromise(timeout, 'Page creation timeout - browser may be unresponsive')
|
|
42
65
|
]);
|
|
43
66
|
}
|
|
44
67
|
|
|
@@ -52,9 +75,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
52
75
|
try {
|
|
53
76
|
await Promise.race([
|
|
54
77
|
page.setRequestInterception(true),
|
|
55
|
-
|
|
56
|
-
setTimeout(() => reject(new Error('Request interception timeout - first attempt')), timeout)
|
|
57
|
-
)
|
|
78
|
+
createTimeoutPromise(timeout, 'Request interception timeout - first attempt')
|
|
58
79
|
]);
|
|
59
80
|
} catch (firstError) {
|
|
60
81
|
// Check for immediate critical failures
|
|
@@ -68,9 +89,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
68
89
|
try {
|
|
69
90
|
await Promise.race([
|
|
70
91
|
page.setRequestInterception(true),
|
|
71
|
-
|
|
72
|
-
setTimeout(() => reject(new Error('Request interception timeout - retry failed')), timeout * 2)
|
|
73
|
-
)
|
|
92
|
+
createTimeoutPromise(timeout * 2, 'Request interception timeout - retry failed')
|
|
74
93
|
]);
|
|
75
94
|
} catch (retryError) {
|
|
76
95
|
if (retryError.message.includes('Network.enable timed out') ||
|
|
@@ -128,7 +147,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
128
147
|
|
|
129
148
|
if (!cdpLoggingNeeded) {
|
|
130
149
|
// Return a null session with no-op cleanup for consistent API
|
|
131
|
-
return
|
|
150
|
+
return createSessionResult();
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
// Log which CDP mode is being used
|
|
@@ -151,9 +170,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
151
170
|
// Add timeout protection for CDP session creation
|
|
152
171
|
cdpSession = await Promise.race([
|
|
153
172
|
page.createCDPSession(),
|
|
154
|
-
|
|
155
|
-
setTimeout(() => reject(new Error('CDP session creation timeout')), 20000)
|
|
156
|
-
)
|
|
173
|
+
createTimeoutPromise(20000, 'CDP session creation timeout')
|
|
157
174
|
]);
|
|
158
175
|
|
|
159
176
|
// Enable network domain - required for network event monitoring
|
|
@@ -166,19 +183,17 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
166
183
|
const initiator = params.initiator ? params.initiator.type : 'unknown';
|
|
167
184
|
|
|
168
185
|
// Extract hostname for logging context (handles URL parsing errors gracefully)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
|
|
186
|
+
const hostnameForLog = (() => {
|
|
187
|
+
try {
|
|
188
|
+
const currentHostname = new URL(currentUrl).hostname;
|
|
189
|
+
const requestHostname = new URL(requestUrl).hostname;
|
|
190
|
+
return currentHostname !== requestHostname
|
|
191
|
+
? `${currentHostname}?${requestHostname}`
|
|
192
|
+
: currentHostname;
|
|
193
|
+
} catch (_) {
|
|
194
|
+
return 'unknown-host';
|
|
178
195
|
}
|
|
179
|
-
}
|
|
180
|
-
// Ignore URL parsing errors for logging context
|
|
181
|
-
}
|
|
196
|
+
})();
|
|
182
197
|
|
|
183
198
|
// Log the request with context only if debug mode is enabled
|
|
184
199
|
if (forceDebug) {
|
|
@@ -207,7 +222,8 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
207
222
|
}
|
|
208
223
|
}
|
|
209
224
|
}
|
|
210
|
-
}
|
|
225
|
+
},
|
|
226
|
+
isEnhanced: false
|
|
211
227
|
};
|
|
212
228
|
|
|
213
229
|
} catch (cdpErr) {
|
|
@@ -218,7 +234,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
218
234
|
try {
|
|
219
235
|
return new URL(currentUrl).hostname;
|
|
220
236
|
} catch {
|
|
221
|
-
return currentUrl.substring(0, 50)
|
|
237
|
+
return `${currentUrl.substring(0, 50)}...`;
|
|
222
238
|
}
|
|
223
239
|
})();
|
|
224
240
|
|
|
@@ -239,10 +255,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
239
255
|
console.warn(formatLogMessage('warn', `[cdp] Failed to attach CDP session for ${currentUrl}: ${cdpErr.message}`));
|
|
240
256
|
|
|
241
257
|
// Return null session with no-op cleanup for consistent API
|
|
242
|
-
return
|
|
243
|
-
session: null,
|
|
244
|
-
cleanup: async () => {}
|
|
245
|
-
};
|
|
258
|
+
return createSessionResult();
|
|
246
259
|
}
|
|
247
260
|
}
|
|
248
261
|
|
|
@@ -278,17 +291,25 @@ function validateCDPConfig(siteConfig, globalCDP, cdpSpecificDomains = []) {
|
|
|
278
291
|
warnings.push('cdp_specific is empty - no domains will have CDP enabled');
|
|
279
292
|
} else {
|
|
280
293
|
// Validate domain format
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
294
|
+
const hasInvalidDomains = siteConfig.cdp_specific.some(domain =>
|
|
295
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
296
|
+
);
|
|
297
|
+
|
|
298
|
+
if (hasInvalidDomains) {
|
|
299
|
+
// Only filter invalid domains if we need to show them
|
|
300
|
+
const invalidDomains = siteConfig.cdp_specific.filter(domain =>
|
|
301
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
302
|
+
);
|
|
285
303
|
warnings.push(`cdp_specific contains invalid domains: ${invalidDomains.join(', ')}`);
|
|
286
304
|
}
|
|
287
305
|
}
|
|
288
306
|
}
|
|
289
307
|
|
|
290
308
|
// Performance recommendations
|
|
291
|
-
|
|
309
|
+
const cdpEnabled = globalCDP || siteConfig.cdp === true ||
|
|
310
|
+
(Array.isArray(siteConfig.cdp_specific) && siteConfig.cdp_specific.length > 0);
|
|
311
|
+
|
|
312
|
+
if (cdpEnabled) {
|
|
292
313
|
recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
|
|
293
314
|
|
|
294
315
|
if (siteConfig.timeout && siteConfig.timeout < 30000) {
|
|
@@ -332,7 +353,8 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
332
353
|
const basicSession = await createCDPSession(page, currentUrl, options);
|
|
333
354
|
|
|
334
355
|
if (!basicSession.session) {
|
|
335
|
-
|
|
356
|
+
// Ensure enhanced flag is set even for null sessions
|
|
357
|
+
return { ...basicSession, isEnhanced: false };
|
|
336
358
|
}
|
|
337
359
|
|
|
338
360
|
const { session } = basicSession;
|
|
@@ -377,7 +399,7 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
377
399
|
|
|
378
400
|
// Graceful degradation: return basic session if enhanced features fail
|
|
379
401
|
// This ensures your application continues working even if advanced features break
|
|
380
|
-
return basicSession;
|
|
402
|
+
return { ...basicSession, isEnhanced: false };
|
|
381
403
|
}
|
|
382
404
|
}
|
|
383
405
|
|
package/lib/nettools.js
CHANGED
|
@@ -111,7 +111,7 @@ function execWithTimeout(command, timeout = 10000) {
|
|
|
111
111
|
* @param {string} mode - Selection mode: 'random' (default) or 'cycle'
|
|
112
112
|
* @returns {string|null} Selected whois server or null if none specified
|
|
113
113
|
*/
|
|
114
|
-
function selectWhoisServer(whoisServer, mode = 'random'){
|
|
114
|
+
function selectWhoisServer(whoisServer = '', mode = 'random'){
|
|
115
115
|
if (!whoisServer) {
|
|
116
116
|
return null; // Use default whois behavior
|
|
117
117
|
}
|
|
@@ -201,7 +201,7 @@ function suggestWhoisServers(domain, failedServer = null) {
|
|
|
201
201
|
* @param {boolean} debugMode - Enable debug logging (default: false)
|
|
202
202
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
203
203
|
*/
|
|
204
|
-
async function whoisLookup(domain, timeout = 10000, whoisServer =
|
|
204
|
+
async function whoisLookup(domain = '', timeout = 10000, whoisServer = '', debugMode = false, logFunc = null) {
|
|
205
205
|
const startTime = Date.now();
|
|
206
206
|
let cleanDomain, selectedServer, whoisCommand;
|
|
207
207
|
|
|
@@ -357,7 +357,7 @@ async function whoisLookup(domain, timeout = 10000, whoisServer = null, debugMod
|
|
|
357
357
|
* @param {number} whoisDelay - Delay in milliseconds before whois requests (default: 2000)
|
|
358
358
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
359
359
|
*/
|
|
360
|
-
async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer =
|
|
360
|
+
async function whoisLookupWithRetry(domain = '', timeout = 10000, whoisServer = '', debugMode = false, retryOptions = {}, whoisDelay = 8000, logFunc = null) {
|
|
361
361
|
const {
|
|
362
362
|
maxRetries = 3,
|
|
363
363
|
timeoutMultiplier = 1.5,
|
|
@@ -369,22 +369,28 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
369
369
|
let serversToTry = [];
|
|
370
370
|
|
|
371
371
|
// Build list of servers to try
|
|
372
|
-
if (whoisServer) {
|
|
372
|
+
if (whoisServer && whoisServer !== '') {
|
|
373
373
|
if (Array.isArray(whoisServer)) {
|
|
374
374
|
serversToTry = [...whoisServer]; // Copy array to avoid modifying original
|
|
375
375
|
} else {
|
|
376
376
|
serversToTry = [whoisServer];
|
|
377
377
|
}
|
|
378
378
|
} else {
|
|
379
|
-
serversToTry = [
|
|
379
|
+
serversToTry = ['']; // Default server (empty string instead of null)
|
|
380
380
|
}
|
|
381
381
|
|
|
382
382
|
// Add fallback servers if enabled and we have custom servers
|
|
383
|
-
if (useFallbackServers && whoisServer) {
|
|
383
|
+
if (useFallbackServers && whoisServer && whoisServer !== '') {
|
|
384
384
|
const fallbacks = suggestWhoisServers(domain).slice(0, 3);
|
|
385
385
|
// Only add fallbacks that aren't already in our list
|
|
386
|
-
const existingServers = serversToTry.filter(s => s !==
|
|
387
|
-
|
|
386
|
+
const existingServers = serversToTry.filter(s => s !== '');
|
|
387
|
+
const existingServerCount = existingServers.length;
|
|
388
|
+
const newFallbacks = fallbacks.filter(fb => {
|
|
389
|
+
for (let i = 0; i < existingServerCount; i++) {
|
|
390
|
+
if (existingServers[i] === fb) return false;
|
|
391
|
+
}
|
|
392
|
+
return true;
|
|
393
|
+
});
|
|
388
394
|
serversToTry.push(...newFallbacks);
|
|
389
395
|
}
|
|
390
396
|
|
|
@@ -393,30 +399,32 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
393
399
|
let serversAttempted = [];
|
|
394
400
|
|
|
395
401
|
if (debugMode) {
|
|
402
|
+
const totalServers = serversToTry.length;
|
|
396
403
|
if (logFunc) {
|
|
397
|
-
|
|
404
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`);
|
|
398
405
|
logFunc(`${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`);
|
|
399
406
|
logFunc(`${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`);
|
|
400
407
|
} else {
|
|
401
|
-
|
|
408
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`));
|
|
402
409
|
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`));
|
|
403
410
|
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`));
|
|
404
411
|
}
|
|
405
412
|
}
|
|
406
413
|
|
|
407
414
|
// Try each server with retry logic
|
|
408
|
-
|
|
415
|
+
const serverCount = serversToTry.length;
|
|
416
|
+
for (let serverIndex = 0; serverIndex < serverCount; serverIndex++) {
|
|
409
417
|
const server = serversToTry[serverIndex];
|
|
410
418
|
let currentTimeout = timeout;
|
|
411
419
|
let retryCount = 0;
|
|
412
420
|
serversAttempted.push(server);
|
|
413
421
|
|
|
414
422
|
if (debugMode) {
|
|
415
|
-
const serverName = server
|
|
423
|
+
const serverName = (server && server !== '') ? server : 'default';
|
|
416
424
|
if (logFunc) {
|
|
417
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${
|
|
425
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`);
|
|
418
426
|
} else {
|
|
419
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${
|
|
427
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`));
|
|
420
428
|
}
|
|
421
429
|
}
|
|
422
430
|
|
|
@@ -426,7 +434,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
426
434
|
const attemptNum = retryCount + 1;
|
|
427
435
|
|
|
428
436
|
if (debugMode) {
|
|
429
|
-
const serverName = server
|
|
437
|
+
const serverName = (server && server !== '') ? server : 'default';
|
|
430
438
|
if (logFunc) {
|
|
431
439
|
logFunc(`${messageColors.highlight('[whois-retry]')} Attempt ${attemptNum}/${maxRetries} on server ${serverName} (timeout: ${currentTimeout}ms)`);
|
|
432
440
|
} else {
|
|
@@ -481,7 +489,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
481
489
|
}
|
|
482
490
|
|
|
483
491
|
try {
|
|
484
|
-
const result = await whoisLookup(domain, currentTimeout, server, debugMode, logFunc);
|
|
492
|
+
const result = await whoisLookup(domain, currentTimeout, server || '', debugMode, logFunc);
|
|
485
493
|
|
|
486
494
|
if (result.success) {
|
|
487
495
|
if (debugMode) {
|
|
@@ -493,25 +501,25 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
493
501
|
}
|
|
494
502
|
|
|
495
503
|
// Add retry info to result
|
|
496
|
-
|
|
497
|
-
|
|
504
|
+
// V8 Optimized: Object.assign performs better than spread
|
|
505
|
+
return Object.assign({}, result, {
|
|
498
506
|
retryInfo: {
|
|
499
507
|
totalAttempts: totalAttempts,
|
|
500
|
-
maxAttempts:
|
|
508
|
+
maxAttempts: serverCount * maxRetries,
|
|
501
509
|
serversAttempted: serversAttempted,
|
|
502
510
|
finalServer: result.whoisServer,
|
|
503
511
|
retriedAfterFailure: totalAttempts > 1,
|
|
504
512
|
serverRetries: retryCount,
|
|
505
513
|
serverIndex: serverIndex
|
|
506
514
|
}
|
|
507
|
-
};
|
|
515
|
+
});
|
|
508
516
|
}
|
|
509
517
|
|
|
510
518
|
// Determine if we should retry based on error type
|
|
511
519
|
const shouldRetry = (result.isTimeout && retryOnTimeout) || (!result.isTimeout && retryOnError);
|
|
512
520
|
|
|
513
521
|
if (debugMode) {
|
|
514
|
-
const serverName = result.whoisServer
|
|
522
|
+
const serverName = (result.whoisServer && result.whoisServer !== '') ? result.whoisServer : 'default';
|
|
515
523
|
const errorType = result.isTimeout ? 'TIMEOUT' : 'ERROR';
|
|
516
524
|
if (logFunc) {
|
|
517
525
|
logFunc(`${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${result.error}`);
|
|
@@ -533,12 +541,11 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
533
541
|
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping retry on same server (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`));
|
|
534
542
|
}
|
|
535
543
|
}
|
|
536
|
-
} else if (serverIndex <
|
|
544
|
+
} else if (serverIndex < serverCount - 1) {
|
|
537
545
|
if (logFunc) {
|
|
538
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} Max retries reached for server, will try next server
|
|
546
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`);
|
|
539
547
|
} else {
|
|
540
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Max retries reached for server, will try next server
|
|
541
|
-
}
|
|
548
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`)); }
|
|
542
549
|
}
|
|
543
550
|
}
|
|
544
551
|
|
|
@@ -555,7 +562,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
555
562
|
|
|
556
563
|
} catch (error) {
|
|
557
564
|
if (debugMode) {
|
|
558
|
-
const serverName = server
|
|
565
|
+
const serverName = (server && server !== '') ? server : 'default'
|
|
559
566
|
if (logFunc) {
|
|
560
567
|
logFunc(`${messageColors.highlight('[whois-retry]')} EXCEPTION on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${error.message}`);
|
|
561
568
|
} else {
|
|
@@ -567,7 +574,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
567
574
|
success: false,
|
|
568
575
|
error: error.message,
|
|
569
576
|
domain: domain,
|
|
570
|
-
whoisServer: server,
|
|
577
|
+
whoisServer: server || '',
|
|
571
578
|
isTimeout: error.message.includes('timeout'),
|
|
572
579
|
duration: 0
|
|
573
580
|
};
|
|
@@ -589,10 +596,11 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
589
596
|
|
|
590
597
|
// All attempts failed
|
|
591
598
|
if (debugMode) {
|
|
599
|
+
const attemptedServerCount = serversAttempted.length;
|
|
592
600
|
if (logFunc) {
|
|
593
|
-
logFunc(`${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${
|
|
601
|
+
logFunc(`${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`);
|
|
594
602
|
} else {
|
|
595
|
-
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${
|
|
603
|
+
console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`));
|
|
596
604
|
}
|
|
597
605
|
if (lastError) {
|
|
598
606
|
if (logFunc) {
|
|
@@ -604,17 +612,17 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
604
612
|
}
|
|
605
613
|
|
|
606
614
|
// Return the last error with retry info
|
|
607
|
-
|
|
608
|
-
|
|
615
|
+
// V8 Optimized: Object.assign instead of spread operator
|
|
616
|
+
return Object.assign({}, lastError, {
|
|
609
617
|
retryInfo: {
|
|
610
618
|
totalAttempts: totalAttempts,
|
|
611
|
-
maxAttempts:
|
|
619
|
+
maxAttempts: serverCount * maxRetries,
|
|
612
620
|
serversAttempted: serversAttempted,
|
|
613
|
-
finalServer: lastError?.whoisServer ||
|
|
621
|
+
finalServer: lastError?.whoisServer || '',
|
|
614
622
|
retriedAfterFailure: totalAttempts > 1,
|
|
615
623
|
allAttemptsFailed: true
|
|
616
624
|
}
|
|
617
|
-
};
|
|
625
|
+
});
|
|
618
626
|
}
|
|
619
627
|
|
|
620
628
|
/**
|
|
@@ -624,7 +632,7 @@ async function whoisLookupWithRetry(domain, timeout = 10000, whoisServer = null,
|
|
|
624
632
|
* @param {number} timeout - Timeout in milliseconds (default: 5000)
|
|
625
633
|
* @returns {Promise<Object>} Object with success status and output/error
|
|
626
634
|
*/
|
|
627
|
-
async function digLookup(domain, recordType = 'A', timeout = 5000) {
|
|
635
|
+
async function digLookup(domain = '', recordType = 'A', timeout = 5000) {
|
|
628
636
|
try {
|
|
629
637
|
// Clean domain
|
|
630
638
|
const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
|
|
@@ -874,7 +882,7 @@ function createNetToolsHandler(config) {
|
|
|
874
882
|
if (forceDebug) {
|
|
875
883
|
logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Overall timeout for domain ${domain}, continuing with next...`);
|
|
876
884
|
}
|
|
877
|
-
},
|
|
885
|
+
}, 65000); // 65 second overall timeout
|
|
878
886
|
|
|
879
887
|
// Wrap entire function in timeout protection
|
|
880
888
|
return Promise.race([
|
|
@@ -885,7 +893,7 @@ function createNetToolsHandler(config) {
|
|
|
885
893
|
clearTimeout(netlookupTimeout);
|
|
886
894
|
}
|
|
887
895
|
})(),
|
|
888
|
-
new Promise((_, reject) => setTimeout(() => reject(new Error('NetTools overall timeout')),
|
|
896
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('NetTools overall timeout')), 65000))
|
|
889
897
|
]).catch(err => {
|
|
890
898
|
if (forceDebug) {
|
|
891
899
|
logToConsoleAndFile(`${messageColors.highlight('[nettools]')} ${err.message} for ${domain}, continuing...`);
|
|
@@ -952,7 +960,7 @@ function createNetToolsHandler(config) {
|
|
|
952
960
|
|
|
953
961
|
// Check whois cache first - cache key includes server for accuracy
|
|
954
962
|
const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
|
|
955
|
-
const whoisCacheKey = `${domain}-${selectedServer
|
|
963
|
+
const whoisCacheKey = `${domain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
|
|
956
964
|
const now = Date.now();
|
|
957
965
|
let whoisResult = null;
|
|
958
966
|
|
|
@@ -961,16 +969,15 @@ function createNetToolsHandler(config) {
|
|
|
961
969
|
if (now - cachedEntry.timestamp < WHOIS_CACHE_TTL) {
|
|
962
970
|
if (forceDebug) {
|
|
963
971
|
const age = Math.round((now - cachedEntry.timestamp) / 1000);
|
|
964
|
-
const serverInfo = selectedServer ? ` (server: ${selectedServer})` : ' (default server)';
|
|
972
|
+
const serverInfo = (selectedServer && selectedServer !== '') ? ` (server: ${selectedServer})` : ' (default server)';
|
|
965
973
|
logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${domain}${serverInfo} [age: ${age}s]`);
|
|
966
974
|
}
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
// Add cache metadata
|
|
975
|
+
// V8 Optimized: Object.assign is faster than spread for object merging
|
|
976
|
+
whoisResult = Object.assign({}, cachedEntry.result, {
|
|
970
977
|
fromCache: true,
|
|
971
978
|
cacheAge: now - cachedEntry.timestamp,
|
|
972
979
|
originalTimestamp: cachedEntry.timestamp
|
|
973
|
-
};
|
|
980
|
+
});
|
|
974
981
|
} else {
|
|
975
982
|
// Cache expired, remove it
|
|
976
983
|
whoisResultCache.delete(whoisCacheKey);
|
|
@@ -983,7 +990,7 @@ function createNetToolsHandler(config) {
|
|
|
983
990
|
// Perform fresh lookup if not cached
|
|
984
991
|
if (!whoisResult) {
|
|
985
992
|
if (forceDebug) {
|
|
986
|
-
const serverInfo = selectedServer ? ` using server ${selectedServer}` : ' using default server';
|
|
993
|
+
const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
|
|
987
994
|
logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${domain}${serverInfo}`);
|
|
988
995
|
}
|
|
989
996
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fanboynz/network-scanner",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.28",
|
|
4
4
|
"description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
|
|
5
5
|
"main": "nwss.js",
|
|
6
6
|
"scripts": {
|