@fanboynz/network-scanner 2.0.27 → 2.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clauderc +30 -0
- package/README.md +30 -0
- package/lib/browserhealth.js +139 -57
- package/lib/cdp.js +60 -38
- package/lib/fingerprint.js +104 -24
- package/lib/nettools.js +96 -65
- package/nwss.js +26 -10
- package/package.json +1 -1
package/.clauderc
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Network scanner that monitors website requests and generates blocking rules. Uses Puppeteer to load sites, intercepts network traffic, matches patterns, and outputs rules in various formats (adblock, dnsmasq, hosts file, etc.).",
|
|
3
|
+
|
|
4
|
+
"conventions": [
|
|
5
|
+
"Store modular functionality in ./lib/ directory with focused, single-purpose modules",
|
|
6
|
+
"Use messageColors and formatLogMessage from ./lib/colorize for consistent console output",
|
|
7
|
+
"Implement timeout protection for all Puppeteer operations using Promise.race patterns",
|
|
8
|
+
"Handle browser lifecycle with comprehensive cleanup in try-finally blocks",
|
|
9
|
+
"Validate all external tool availability before use (grep, curl, whois, dig)",
|
|
10
|
+
"Use forceDebug flag for detailed logging, silentMode for minimal output"
|
|
11
|
+
],
|
|
12
|
+
|
|
13
|
+
"files": {
|
|
14
|
+
"important": [
|
|
15
|
+
"nwss.js",
|
|
16
|
+
"config.json",
|
|
17
|
+
"lib/*.js",
|
|
18
|
+
"*.md",
|
|
19
|
+
"nwss.1"
|
|
20
|
+
],
|
|
21
|
+
"ignore": [
|
|
22
|
+
"node_modules/**",
|
|
23
|
+
"logs/**",
|
|
24
|
+
"sources/**",
|
|
25
|
+
".cache/**",
|
|
26
|
+
"*.log",
|
|
27
|
+
"*.gz"
|
|
28
|
+
]
|
|
29
|
+
}
|
|
30
|
+
}
|
package/README.md
CHANGED
|
@@ -281,6 +281,36 @@ These options go at the root level of your config.json:
|
|
|
281
281
|
| `cache_requests` | Boolean | `false` | Enable HTTP request response caching |
|
|
282
282
|
|
|
283
283
|
---
|
|
284
|
+
#### Special Characters in searchstring
|
|
285
|
+
|
|
286
|
+
The `searchstring` parameter supports all characters including special symbols. Only double quotes need JSON escaping:
|
|
287
|
+
|
|
288
|
+
```json
|
|
289
|
+
{
|
|
290
|
+
"searchstring": [
|
|
291
|
+
")}return n}function N(n,e,r){try{\"function\"==typeof",
|
|
292
|
+
"addEventListener(\"click\",function(){",
|
|
293
|
+
"{\"status\":\"success\",\"data\":[",
|
|
294
|
+
"console.log('Debug: ' + value);",
|
|
295
|
+
"`API endpoint: ${baseUrl}/users`",
|
|
296
|
+
"@media screen and (max-width: 768px)",
|
|
297
|
+
"if(e&&e.preventDefault){e.preventDefault()}",
|
|
298
|
+
"__webpack_require__(/*! ./module */ \"./src/module.js\")",
|
|
299
|
+
"console.log('Hello world')",
|
|
300
|
+
"#header { background-color: #ff0000; }",
|
|
301
|
+
"$(document).ready(function() {",
|
|
302
|
+
"completion: 85% @ $1,500 budget",
|
|
303
|
+
"SELECT * FROM users WHERE id = *",
|
|
304
|
+
"regex: ^[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}$",
|
|
305
|
+
"typeof window !== 'undefined'"
|
|
306
|
+
]
|
|
307
|
+
}
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
**Character escaping rules:**
|
|
311
|
+
- `"` becomes `\"` (required in JSON)
|
|
312
|
+
- `\` becomes `\\` (if searching for literal backslashes)
|
|
313
|
+
- All other characters are used literally: `'` `` ` `` `@` `#` `$` `%` `*` `^` `[` `]` `{` `}` `(` `)` `;` `=` `!` `?` `:`
|
|
284
314
|
|
|
285
315
|
## Usage Examples
|
|
286
316
|
|
package/lib/browserhealth.js
CHANGED
|
@@ -33,6 +33,18 @@ const PAGE_IDLE_THRESHOLD = 25000; // 25 seconds of inactivity before considerin
|
|
|
33
33
|
async function performGroupWindowCleanup(browserInstance, groupDescription, forceDebug, cleanupMode = true) {
|
|
34
34
|
try {
|
|
35
35
|
// Wait before cleanup to allow any final operations to complete
|
|
36
|
+
// Initialize result object with ALL possible properties upfront for V8 optimization
|
|
37
|
+
const result = {
|
|
38
|
+
success: false,
|
|
39
|
+
closedCount: 0,
|
|
40
|
+
totalPages: 0,
|
|
41
|
+
mainPagePreserved: false,
|
|
42
|
+
delayUsed: 0,
|
|
43
|
+
estimatedMemoryFreed: 0,
|
|
44
|
+
estimatedMemoryFreedFormatted: '',
|
|
45
|
+
cleanupMode: '',
|
|
46
|
+
error: null
|
|
47
|
+
};
|
|
36
48
|
const modeText = cleanupMode === "all" ? "aggressive cleanup of old windows" : "conservative cleanup of extra windows"
|
|
37
49
|
if (forceDebug) {
|
|
38
50
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Waiting ${WINDOW_CLEANUP_DELAY_MS}ms before ${modeText} for group: ${groupDescription}`));
|
|
@@ -47,8 +59,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
47
59
|
|
|
48
60
|
// Find the main page - typically the first page that's about:blank or has been there longest
|
|
49
61
|
for (const page of allPages) {
|
|
50
|
-
|
|
51
|
-
|
|
62
|
+
// Cache page.url() call to avoid repeated DOM/browser communication
|
|
63
|
+
const pageUrl = page.url();
|
|
64
|
+
if (pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://')) {
|
|
52
65
|
if (!mainPuppeteerPage) {
|
|
53
66
|
mainPuppeteerPage = page; // First blank page is likely the main window
|
|
54
67
|
} else {
|
|
@@ -56,6 +69,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
56
69
|
}
|
|
57
70
|
} else {
|
|
58
71
|
// Any page with actual content should be evaluated for closure
|
|
72
|
+
// Cache page state checks to avoid multiple browser calls
|
|
73
|
+
const isPageClosed = page.isClosed();
|
|
74
|
+
|
|
59
75
|
if (cleanupMode === "all") {
|
|
60
76
|
// Aggressive mode: close all content pages
|
|
61
77
|
pagesToClose.push(page);
|
|
@@ -75,7 +91,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
75
91
|
mainPuppeteerPage = allPages[0]; // Fallback to first page
|
|
76
92
|
pagesToClose = allPages.slice(1);
|
|
77
93
|
if (forceDebug) {
|
|
78
|
-
|
|
94
|
+
// Cache URL call for logging
|
|
95
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
96
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] No blank page found, using first page as main: ${mainPageUrl}`));
|
|
79
97
|
}
|
|
80
98
|
}
|
|
81
99
|
|
|
@@ -83,14 +101,11 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
83
101
|
if (forceDebug) {
|
|
84
102
|
console.log(formatLogMessage('debug', `[group_window_cleanup] No windows to close for group: ${groupDescription}`));
|
|
85
103
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
mainPagePreserved: true,
|
|
92
|
-
cleanupMode: cleanupMode === "all" ? "all" : "default"
|
|
93
|
-
};
|
|
104
|
+
result.success = true;
|
|
105
|
+
result.totalPages = allPages.length;
|
|
106
|
+
result.mainPagePreserved = true;
|
|
107
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
108
|
+
return result;
|
|
94
109
|
}
|
|
95
110
|
|
|
96
111
|
// Estimate memory usage before closing
|
|
@@ -102,7 +117,9 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
102
117
|
let pageMemoryEstimate = 0;
|
|
103
118
|
|
|
104
119
|
try {
|
|
105
|
-
|
|
120
|
+
// Cache page.isClosed() check to avoid repeated browser calls
|
|
121
|
+
const isPageClosed = page.isClosed();
|
|
122
|
+
if (!isPageClosed) {
|
|
106
123
|
// Get page metrics if available
|
|
107
124
|
const metrics = await Promise.race([
|
|
108
125
|
page.metrics(),
|
|
@@ -138,12 +155,15 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
138
155
|
// Close identified old/unused pages
|
|
139
156
|
const closePromises = pagesToClose.map(async (page, index) => {
|
|
140
157
|
try {
|
|
141
|
-
|
|
158
|
+
// Cache page state and URL for this operation
|
|
159
|
+
const isPageClosed = page.isClosed();
|
|
160
|
+
const pageUrl = page.url();
|
|
161
|
+
if (!isPageClosed) {
|
|
142
162
|
if (forceDebug) {
|
|
143
|
-
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${
|
|
163
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Closing page: ${pageUrl}`));
|
|
144
164
|
}
|
|
145
165
|
await page.close();
|
|
146
|
-
return { success: true, url:
|
|
166
|
+
return { success: true, url: pageUrl || `page-${index}`, estimatedMemory: pageMemoryEstimates[index] };
|
|
147
167
|
}
|
|
148
168
|
return { success: false, reason: 'already_closed', estimatedMemory: 0 };
|
|
149
169
|
} catch (closeErr) {
|
|
@@ -177,25 +197,39 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
177
197
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Closed ${successfulCloses}/${pagesToClose.length} old windows for completed group: ${groupDescription} after ${WINDOW_CLEANUP_DELAY_MS}ms delay`));
|
|
178
198
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Estimated memory freed: ${formatMemory(actualMemoryFreed)}`));
|
|
179
199
|
if (mainPuppeteerPage) {
|
|
180
|
-
|
|
200
|
+
// Cache URL for final logging
|
|
201
|
+
const mainPageUrl = mainPuppeteerPage.url();
|
|
202
|
+
console.log(formatLogMessage('debug', `[group_window_cleanup] Main Puppeteer window preserved: ${mainPageUrl}`));
|
|
181
203
|
}
|
|
182
204
|
}
|
|
183
205
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
206
|
+
// Update result object instead of creating new one
|
|
207
|
+
result.success = true;
|
|
208
|
+
result.closedCount = successfulCloses;
|
|
209
|
+
result.totalPages = allPages.length;
|
|
210
|
+
result.mainPagePreserved = mainPuppeteerPage && !mainPuppeteerPage.isClosed();
|
|
211
|
+
result.delayUsed = WINDOW_CLEANUP_DELAY_MS;
|
|
212
|
+
result.estimatedMemoryFreed = actualMemoryFreed;
|
|
213
|
+
result.estimatedMemoryFreedFormatted = formatMemory(actualMemoryFreed);
|
|
214
|
+
result.cleanupMode = cleanupMode === "all" ? "all" : "default";
|
|
215
|
+
return result;
|
|
194
216
|
} catch (cleanupErr) {
|
|
195
217
|
if (forceDebug) {
|
|
196
218
|
console.log(formatLogMessage('debug', `[group_window_cleanup] Group cleanup failed for ${groupDescription}: ${cleanupErr.message}`));
|
|
197
219
|
}
|
|
198
|
-
|
|
220
|
+
// Initialize result object with consistent shape for error case
|
|
221
|
+
const result = {
|
|
222
|
+
success: false,
|
|
223
|
+
closedCount: 0,
|
|
224
|
+
totalPages: 0,
|
|
225
|
+
mainPagePreserved: false,
|
|
226
|
+
delayUsed: 0,
|
|
227
|
+
estimatedMemoryFreed: 0,
|
|
228
|
+
estimatedMemoryFreedFormatted: '',
|
|
229
|
+
cleanupMode: '',
|
|
230
|
+
error: cleanupErr.message
|
|
231
|
+
};
|
|
232
|
+
return result;
|
|
199
233
|
}
|
|
200
234
|
}
|
|
201
235
|
|
|
@@ -207,14 +241,17 @@ async function performGroupWindowCleanup(browserInstance, groupDescription, forc
|
|
|
207
241
|
*/
|
|
208
242
|
async function isPageSafeToClose(page, forceDebug) {
|
|
209
243
|
try {
|
|
210
|
-
|
|
244
|
+
// Cache page.isClosed() to avoid repeated browser communication
|
|
245
|
+
const isPageClosed = page.isClosed();
|
|
246
|
+
if (isPageClosed) {
|
|
211
247
|
return true; // Already closed
|
|
212
248
|
}
|
|
213
249
|
|
|
214
250
|
// EXTRA SAFETY: Never close pages that might be in injection process
|
|
215
251
|
try {
|
|
216
|
-
|
|
217
|
-
|
|
252
|
+
// Cache page.url() for safety checks
|
|
253
|
+
const pageUrl = page.url();
|
|
254
|
+
if (pageUrl && pageUrl !== 'about:blank' && Date.now() - (pageCreationTracker.get(page) || 0) < 30000) {
|
|
218
255
|
return false; // Don't close recently created pages (within 30 seconds)
|
|
219
256
|
}
|
|
220
257
|
} catch (err) { /* ignore */ }
|
|
@@ -228,7 +265,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
228
265
|
// Check if page is actively processing
|
|
229
266
|
if (usage.isProcessing) {
|
|
230
267
|
if (forceDebug) {
|
|
231
|
-
|
|
268
|
+
// Cache URL for debug logging
|
|
269
|
+
const pageUrl = page.url();
|
|
270
|
+
console.log(formatLogMessage('debug', `[realtime_cleanup] Page still processing: ${pageUrl.substring(0, 50)}...`));
|
|
232
271
|
}
|
|
233
272
|
return false;
|
|
234
273
|
}
|
|
@@ -257,7 +296,9 @@ async function isPageSafeToClose(page, forceDebug) {
|
|
|
257
296
|
*/
|
|
258
297
|
function updatePageUsage(page, isProcessing = false) {
|
|
259
298
|
try {
|
|
260
|
-
|
|
299
|
+
// Cache page.isClosed() to avoid repeated calls
|
|
300
|
+
const isPageClosed = page.isClosed();
|
|
301
|
+
if (!isPageClosed) {
|
|
261
302
|
pageUsageTracker.set(page, {
|
|
262
303
|
lastActivity: Date.now(),
|
|
263
304
|
isProcessing: isProcessing
|
|
@@ -281,12 +322,27 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
281
322
|
try {
|
|
282
323
|
const allPages = await browserInstance.pages();
|
|
283
324
|
|
|
325
|
+
// Initialize result object with consistent shape
|
|
326
|
+
const result = {
|
|
327
|
+
success: false,
|
|
328
|
+
closedCount: 0,
|
|
329
|
+
totalPages: 0,
|
|
330
|
+
remainingPages: 0,
|
|
331
|
+
threshold: 0,
|
|
332
|
+
cleanupDelay: 0,
|
|
333
|
+
reason: '',
|
|
334
|
+
error: null
|
|
335
|
+
};
|
|
336
|
+
|
|
284
337
|
// Skip cleanup if we don't have enough pages to warrant it
|
|
285
338
|
if (allPages.length <= Math.max(threshold, REALTIME_CLEANUP_MIN_PAGES)) {
|
|
286
339
|
if (forceDebug) {
|
|
287
340
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Only ${allPages.length} pages open, threshold is ${threshold} - no cleanup needed`));
|
|
288
341
|
}
|
|
289
|
-
|
|
342
|
+
result.success = true;
|
|
343
|
+
result.totalPages = allPages.length;
|
|
344
|
+
result.reason = 'below_threshold';
|
|
345
|
+
return result;
|
|
290
346
|
}
|
|
291
347
|
|
|
292
348
|
// Use the provided total delay (already includes appropriate buffer)
|
|
@@ -324,13 +380,18 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
324
380
|
|
|
325
381
|
// Find main Puppeteer page (usually about:blank)
|
|
326
382
|
let mainPage = allPagesAfterDelay.find(page => {
|
|
327
|
-
|
|
328
|
-
|
|
383
|
+
// Cache page.url() for main page detection
|
|
384
|
+
const pageUrl = page.url();
|
|
385
|
+
return pageUrl === 'about:blank' || pageUrl === '' || pageUrl.startsWith('chrome://');
|
|
329
386
|
}) || allPagesAfterDelay[0]; // Fallback to first page
|
|
330
387
|
|
|
331
388
|
// Get pages sorted by creation time (oldest first)
|
|
332
389
|
const sortedPages = allPagesAfterDelay
|
|
333
|
-
.filter(page =>
|
|
390
|
+
.filter(page => {
|
|
391
|
+
// Cache page.isClosed() for filtering
|
|
392
|
+
const isPageClosed = page.isClosed();
|
|
393
|
+
return page !== mainPage && !isPageClosed;
|
|
394
|
+
})
|
|
334
395
|
.sort((a, b) => {
|
|
335
396
|
const timeA = pageCreationTracker.get(a) || 0;
|
|
336
397
|
const timeB = pageCreationTracker.get(b) || 0;
|
|
@@ -360,15 +421,21 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
360
421
|
`${pagesToClose.length} pages still active`;
|
|
361
422
|
console.log(formatLogMessage('debug', `[realtime_cleanup] No pages need closing (${reason})`));
|
|
362
423
|
}
|
|
363
|
-
|
|
424
|
+
result.success = true;
|
|
425
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
426
|
+
result.reason = 'no_cleanup_needed';
|
|
427
|
+
return result;
|
|
364
428
|
}
|
|
365
429
|
|
|
366
430
|
// Close oldest pages
|
|
367
431
|
let closedCount = 0;
|
|
368
432
|
for (const page of safePagesToClose) {
|
|
369
433
|
try {
|
|
370
|
-
|
|
371
|
-
|
|
434
|
+
// Cache both page state and URL for this iteration
|
|
435
|
+
const isPageClosed = page.isClosed();
|
|
436
|
+
const pageUrl = page.url();
|
|
437
|
+
|
|
438
|
+
if (!isPageClosed) {
|
|
372
439
|
await page.close();
|
|
373
440
|
pageCreationTracker.delete(page); // Remove from tracker
|
|
374
441
|
closedCount++;
|
|
@@ -390,20 +457,30 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
390
457
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Closed ${closedCount}/${pagesToClose.length} oldest pages (${unsafePagesCount} skipped for safety), ${remainingPages} pages remaining`));
|
|
391
458
|
}
|
|
392
459
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
};
|
|
460
|
+
result.success = true;
|
|
461
|
+
result.closedCount = closedCount;
|
|
462
|
+
result.totalPages = allPagesAfterDelay.length;
|
|
463
|
+
result.remainingPages = remainingPages;
|
|
464
|
+
result.threshold = threshold;
|
|
465
|
+
result.cleanupDelay = cleanupDelay;
|
|
466
|
+
result.reason = 'cleanup_completed';
|
|
467
|
+
return result;
|
|
402
468
|
} catch (cleanupErr) {
|
|
403
469
|
if (forceDebug) {
|
|
404
470
|
console.log(formatLogMessage('debug', `[realtime_cleanup] Cleanup failed: ${cleanupErr.message}`));
|
|
405
471
|
}
|
|
406
|
-
|
|
472
|
+
// Initialize result object with consistent shape for error case
|
|
473
|
+
const result = {
|
|
474
|
+
success: false,
|
|
475
|
+
closedCount: 0,
|
|
476
|
+
totalPages: 0,
|
|
477
|
+
remainingPages: 0,
|
|
478
|
+
threshold: 0,
|
|
479
|
+
cleanupDelay: 0,
|
|
480
|
+
reason: '',
|
|
481
|
+
error: cleanupErr.message
|
|
482
|
+
};
|
|
483
|
+
return result;
|
|
407
484
|
}
|
|
408
485
|
}
|
|
409
486
|
|
|
@@ -415,14 +492,15 @@ async function performRealtimeWindowCleanup(browserInstance, threshold = REALTIM
|
|
|
415
492
|
*/
|
|
416
493
|
async function isPageFromPreviousScan(page, forceDebug) {
|
|
417
494
|
try {
|
|
418
|
-
|
|
495
|
+
// Cache page.url() for all checks in this function
|
|
496
|
+
const pageUrl = page.url();
|
|
419
497
|
|
|
420
498
|
// Always consider these as old/closeable
|
|
421
|
-
if (
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
499
|
+
if (pageUrl === 'about:blank' ||
|
|
500
|
+
pageUrl === '' ||
|
|
501
|
+
pageUrl.startsWith('chrome://') ||
|
|
502
|
+
pageUrl.startsWith('chrome-error://') ||
|
|
503
|
+
pageUrl.startsWith('data:')) {
|
|
426
504
|
return false; // Don't close blank pages here, handled separately
|
|
427
505
|
}
|
|
428
506
|
|
|
@@ -446,7 +524,9 @@ async function isPageFromPreviousScan(page, forceDebug) {
|
|
|
446
524
|
return false; // Conservative - don't close unless we're sure
|
|
447
525
|
} catch (err) {
|
|
448
526
|
if (forceDebug) {
|
|
449
|
-
|
|
527
|
+
// Cache URL for error logging
|
|
528
|
+
const pageUrl = page.url();
|
|
529
|
+
console.log(formatLogMessage('debug', `[isPageFromPreviousScan] Error evaluating page ${pageUrl}: ${err.message}`));
|
|
450
530
|
}
|
|
451
531
|
return false; // Conservative - don't close if we can't evaluate
|
|
452
532
|
}
|
|
@@ -1003,7 +1083,9 @@ async function isBrowserHealthy(browserInstance, includeNetworkTest = true) {
|
|
|
1003
1083
|
*/
|
|
1004
1084
|
async function cleanupPageBeforeReload(page, forceDebug = false) {
|
|
1005
1085
|
try {
|
|
1006
|
-
|
|
1086
|
+
// Cache page.isClosed() to avoid repeated browser calls
|
|
1087
|
+
const isPageClosed = page.isClosed();
|
|
1088
|
+
if (isPageClosed) {
|
|
1007
1089
|
return false;
|
|
1008
1090
|
}
|
|
1009
1091
|
|
package/lib/cdp.js
CHANGED
|
@@ -27,6 +27,31 @@
|
|
|
27
27
|
|
|
28
28
|
const { formatLogMessage } = require('./colorize');
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Creates a reusable timeout promise to reduce function allocation overhead
|
|
32
|
+
* @param {number} ms - Timeout in milliseconds
|
|
33
|
+
* @param {string} message - Error message for timeout
|
|
34
|
+
* @returns {Promise} Promise that rejects after timeout
|
|
35
|
+
*/
|
|
36
|
+
function createTimeoutPromise(ms, message) {
|
|
37
|
+
return new Promise((_, reject) =>
|
|
38
|
+
setTimeout(() => reject(new Error(message)), ms)
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Creates a standardized session result object for consistent V8 optimization
|
|
44
|
+
* @param {object|null} session - CDP session or null
|
|
45
|
+
* @param {Function} cleanup - Cleanup function
|
|
46
|
+
* @param {boolean} isEnhanced - Whether enhanced features are active
|
|
47
|
+
* @returns {object} Standardized session object
|
|
48
|
+
*/
|
|
49
|
+
const createSessionResult = (session = null, cleanup = async () => {}, isEnhanced = false) => ({
|
|
50
|
+
session,
|
|
51
|
+
cleanup,
|
|
52
|
+
isEnhanced
|
|
53
|
+
});
|
|
54
|
+
|
|
30
55
|
/**
|
|
31
56
|
* Creates a new page with timeout protection to prevent CDP hangs
|
|
32
57
|
* @param {import('puppeteer').Browser} browser - Browser instance
|
|
@@ -36,9 +61,7 @@ const { formatLogMessage } = require('./colorize');
|
|
|
36
61
|
async function createPageWithTimeout(browser, timeout = 30000) {
|
|
37
62
|
return Promise.race([
|
|
38
63
|
browser.newPage(),
|
|
39
|
-
|
|
40
|
-
setTimeout(() => reject(new Error('Page creation timeout - browser may be unresponsive')), timeout)
|
|
41
|
-
)
|
|
64
|
+
createTimeoutPromise(timeout, 'Page creation timeout - browser may be unresponsive')
|
|
42
65
|
]);
|
|
43
66
|
}
|
|
44
67
|
|
|
@@ -52,9 +75,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
52
75
|
try {
|
|
53
76
|
await Promise.race([
|
|
54
77
|
page.setRequestInterception(true),
|
|
55
|
-
|
|
56
|
-
setTimeout(() => reject(new Error('Request interception timeout - first attempt')), timeout)
|
|
57
|
-
)
|
|
78
|
+
createTimeoutPromise(timeout, 'Request interception timeout - first attempt')
|
|
58
79
|
]);
|
|
59
80
|
} catch (firstError) {
|
|
60
81
|
// Check for immediate critical failures
|
|
@@ -68,9 +89,7 @@ async function setRequestInterceptionWithTimeout(page, timeout = 15000) {
|
|
|
68
89
|
try {
|
|
69
90
|
await Promise.race([
|
|
70
91
|
page.setRequestInterception(true),
|
|
71
|
-
|
|
72
|
-
setTimeout(() => reject(new Error('Request interception timeout - retry failed')), timeout * 2)
|
|
73
|
-
)
|
|
92
|
+
createTimeoutPromise(timeout * 2, 'Request interception timeout - retry failed')
|
|
74
93
|
]);
|
|
75
94
|
} catch (retryError) {
|
|
76
95
|
if (retryError.message.includes('Network.enable timed out') ||
|
|
@@ -128,7 +147,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
128
147
|
|
|
129
148
|
if (!cdpLoggingNeeded) {
|
|
130
149
|
// Return a null session with no-op cleanup for consistent API
|
|
131
|
-
return
|
|
150
|
+
return createSessionResult();
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
// Log which CDP mode is being used
|
|
@@ -151,9 +170,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
151
170
|
// Add timeout protection for CDP session creation
|
|
152
171
|
cdpSession = await Promise.race([
|
|
153
172
|
page.createCDPSession(),
|
|
154
|
-
|
|
155
|
-
setTimeout(() => reject(new Error('CDP session creation timeout')), 20000)
|
|
156
|
-
)
|
|
173
|
+
createTimeoutPromise(20000, 'CDP session creation timeout')
|
|
157
174
|
]);
|
|
158
175
|
|
|
159
176
|
// Enable network domain - required for network event monitoring
|
|
@@ -166,19 +183,17 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
166
183
|
const initiator = params.initiator ? params.initiator.type : 'unknown';
|
|
167
184
|
|
|
168
185
|
// Extract hostname for logging context (handles URL parsing errors gracefully)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
}
|
|
177
|
-
|
|
186
|
+
const hostnameForLog = (() => {
|
|
187
|
+
try {
|
|
188
|
+
const currentHostname = new URL(currentUrl).hostname;
|
|
189
|
+
const requestHostname = new URL(requestUrl).hostname;
|
|
190
|
+
return currentHostname !== requestHostname
|
|
191
|
+
? `${currentHostname}?${requestHostname}`
|
|
192
|
+
: currentHostname;
|
|
193
|
+
} catch (_) {
|
|
194
|
+
return 'unknown-host';
|
|
178
195
|
}
|
|
179
|
-
}
|
|
180
|
-
// Ignore URL parsing errors for logging context
|
|
181
|
-
}
|
|
196
|
+
})();
|
|
182
197
|
|
|
183
198
|
// Log the request with context only if debug mode is enabled
|
|
184
199
|
if (forceDebug) {
|
|
@@ -207,7 +222,8 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
207
222
|
}
|
|
208
223
|
}
|
|
209
224
|
}
|
|
210
|
-
}
|
|
225
|
+
},
|
|
226
|
+
isEnhanced: false
|
|
211
227
|
};
|
|
212
228
|
|
|
213
229
|
} catch (cdpErr) {
|
|
@@ -218,7 +234,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
218
234
|
try {
|
|
219
235
|
return new URL(currentUrl).hostname;
|
|
220
236
|
} catch {
|
|
221
|
-
return currentUrl.substring(0, 50)
|
|
237
|
+
return `${currentUrl.substring(0, 50)}...`;
|
|
222
238
|
}
|
|
223
239
|
})();
|
|
224
240
|
|
|
@@ -239,10 +255,7 @@ async function createCDPSession(page, currentUrl, options = {}) {
|
|
|
239
255
|
console.warn(formatLogMessage('warn', `[cdp] Failed to attach CDP session for ${currentUrl}: ${cdpErr.message}`));
|
|
240
256
|
|
|
241
257
|
// Return null session with no-op cleanup for consistent API
|
|
242
|
-
return
|
|
243
|
-
session: null,
|
|
244
|
-
cleanup: async () => {}
|
|
245
|
-
};
|
|
258
|
+
return createSessionResult();
|
|
246
259
|
}
|
|
247
260
|
}
|
|
248
261
|
|
|
@@ -278,17 +291,25 @@ function validateCDPConfig(siteConfig, globalCDP, cdpSpecificDomains = []) {
|
|
|
278
291
|
warnings.push('cdp_specific is empty - no domains will have CDP enabled');
|
|
279
292
|
} else {
|
|
280
293
|
// Validate domain format
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
294
|
+
const hasInvalidDomains = siteConfig.cdp_specific.some(domain =>
|
|
295
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
296
|
+
);
|
|
297
|
+
|
|
298
|
+
if (hasInvalidDomains) {
|
|
299
|
+
// Only filter invalid domains if we need to show them
|
|
300
|
+
const invalidDomains = siteConfig.cdp_specific.filter(domain =>
|
|
301
|
+
typeof domain !== 'string' || domain.trim() === ''
|
|
302
|
+
);
|
|
285
303
|
warnings.push(`cdp_specific contains invalid domains: ${invalidDomains.join(', ')}`);
|
|
286
304
|
}
|
|
287
305
|
}
|
|
288
306
|
}
|
|
289
307
|
|
|
290
308
|
// Performance recommendations
|
|
291
|
-
|
|
309
|
+
const cdpEnabled = globalCDP || siteConfig.cdp === true ||
|
|
310
|
+
(Array.isArray(siteConfig.cdp_specific) && siteConfig.cdp_specific.length > 0);
|
|
311
|
+
|
|
312
|
+
if (cdpEnabled) {
|
|
292
313
|
recommendations.push('CDP logging enabled - this may impact performance for high-traffic sites');
|
|
293
314
|
|
|
294
315
|
if (siteConfig.timeout && siteConfig.timeout < 30000) {
|
|
@@ -332,7 +353,8 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
332
353
|
const basicSession = await createCDPSession(page, currentUrl, options);
|
|
333
354
|
|
|
334
355
|
if (!basicSession.session) {
|
|
335
|
-
|
|
356
|
+
// Ensure enhanced flag is set even for null sessions
|
|
357
|
+
return { ...basicSession, isEnhanced: false };
|
|
336
358
|
}
|
|
337
359
|
|
|
338
360
|
const { session } = basicSession;
|
|
@@ -377,7 +399,7 @@ async function createEnhancedCDPSession(page, currentUrl, options = {}) {
|
|
|
377
399
|
|
|
378
400
|
// Graceful degradation: return basic session if enhanced features fail
|
|
379
401
|
// This ensures your application continues working even if advanced features break
|
|
380
|
-
return basicSession;
|
|
402
|
+
return { ...basicSession, isEnhanced: false };
|
|
381
403
|
}
|
|
382
404
|
}
|
|
383
405
|
|