@fanboynz/network-scanner 2.0.66 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/npm-publish.yml +134 -10
- package/CHANGELOG.md +135 -0
- package/CLAUDE.md +18 -7
- package/README.md +12 -4
- package/lib/adblock-rust.js +23 -18
- package/lib/adblock.js +127 -82
- package/lib/browserexit.js +210 -200
- package/lib/browserhealth.js +84 -60
- package/lib/cdp.js +103 -81
- package/lib/clear_sitedata.js +61 -159
- package/lib/cloudflare.js +579 -409
- package/lib/colorize.js +29 -12
- package/lib/compare.js +16 -8
- package/lib/compress.js +2 -1
- package/lib/curl.js +287 -220
- package/lib/domain-cache.js +87 -40
- package/lib/dry-run.js +137 -194
- package/lib/fingerprint.js +20 -18
- package/lib/flowproxy.js +391 -188
- package/lib/ghost-cursor.js +8 -7
- package/lib/grep.js +248 -171
- package/lib/ignore_similar.js +70 -124
- package/lib/interaction.js +132 -235
- package/lib/nettools.js +309 -87
- package/lib/openvpn_vpn.js +12 -11
- package/lib/output.js +92 -59
- package/lib/post-processing.js +216 -162
- package/lib/redirect.js +46 -30
- package/lib/referrer.js +158 -165
- package/lib/searchstring.js +290 -381
- package/lib/smart-cache.js +141 -91
- package/lib/socks-relay.js +8 -7
- package/lib/spawn-async.js +137 -0
- package/lib/validate_rules.js +188 -176
- package/lib/wireguard_vpn.js +111 -117
- package/nwss.js +740 -156
- package/package.json +4 -4
package/lib/dry-run.js
CHANGED
|
@@ -4,11 +4,13 @@
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const { messageColors, formatLogMessage } = require('./colorize');
|
|
6
6
|
|
|
7
|
-
// Constants for dry run collection keys
|
|
7
|
+
// Constants for dry run collection keys. SEARCH_STRING was removed —
|
|
8
|
+
// addDryRunSearchString had zero callers, so the map was never populated
|
|
9
|
+
// and the downstream "Searchstring Match" enhancement always produced
|
|
10
|
+
// null. See the cleanup comment in processDryRunResults.
|
|
8
11
|
const DRY_RUN_KEYS = {
|
|
9
12
|
MATCHES: 'dryRunMatches',
|
|
10
|
-
NET_TOOLS: 'dryRunNetTools'
|
|
11
|
-
SEARCH_STRING: 'dryRunSearchString'
|
|
13
|
+
NET_TOOLS: 'dryRunNetTools'
|
|
12
14
|
};
|
|
13
15
|
|
|
14
16
|
/**
|
|
@@ -23,7 +25,6 @@ function initializeDryRunCollections(matchedDomains) {
|
|
|
23
25
|
|
|
24
26
|
matchedDomains.set(DRY_RUN_KEYS.MATCHES, []);
|
|
25
27
|
matchedDomains.set(DRY_RUN_KEYS.NET_TOOLS, []);
|
|
26
|
-
matchedDomains.set(DRY_RUN_KEYS.SEARCH_STRING, new Map());
|
|
27
28
|
}
|
|
28
29
|
|
|
29
30
|
/**
|
|
@@ -35,10 +36,15 @@ function validateMatchData(matchData) {
|
|
|
35
36
|
if (!matchData || typeof matchData !== 'object') {
|
|
36
37
|
throw new Error('Match data must be an object');
|
|
37
38
|
}
|
|
38
|
-
|
|
39
|
+
|
|
39
40
|
const requiredFields = ['regex', 'domain', 'resourceType', 'fullUrl'];
|
|
40
41
|
for (const field of requiredFields) {
|
|
41
|
-
|
|
42
|
+
// Check VALUE, not just key presence. The old `field in matchData`
|
|
43
|
+
// accepted `{regex: undefined, ...}` because `in` only tests for
|
|
44
|
+
// the property's existence on the object. The downstream output
|
|
45
|
+
// then printed 'unknown' via `item.regex || 'unknown'` defensive
|
|
46
|
+
// fallbacks — validation that doesn't catch this defeats its purpose.
|
|
47
|
+
if (matchData[field] === undefined || matchData[field] === null) {
|
|
42
48
|
throw new Error(`Match data missing required field: ${field}`);
|
|
43
49
|
}
|
|
44
50
|
}
|
|
@@ -53,10 +59,11 @@ function validateNetToolsData(netToolsData) {
|
|
|
53
59
|
if (!netToolsData || typeof netToolsData !== 'object') {
|
|
54
60
|
throw new Error('NetTools data must be an object');
|
|
55
61
|
}
|
|
56
|
-
|
|
62
|
+
|
|
57
63
|
const requiredFields = ['domain', 'tool', 'matchType', 'matchedTerm'];
|
|
58
64
|
for (const field of requiredFields) {
|
|
59
|
-
|
|
65
|
+
// Value check (see validateMatchData for the rationale).
|
|
66
|
+
if (netToolsData[field] === undefined || netToolsData[field] === null) {
|
|
60
67
|
throw new Error(`NetTools data missing required field: ${field}`);
|
|
61
68
|
}
|
|
62
69
|
}
|
|
@@ -108,36 +115,6 @@ function addDryRunNetTools(matchedDomains, netToolsData) {
|
|
|
108
115
|
});
|
|
109
116
|
}
|
|
110
117
|
|
|
111
|
-
/**
|
|
112
|
-
* Add a search string result to dry run collections
|
|
113
|
-
* @param {Map} matchedDomains - The matched domains map
|
|
114
|
-
* @param {string} url - The URL that was searched
|
|
115
|
-
* @param {Object} searchResult - Search result data
|
|
116
|
-
* @throws {Error} If parameters are invalid
|
|
117
|
-
*/
|
|
118
|
-
function addDryRunSearchString(matchedDomains, url, searchResult) {
|
|
119
|
-
if (!(matchedDomains instanceof Map)) {
|
|
120
|
-
throw new Error('matchedDomains must be a Map instance');
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
if (!url || typeof url !== 'string') {
|
|
124
|
-
throw new Error('URL must be a non-empty string');
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
if (!searchResult || typeof searchResult !== 'object') {
|
|
128
|
-
throw new Error('Search result must be an object');
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
if (!matchedDomains.has(DRY_RUN_KEYS.SEARCH_STRING)) {
|
|
132
|
-
throw new Error('Dry run collections not initialized. Call initializeDryRunCollections first.');
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
matchedDomains.get(DRY_RUN_KEYS.SEARCH_STRING).set(url, {
|
|
136
|
-
...searchResult,
|
|
137
|
-
timestamp: new Date().toISOString()
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
118
|
/**
|
|
142
119
|
* Safely truncate long URLs for display
|
|
143
120
|
* @param {string} url - URL to truncate
|
|
@@ -151,19 +128,6 @@ function truncateUrl(url, maxLength = 80) {
|
|
|
151
128
|
return url.substring(0, maxLength - 3) + '...';
|
|
152
129
|
}
|
|
153
130
|
|
|
154
|
-
/**
|
|
155
|
-
* Format search string match information
|
|
156
|
-
* @param {Object} searchStringMatch - Search string match data
|
|
157
|
-
* @returns {string} Formatted match description
|
|
158
|
-
*/
|
|
159
|
-
function formatSearchStringMatch(searchStringMatch) {
|
|
160
|
-
if (!searchStringMatch) return null;
|
|
161
|
-
|
|
162
|
-
const matchType = searchStringMatch.type || 'unknown';
|
|
163
|
-
const term = searchStringMatch.term || 'unknown';
|
|
164
|
-
return `${matchType} - "${term}"`;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
131
|
/**
|
|
168
132
|
* Generate adblock rule from domain and resource type
|
|
169
133
|
* @param {string} domain - Domain name
|
|
@@ -192,112 +156,130 @@ function generateAdblockRule(domain, resourceType = null) {
|
|
|
192
156
|
function outputDryRunResults(url, matchedItems = [], netToolsResults = [], pageTitle = '', outputFile = null, dryRunOutput = []) {
|
|
193
157
|
try {
|
|
194
158
|
const lines = [];
|
|
159
|
+
|
|
160
|
+
// emit() — single source of truth for output. Writes the plain
|
|
161
|
+
// version to the file-output array AND the (possibly colored)
|
|
162
|
+
// version to the console. Previously every output line was a
|
|
163
|
+
// paired lines.push(...) + console.log(...) statement, often in
|
|
164
|
+
// separate blocks (file pushes first, then console logs), so
|
|
165
|
+
// drift between file and terminal output was a real risk every
|
|
166
|
+
// time someone edited only one half of a pair.
|
|
167
|
+
const emit = (plain, colored = plain) => {
|
|
168
|
+
lines.push(plain);
|
|
169
|
+
console.log(colored);
|
|
170
|
+
};
|
|
171
|
+
|
|
195
172
|
const truncatedUrl = truncateUrl(url);
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
173
|
+
|
|
174
|
+
emit(
|
|
175
|
+
`\n=== DRY RUN RESULTS === ${truncatedUrl}`,
|
|
176
|
+
`\n${messageColors.scanning('=== DRY RUN RESULTS ===')} ${truncatedUrl}`
|
|
177
|
+
);
|
|
178
|
+
|
|
200
179
|
if (pageTitle && pageTitle.trim()) {
|
|
201
180
|
const cleanTitle = pageTitle.trim().substring(0, 200); // Limit title length
|
|
202
|
-
|
|
203
|
-
|
|
181
|
+
emit(
|
|
182
|
+
`Title: ${cleanTitle}`,
|
|
183
|
+
`${messageColors.info('Title:')} ${cleanTitle}`
|
|
184
|
+
);
|
|
204
185
|
}
|
|
205
|
-
|
|
186
|
+
|
|
206
187
|
const totalMatches = matchedItems.length + netToolsResults.length;
|
|
207
|
-
|
|
188
|
+
|
|
208
189
|
if (totalMatches === 0) {
|
|
209
190
|
const noMatchMsg = `No matching rules found on ${truncatedUrl}`;
|
|
210
|
-
|
|
211
|
-
|
|
191
|
+
emit(noMatchMsg, messageColors.warn(noMatchMsg));
|
|
192
|
+
|
|
212
193
|
if (outputFile) {
|
|
213
194
|
dryRunOutput.push(...lines);
|
|
214
195
|
dryRunOutput.push(''); // Add empty line
|
|
215
196
|
}
|
|
216
|
-
console.log(messageColors.warn(noMatchMsg));
|
|
217
197
|
return;
|
|
218
198
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
199
|
+
|
|
200
|
+
emit(
|
|
201
|
+
`Matches found: ${totalMatches}`,
|
|
202
|
+
`${messageColors.success('Matches found:')} ${totalMatches}`
|
|
203
|
+
);
|
|
204
|
+
|
|
223
205
|
// Process regex matches
|
|
224
206
|
matchedItems.forEach((item, index) => {
|
|
225
207
|
try {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
console.log(` Resource Type: ${item.resourceType || 'unknown'}`);
|
|
237
|
-
console.log(` Full URL: ${truncateUrl(item.fullUrl || '')}`);
|
|
238
|
-
|
|
239
|
-
// Show blocked status if applicable
|
|
208
|
+
emit(''); // blank separator before each match item
|
|
209
|
+
emit(
|
|
210
|
+
`[${index + 1}] Regex Match:`,
|
|
211
|
+
`${messageColors.highlight(`[${index + 1}]`)} ${messageColors.match('Regex Match:')}`
|
|
212
|
+
);
|
|
213
|
+
emit(` Pattern: ${item.regex || 'unknown'}`);
|
|
214
|
+
emit(` Domain: ${item.domain || 'unknown'}`);
|
|
215
|
+
emit(` Resource Type: ${item.resourceType || 'unknown'}`);
|
|
216
|
+
emit(` Full URL: ${truncateUrl(item.fullUrl || '')}`);
|
|
217
|
+
|
|
240
218
|
if (item.wasBlocked) {
|
|
241
|
-
|
|
242
|
-
|
|
219
|
+
emit(
|
|
220
|
+
` Status: BLOCKED (even_blocked enabled)`,
|
|
221
|
+
` ${messageColors.warn('Status:')} BLOCKED (even_blocked enabled)`
|
|
222
|
+
);
|
|
243
223
|
}
|
|
244
|
-
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
console.log(` ${messageColors.warn('? Searchstring:')} No matches found in content`);
|
|
224
|
+
|
|
225
|
+
// Searchstring "not found" — see processDryRunResults comment
|
|
226
|
+
// for why the positive-match branch was removed.
|
|
227
|
+
if (item.searchStringChecked) {
|
|
228
|
+
emit(
|
|
229
|
+
` ✗ Searchstring: No matches found in content`,
|
|
230
|
+
` ${messageColors.warn('✗ Searchstring:')} No matches found in content`
|
|
231
|
+
);
|
|
253
232
|
}
|
|
254
|
-
|
|
255
|
-
// Generate adblock rule
|
|
233
|
+
|
|
256
234
|
const adblockRule = generateAdblockRule(item.domain, item.resourceType);
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
235
|
+
emit(
|
|
236
|
+
` Adblock Rule: ${adblockRule}`,
|
|
237
|
+
` ${messageColors.info('Adblock Rule:')} ${adblockRule}`
|
|
238
|
+
);
|
|
239
|
+
|
|
260
240
|
} catch (itemErr) {
|
|
261
241
|
const errorMsg = `Error processing match item ${index + 1}: ${itemErr.message}`;
|
|
262
|
-
|
|
263
|
-
|
|
242
|
+
emit(
|
|
243
|
+
` Error: ${errorMsg}`,
|
|
244
|
+
` ${messageColors.warn('Error:')} ${errorMsg}`
|
|
245
|
+
);
|
|
264
246
|
}
|
|
265
247
|
});
|
|
266
|
-
|
|
267
|
-
// Process nettools results
|
|
248
|
+
|
|
249
|
+
// Process nettools results
|
|
268
250
|
netToolsResults.forEach((result, index) => {
|
|
269
251
|
try {
|
|
270
252
|
const resultIndex = matchedItems.length + index + 1;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
253
|
+
emit(''); // blank separator before each nettools item
|
|
254
|
+
emit(
|
|
255
|
+
`[${resultIndex}] NetTools Match:`,
|
|
256
|
+
`${messageColors.highlight(`[${resultIndex}]`)} ${messageColors.match('NetTools Match:')}`
|
|
257
|
+
);
|
|
258
|
+
emit(` Domain: ${result.domain || 'unknown'}`);
|
|
259
|
+
emit(` Tool: ${(result.tool || 'unknown').toUpperCase()}`);
|
|
260
|
+
|
|
276
261
|
const matchDesc = `${result.matchType || 'unknown'} - "${result.matchedTerm || 'unknown'}"`;
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
console.log(`\n${messageColors.highlight(`[${resultIndex}]`)} ${messageColors.match('NetTools Match:')}`);
|
|
284
|
-
console.log(` Domain: ${result.domain || 'unknown'}`);
|
|
285
|
-
console.log(` Tool: ${(result.tool || 'unknown').toUpperCase()}`);
|
|
286
|
-
console.log(` ${messageColors.success('? Match:')} ${matchDesc}`);
|
|
287
|
-
|
|
262
|
+
emit(
|
|
263
|
+
` ✓ Match: ${matchDesc}`,
|
|
264
|
+
` ${messageColors.success('✓ Match:')} ${matchDesc}`
|
|
265
|
+
);
|
|
266
|
+
|
|
288
267
|
if (result.details) {
|
|
289
|
-
|
|
268
|
+
emit(` Details: ${result.details}`);
|
|
290
269
|
}
|
|
291
|
-
|
|
292
|
-
// Generate adblock rule for nettools matches
|
|
270
|
+
|
|
293
271
|
const adblockRule = generateAdblockRule(result.domain);
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
272
|
+
emit(
|
|
273
|
+
` Adblock Rule: ${adblockRule}`,
|
|
274
|
+
` ${messageColors.info('Adblock Rule:')} ${adblockRule}`
|
|
275
|
+
);
|
|
276
|
+
|
|
297
277
|
} catch (resultErr) {
|
|
298
278
|
const errorMsg = `Error processing nettools result ${index + 1}: ${resultErr.message}`;
|
|
299
|
-
|
|
300
|
-
|
|
279
|
+
emit(
|
|
280
|
+
` Error: ${errorMsg}`,
|
|
281
|
+
` ${messageColors.warn('Error:')} ${errorMsg}`
|
|
282
|
+
);
|
|
301
283
|
}
|
|
302
284
|
});
|
|
303
285
|
|
|
@@ -306,7 +288,7 @@ function outputDryRunResults(url, matchedItems = [], netToolsResults = [], pageT
|
|
|
306
288
|
dryRunOutput.push(...lines);
|
|
307
289
|
dryRunOutput.push(''); // Add empty line between sites
|
|
308
290
|
}
|
|
309
|
-
|
|
291
|
+
|
|
310
292
|
} catch (outputErr) {
|
|
311
293
|
const errorMsg = `Error in outputDryRunResults: ${outputErr.message}`;
|
|
312
294
|
console.error(messageColors.error(errorMsg));
|
|
@@ -347,34 +329,27 @@ async function processDryRunResults(currentUrl, matchedDomains, page, outputFile
|
|
|
347
329
|
if (forceDebug) {
|
|
348
330
|
console.log(formatLogMessage('debug', `Failed to get page title for ${currentUrl}: ${titleErr.message}`));
|
|
349
331
|
}
|
|
350
|
-
pageTitle
|
|
332
|
+
// Leave pageTitle as '' (its initial value) on failure — the
|
|
333
|
+
// truthy check in outputDryRunResults then skips the Title line
|
|
334
|
+
// entirely. Previously we set 'Title unavailable' here, which
|
|
335
|
+
// was truthy and got printed as if it were the page's real
|
|
336
|
+
// title: 'Title: Title unavailable'.
|
|
351
337
|
}
|
|
352
338
|
|
|
353
339
|
// Get collected matches with safe fallbacks
|
|
354
340
|
const dryRunMatches = matchedDomains.get(DRY_RUN_KEYS.MATCHES) || [];
|
|
355
341
|
const dryRunNetTools = matchedDomains.get(DRY_RUN_KEYS.NET_TOOLS) || [];
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
//
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
} catch (enhanceErr) {
|
|
368
|
-
if (forceDebug) {
|
|
369
|
-
console.log(formatLogMessage('debug', `Error enhancing match ${index}: ${enhanceErr.message}`));
|
|
370
|
-
}
|
|
371
|
-
return {
|
|
372
|
-
...match,
|
|
373
|
-
searchStringMatch: null,
|
|
374
|
-
searchStringChecked: false
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
});
|
|
342
|
+
|
|
343
|
+
// Enhance matches with the searchstring-checked flag from the
|
|
344
|
+
// incoming match data. Previously this also looked up positive
|
|
345
|
+
// searchstring results in a `dryRunSearchString` Map — but
|
|
346
|
+
// `addDryRunSearchString` was never wired to any caller, so the
|
|
347
|
+
// map was always empty and `searchStringMatch` was always null.
|
|
348
|
+
// Removed that dead lookup and the per-item try/catch it required.
|
|
349
|
+
const enhancedMatches = dryRunMatches.map((match) => ({
|
|
350
|
+
...match,
|
|
351
|
+
searchStringChecked: Boolean(match.needsSearchStringCheck)
|
|
352
|
+
}));
|
|
378
353
|
|
|
379
354
|
outputDryRunResults(currentUrl, enhancedMatches, dryRunNetTools, pageTitle, outputFile, dryRunOutput);
|
|
380
355
|
|
|
@@ -443,7 +418,7 @@ function writeDryRunOutput(outputFile, dryRunOutput, silentMode = false) {
|
|
|
443
418
|
fs.writeFileSync(outputFile, dryRunContent);
|
|
444
419
|
|
|
445
420
|
if (!silentMode) {
|
|
446
|
-
console.log(`${messageColors.fileOp('
|
|
421
|
+
console.log(`${messageColors.fileOp('📄 Dry run results saved to:')} ${outputFile}`);
|
|
447
422
|
}
|
|
448
423
|
|
|
449
424
|
return {
|
|
@@ -456,60 +431,28 @@ function writeDryRunOutput(outputFile, dryRunOutput, silentMode = false) {
|
|
|
456
431
|
|
|
457
432
|
} catch (writeErr) {
|
|
458
433
|
const errorMsg = `Failed to write dry run output to ${outputFile}: ${writeErr.message}`;
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
434
|
+
// Matches outputDryRunResults / processDryRunResults error format —
|
|
435
|
+
// was bare `console.error('✗ ${errorMsg}')` here, the odd one out
|
|
436
|
+
// among the three error paths in this module.
|
|
437
|
+
console.error(messageColors.error(errorMsg));
|
|
438
|
+
|
|
439
|
+
return {
|
|
440
|
+
success: false,
|
|
463
441
|
error: errorMsg,
|
|
464
442
|
written: false
|
|
465
443
|
};
|
|
466
444
|
}
|
|
467
445
|
}
|
|
468
446
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
function getDryRunStats(matchedDomains) {
|
|
475
|
-
if (!(matchedDomains instanceof Map)) {
|
|
476
|
-
return { error: 'Invalid matchedDomains Map' };
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
const matches = matchedDomains.get(DRY_RUN_KEYS.MATCHES) || [];
|
|
480
|
-
const netTools = matchedDomains.get(DRY_RUN_KEYS.NET_TOOLS) || [];
|
|
481
|
-
const searchStrings = matchedDomains.get(DRY_RUN_KEYS.SEARCH_STRING) || new Map();
|
|
482
|
-
|
|
483
|
-
return {
|
|
484
|
-
totalMatches: matches.length + netTools.length,
|
|
485
|
-
regexMatches: matches.length,
|
|
486
|
-
netToolsMatches: netTools.length,
|
|
487
|
-
searchStringResults: searchStrings.size,
|
|
488
|
-
domains: new Set([
|
|
489
|
-
...matches.map(m => m.domain).filter(Boolean),
|
|
490
|
-
...netTools.map(n => n.domain).filter(Boolean)
|
|
491
|
-
]).size
|
|
492
|
-
};
|
|
493
|
-
}
|
|
494
|
-
|
|
447
|
+
// Public surface used by nwss.js. Internal helpers (truncateUrl,
|
|
448
|
+
// generateAdblockRule, validateMatchData, validateNetToolsData,
|
|
449
|
+
// outputDryRunResults) stay module-private. DRY_RUN_KEYS, getDryRunStats,
|
|
450
|
+
// addDryRunSearchString, and formatSearchStringMatch were removed —
|
|
451
|
+
// see comments at their original sites for details.
|
|
495
452
|
module.exports = {
|
|
496
|
-
// Constants
|
|
497
|
-
DRY_RUN_KEYS,
|
|
498
|
-
|
|
499
|
-
// Core functions
|
|
500
453
|
initializeDryRunCollections,
|
|
501
454
|
addDryRunMatch,
|
|
502
455
|
addDryRunNetTools,
|
|
503
|
-
addDryRunSearchString,
|
|
504
456
|
processDryRunResults,
|
|
505
|
-
writeDryRunOutput
|
|
506
|
-
|
|
507
|
-
// Utility functions
|
|
508
|
-
getDryRunStats,
|
|
509
|
-
validateMatchData,
|
|
510
|
-
validateNetToolsData,
|
|
511
|
-
truncateUrl,
|
|
512
|
-
formatSearchStringMatch,
|
|
513
|
-
generateAdblockRule,
|
|
514
|
-
outputDryRunResults
|
|
457
|
+
writeDryRunOutput
|
|
515
458
|
};
|
package/lib/fingerprint.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// and comprehensive bot detection evasion techniques.
|
|
4
4
|
//const { applyErrorSuppression } = require('./error-suppression');
|
|
5
5
|
|
|
6
|
+
const { formatLogMessage } = require('./colorize');
|
|
7
|
+
|
|
6
8
|
// Default values for fingerprint spoofing if not set to 'random'
|
|
7
9
|
const DEFAULT_PLATFORM = 'Win32';
|
|
8
10
|
const DEFAULT_TIMEZONE = 'America/New_York';
|
|
@@ -256,7 +258,7 @@ async function validatePageForInjection(page, currentUrl, forceDebug) {
|
|
|
256
258
|
if (!page || page.isClosed()) return false;
|
|
257
259
|
|
|
258
260
|
if (!page.browser().isConnected()) {
|
|
259
|
-
if (forceDebug) console.log(
|
|
261
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page validation failed - browser disconnected: ${currentUrl}`));
|
|
260
262
|
return false;
|
|
261
263
|
}
|
|
262
264
|
await Promise.race([
|
|
@@ -265,7 +267,7 @@ async function validatePageForInjection(page, currentUrl, forceDebug) {
|
|
|
265
267
|
]);
|
|
266
268
|
return true;
|
|
267
269
|
} catch (validationErr) {
|
|
268
|
-
if (forceDebug) console.log(
|
|
270
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page validation failed - ${validationErr.message}: ${currentUrl}`));
|
|
269
271
|
return false;
|
|
270
272
|
}
|
|
271
273
|
}
|
|
@@ -276,7 +278,7 @@ async function validatePageForInjection(page, currentUrl, forceDebug) {
|
|
|
276
278
|
async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl) {
|
|
277
279
|
if (!siteConfig.userAgent) return;
|
|
278
280
|
|
|
279
|
-
if (forceDebug) console.log(
|
|
281
|
+
if (forceDebug) console.log(formatLogMessage('debug', `User agent spoofing: ${siteConfig.userAgent}`));
|
|
280
282
|
|
|
281
283
|
// Browser connection check
|
|
282
284
|
try {
|
|
@@ -294,16 +296,16 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
294
296
|
try {
|
|
295
297
|
await page.setUserAgent(ua);
|
|
296
298
|
} catch (uaErr) {
|
|
297
|
-
if (forceDebug) console.log(
|
|
299
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Could not set user agent - page closed: ${currentUrl}`));
|
|
298
300
|
return;
|
|
299
301
|
}
|
|
300
302
|
|
|
301
|
-
if (forceDebug) console.log(
|
|
303
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Applying stealth protection for ${currentUrl}`));
|
|
302
304
|
|
|
303
305
|
try {
|
|
304
306
|
// Select GPU once per session — stays consistent across all page loads
|
|
305
307
|
const selectedGpu = selectGpuForUserAgent(ua);
|
|
306
|
-
if (forceDebug) console.log(
|
|
308
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Selected GPU: ${selectedGpu.vendor} / ${selectedGpu.renderer}`));
|
|
307
309
|
|
|
308
310
|
await page.evaluateOnNewDocument((userAgent, debugEnabled, gpuConfig) => {
|
|
309
311
|
|
|
@@ -1745,7 +1747,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1745
1747
|
}, ua, forceDebug, selectedGpu);
|
|
1746
1748
|
} catch (stealthErr) {
|
|
1747
1749
|
if (isSessionClosedError(stealthErr)) {
|
|
1748
|
-
if (forceDebug) console.log(
|
|
1750
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page closed during stealth injection: ${currentUrl}`));
|
|
1749
1751
|
return;
|
|
1750
1752
|
}
|
|
1751
1753
|
console.warn(`[stealth protection failed] ${currentUrl}: ${stealthErr.message}`);
|
|
@@ -1759,7 +1761,7 @@ async function applyUserAgentSpoofing(page, siteConfig, forceDebug, currentUrl)
|
|
|
1759
1761
|
async function applyBraveSpoofing(page, siteConfig, forceDebug, currentUrl) {
|
|
1760
1762
|
if (!siteConfig.isBrave) return;
|
|
1761
1763
|
|
|
1762
|
-
if (forceDebug) console.log(
|
|
1764
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Brave spoofing enabled for ${currentUrl}`));
|
|
1763
1765
|
|
|
1764
1766
|
// Browser connection check
|
|
1765
1767
|
try {
|
|
@@ -1797,10 +1799,10 @@ async function applyBraveSpoofing(page, siteConfig, forceDebug, currentUrl) {
|
|
|
1797
1799
|
}, forceDebug);
|
|
1798
1800
|
} catch (braveErr) {
|
|
1799
1801
|
if (isSessionClosedError(braveErr)) {
|
|
1800
|
-
if (forceDebug) console.log(
|
|
1802
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page closed during Brave injection: ${currentUrl}`));
|
|
1801
1803
|
return;
|
|
1802
1804
|
}
|
|
1803
|
-
if (forceDebug) console.log(
|
|
1805
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Brave spoofing failed: ${currentUrl} - ${braveErr.message}`));
|
|
1804
1806
|
}
|
|
1805
1807
|
}
|
|
1806
1808
|
|
|
@@ -1811,7 +1813,7 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1811
1813
|
const fingerprintSetting = siteConfig.fingerprint_protection;
|
|
1812
1814
|
if (!fingerprintSetting) return;
|
|
1813
1815
|
|
|
1814
|
-
if (forceDebug) console.log(
|
|
1816
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Fingerprint protection enabled for ${currentUrl}`));
|
|
1815
1817
|
|
|
1816
1818
|
// Browser connection check
|
|
1817
1819
|
try {
|
|
@@ -1827,7 +1829,7 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1827
1829
|
try {
|
|
1828
1830
|
currentUserAgent = await page.evaluate(() => navigator.userAgent);
|
|
1829
1831
|
} catch (evalErr) {
|
|
1830
|
-
if (forceDebug) console.log(
|
|
1832
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Could not get user agent - page closed: ${currentUrl}`));
|
|
1831
1833
|
return;
|
|
1832
1834
|
}
|
|
1833
1835
|
|
|
@@ -1960,7 +1962,7 @@ async function applyFingerprintProtection(page, siteConfig, forceDebug, currentU
|
|
|
1960
1962
|
}, { spoof, debugEnabled: forceDebug });
|
|
1961
1963
|
} catch (err) {
|
|
1962
1964
|
if (isSessionClosedError(err)) {
|
|
1963
|
-
if (forceDebug) console.log(
|
|
1965
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Page closed during fingerprint injection: ${currentUrl}`));
|
|
1964
1966
|
return;
|
|
1965
1967
|
}
|
|
1966
1968
|
console.warn(`[fingerprint protection failed] ${currentUrl}: ${err.message}`);
|
|
@@ -1975,13 +1977,13 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
1975
1977
|
try {
|
|
1976
1978
|
// Validate page state before injection
|
|
1977
1979
|
if (!page || page.isClosed()) {
|
|
1978
|
-
if (forceDebug) console.log(
|
|
1980
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Human behavior simulation skipped - page closed`));
|
|
1979
1981
|
return;
|
|
1980
1982
|
}
|
|
1981
1983
|
|
|
1982
1984
|
// Check if browser is still connected
|
|
1983
1985
|
if (!page.browser().isConnected()) {
|
|
1984
|
-
if (forceDebug) console.log(
|
|
1986
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Human behavior simulation skipped - browser disconnected`));
|
|
1985
1987
|
return;
|
|
1986
1988
|
}
|
|
1987
1989
|
|
|
@@ -2093,7 +2095,7 @@ async function simulateHumanBehavior(page, forceDebug) {
|
|
|
2093
2095
|
|
|
2094
2096
|
}, forceDebug);
|
|
2095
2097
|
} catch (err) {
|
|
2096
|
-
if (forceDebug) console.log(
|
|
2098
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Human behavior simulation setup failed: ${err.message}`));
|
|
2097
2099
|
}
|
|
2098
2100
|
}
|
|
2099
2101
|
|
|
@@ -2112,7 +2114,7 @@ async function applyAllFingerprintSpoofing(page, siteConfig, forceDebug, current
|
|
|
2112
2114
|
try {
|
|
2113
2115
|
await fn(page, siteConfig, forceDebug, currentUrl);
|
|
2114
2116
|
} catch (err) {
|
|
2115
|
-
if (forceDebug) console.log(
|
|
2117
|
+
if (forceDebug) console.log(formatLogMessage('debug', `${name} failed for ${currentUrl}: ${err.message}`));
|
|
2116
2118
|
}
|
|
2117
2119
|
}
|
|
2118
2120
|
|
|
@@ -2121,7 +2123,7 @@ async function applyAllFingerprintSpoofing(page, siteConfig, forceDebug, current
|
|
|
2121
2123
|
try {
|
|
2122
2124
|
await simulateHumanBehavior(page, forceDebug);
|
|
2123
2125
|
} catch (behaviorErr) {
|
|
2124
|
-
if (forceDebug) console.log(
|
|
2126
|
+
if (forceDebug) console.log(formatLogMessage('debug', `Human behavior simulation failed for ${currentUrl}: ${behaviorErr.message}`));
|
|
2125
2127
|
}
|
|
2126
2128
|
}
|
|
2127
2129
|
}
|