@fanboynz/network-scanner 1.0.89 → 1.0.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib/evaldocument.js +193 -0
  2. package/nwss.js +12 -156
  3. package/package.json +1 -1
@@ -0,0 +1,193 @@
1
+ /**
2
+ * Module for handling evaluateOnNewDocument functionality
3
+ * Provides Fetch/XHR interception and page protection mechanisms
4
+ */
5
+
6
+ /**
7
+ * Applies evaluateOnNewDocument script injection to a page
8
+ * @param {import('puppeteer').Page} page - Puppeteer page instance
9
+ * @param {string} currentUrl - Current URL being processed
10
+ * @param {Object} siteConfig - Site configuration
11
+ * @param {boolean} globalEvalOnDoc - Global eval-on-doc flag
12
+ * @param {boolean} forceDebug - Debug logging flag
13
+ * @param {Function} formatLogMessage - Log formatting function
14
+ * @returns {Promise<boolean>} Success status of injection
15
+ */
16
+ async function applyEvaluateOnNewDocument(page, currentUrl, siteConfig, globalEvalOnDoc, forceDebug, formatLogMessage) {
17
+ const shouldInjectEvalForPage = siteConfig.evaluateOnNewDocument === true || globalEvalOnDoc;
18
+ let evalOnDocSuccess = false;
19
+
20
+ if (!shouldInjectEvalForPage) {
21
+ return false;
22
+ }
23
+
24
+ if (forceDebug) {
25
+ if (globalEvalOnDoc) {
26
+ console.log(formatLogMessage('debug', `[evalOnDoc] Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
27
+ } else {
28
+ console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
29
+ }
30
+ }
31
+
32
+ // Strategy 1: Try full injection with health check
33
+ let browserResponsive = false;
34
+ try {
35
+ await Promise.race([
36
+ page.browser().version(), // Quick responsiveness test
37
+ new Promise((_, reject) =>
38
+ setTimeout(() => reject(new Error('Browser health check timeout')), 3000)
39
+ )
40
+ ]);
41
+ browserResponsive = true;
42
+ } catch (healthErr) {
43
+ if (forceDebug) {
44
+ console.log(formatLogMessage('debug', `[evalOnDoc] Browser health check failed: ${healthErr.message}`));
45
+ }
46
+ browserResponsive = false;
47
+ }
48
+
49
+ // Strategy 2: Try injection with reduced complexity if browser is responsive
50
+ if (browserResponsive) {
51
+ try {
52
+ await Promise.race([
53
+ page.evaluateOnNewDocument(createFullInterceptionScript()),
54
+ new Promise((_, reject) =>
55
+ setTimeout(() => reject(new Error('Injection timeout')), 8000)
56
+ )
57
+ ]);
58
+ evalOnDocSuccess = true;
59
+ if (forceDebug) {
60
+ console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
61
+ }
62
+ } catch (fullInjectionErr) {
63
+ if (forceDebug) {
64
+ console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
65
+ }
66
+
67
+ // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
68
+ try {
69
+ await Promise.race([
70
+ page.evaluateOnNewDocument(createMinimalInterceptionScript()),
71
+ new Promise((_, reject) =>
72
+ setTimeout(() => reject(new Error('Minimal injection timeout')), 3000)
73
+ )
74
+ ]);
75
+ evalOnDocSuccess = true;
76
+ if (forceDebug) {
77
+ console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection successful for ${currentUrl}`));
78
+ }
79
+ } catch (minimalInjectionErr) {
80
+ if (forceDebug) {
81
+ console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection also failed: ${minimalInjectionErr.message}`));
82
+ }
83
+ evalOnDocSuccess = false;
84
+ }
85
+ }
86
+ } else {
87
+ if (forceDebug) {
88
+ console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
89
+ }
90
+ evalOnDocSuccess = false;
91
+ }
92
+
93
+ // Final status logging
94
+ if (!evalOnDocSuccess) {
95
+ console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
96
+ }
97
+
98
+ return evalOnDocSuccess;
99
+ }
100
+
101
+ /**
102
+ * Creates the full interception script with all protections
103
+ * @returns {Function} Script function for evaluateOnNewDocument
104
+ */
105
+ function createFullInterceptionScript() {
106
+ return () => {
107
+ // Prevent infinite reload loops
108
+ let reloadCount = 0;
109
+ const MAX_RELOADS = 2;
110
+ const originalReload = window.location.reload;
111
+ const originalReplace = window.location.replace;
112
+ const originalAssign = window.location.assign;
113
+
114
+ window.location.reload = function() {
115
+ if (++reloadCount > MAX_RELOADS) {
116
+ console.log('[loop-protection] Blocked excessive reload attempt');
117
+ return;
118
+ }
119
+ return originalReload.apply(this, arguments);
120
+ };
121
+
122
+ // Also protect against location.replace/assign to same URL
123
+ const currentHref = window.location.href;
124
+ window.location.replace = function(url) {
125
+ if (url === currentHref && ++reloadCount > MAX_RELOADS) {
126
+ console.log('[loop-protection] Blocked same-page replace attempt');
127
+ return;
128
+ }
129
+ return originalReplace.apply(this, arguments);
130
+ };
131
+
132
+ // This script intercepts and logs Fetch and XHR requests
133
+ // from within the page context at the earliest possible moment.
134
+ const originalFetch = window.fetch;
135
+ window.fetch = (...args) => {
136
+ try {
137
+ console.log('[evalOnDoc][fetch]', args[0]); // Log fetch requests
138
+ const fetchPromise = originalFetch.apply(this, args);
139
+
140
+ // Add network error handling to prevent page errors
141
+ return fetchPromise.catch(fetchErr => {
142
+ console.log('[evalOnDoc][fetch-error]', args[0], fetchErr.message);
143
+ throw fetchErr; // Re-throw to maintain normal error flow
144
+ });
145
+ } catch (fetchWrapperErr) {
146
+ console.log('[evalOnDoc][fetch-wrapper-error]', fetchWrapperErr.message);
147
+ return originalFetch.apply(this, args);
148
+ }
149
+ };
150
+
151
+ const originalXHROpen = XMLHttpRequest.prototype.open;
152
+ XMLHttpRequest.prototype.open = function (method, xhrUrl) {
153
+ try {
154
+ console.log('[evalOnDoc][xhr]', xhrUrl); // Log XHR requests
155
+
156
+ // Add error handling for XHR
157
+ this.addEventListener('error', function(event) {
158
+ console.log('[evalOnDoc][xhr-error]', xhrUrl, 'Network error occurred');
159
+ });
160
+
161
+ return originalXHROpen.apply(this, arguments);
162
+ } catch (xhrOpenErr) {
163
+ console.log('[evalOnDoc][xhr-open-error]', xhrOpenErr.message);
164
+ return originalXHROpen.apply(this, arguments);
165
+ }
166
+ };
167
+ };
168
+ }
169
+
170
+ /**
171
+ * Creates the minimal interception script (fetch only)
172
+ * @returns {Function} Script function for evaluateOnNewDocument
173
+ */
174
+ function createMinimalInterceptionScript() {
175
+ return () => {
176
+ // Minimal injection - just fetch monitoring
177
+ if (window.fetch) {
178
+ const originalFetch = window.fetch;
179
+ window.fetch = (...args) => {
180
+ try {
181
+ console.log('[evalOnDoc][fetch-minimal]', args[0]);
182
+ return originalFetch.apply(this, args);
183
+ } catch (err) {
184
+ return originalFetch.apply(this, args);
185
+ }
186
+ };
187
+ }
188
+ };
189
+ }
190
+
191
+ module.exports = {
192
+ applyEvaluateOnNewDocument
193
+ };
package/nwss.js CHANGED
@@ -1,4 +1,4 @@
1
- // === Network scanner script (nwss.js) v1.0.89 ===
1
+ // === Network scanner script (nwss.js) v1.0.90 ===
2
2
 
3
3
  // puppeteer for browser automation, fs for file system operations, psl for domain parsing.
4
4
  // const pLimit = require('p-limit'); // Will be dynamically imported
@@ -45,6 +45,9 @@ const { createGlobalHelpers, getTotalDomainsSkipped, getDetectedDomainsCount } =
45
45
  const { createSmartCache } = require('./lib/smart-cache'); // Smart cache system
46
46
  const { clearPersistentCache } = require('./lib/smart-cache');
47
47
 
48
+ // Evaluate on new document functionality
49
+ const { applyEvaluateOnNewDocument } = require('./lib/evaldocument');
50
+
48
51
  // Fast setTimeout helper for Puppeteer 22.x compatibility
49
52
  // Uses standard Promise constructor for better performance than node:timers/promises
50
53
  function fastTimeout(ms) {
@@ -125,7 +128,7 @@ const { navigateWithRedirectHandling, handleRedirectTimeout } = require('./lib/r
125
128
  const { monitorBrowserHealth, isBrowserHealthy, isQuicklyResponsive } = require('./lib/browserhealth');
126
129
 
127
130
  // --- Script Configuration & Constants ---
128
- const VERSION = '1.0.89'; // Script version
131
+ const VERSION = '1.0.90'; // Script version
129
132
 
130
133
  // get startTime
131
134
  const startTime = Date.now();
@@ -1552,158 +1555,9 @@ function setupFrameHandling(page, forceDebug) {
1552
1555
 
1553
1556
  // --- START: evaluateOnNewDocument for Fetch/XHR Interception (Moved and Fixed) ---
1554
1557
  // This script is injected if --eval-on-doc is used or siteConfig.evaluateOnNewDocument is true.
1555
- const shouldInjectEvalForPage = siteConfig.evaluateOnNewDocument === true || globalEvalOnDoc;
1556
- let evalOnDocSuccess = false; // Track injection success for fallback logic
1557
-
1558
- if (shouldInjectEvalForPage) {
1559
- if (forceDebug) {
1560
- if (globalEvalOnDoc) {
1561
- console.log(formatLogMessage('debug', `[evalOnDoc] Global Fetch/XHR interception enabled, applying to: ${currentUrl}`));
1562
- } else { // siteConfig.evaluateOnNewDocument must be true
1563
- console.log(formatLogMessage('debug', `[evalOnDoc] Site-specific Fetch/XHR interception enabled for: ${currentUrl}`));
1564
- }
1565
- }
1566
-
1567
- // Strategy 1: Try full injection with health check
1568
- let browserResponsive = false;
1569
- try {
1570
- await Promise.race([
1571
- browserInstance.version(), // Quick responsiveness test
1572
- new Promise((_, reject) =>
1573
- setTimeout(() => reject(new Error('Browser health check timeout')), 3000)
1574
- )
1575
- ]);
1576
- browserResponsive = true;
1577
- } catch (healthErr) {
1578
- if (forceDebug) {
1579
- console.log(formatLogMessage('debug', `[evalOnDoc] Browser health check failed: ${healthErr.message}`));
1580
- }
1581
- browserResponsive = false;
1582
- }
1583
-
1584
- // Strategy 2: Try injection with reduced complexity if browser is responsive
1585
- if (browserResponsive) {
1586
- try {
1587
- await Promise.race([
1588
- page.evaluateOnNewDocument(() => {
1589
- // Prevent infinite reload loops
1590
- let reloadCount = 0;
1591
- const MAX_RELOADS = 2;
1592
- const originalReload = window.location.reload;
1593
- const originalReplace = window.location.replace;
1594
- const originalAssign = window.location.assign;
1595
-
1596
- window.location.reload = function() {
1597
- if (++reloadCount > MAX_RELOADS) {
1598
- console.log('[loop-protection] Blocked excessive reload attempt');
1599
- return;
1600
- }
1601
- return originalReload.apply(this, arguments);
1602
- };
1603
-
1604
- // Also protect against location.replace/assign to same URL
1605
- const currentHref = window.location.href;
1606
- window.location.replace = function(url) {
1607
- if (url === currentHref && ++reloadCount > MAX_RELOADS) {
1608
- console.log('[loop-protection] Blocked same-page replace attempt');
1609
- return;
1610
- }
1611
- return originalReplace.apply(this, arguments);
1612
- };
1613
-
1614
- // This script intercepts and logs Fetch and XHR requests
1615
- // from within the page context at the earliest possible moment.
1616
- const originalFetch = window.fetch;
1617
- window.fetch = (...args) => {
1618
- try {
1619
- console.log('[evalOnDoc][fetch]', args[0]); // Log fetch requests
1620
- const fetchPromise = originalFetch.apply(this, args);
1621
-
1622
- // Add network error handling to prevent page errors
1623
- return fetchPromise.catch(fetchErr => {
1624
- console.log('[evalOnDoc][fetch-error]', args[0], fetchErr.message);
1625
- throw fetchErr; // Re-throw to maintain normal error flow
1626
- });
1627
- } catch (fetchWrapperErr) {
1628
- console.log('[evalOnDoc][fetch-wrapper-error]', fetchWrapperErr.message);
1629
- return originalFetch.apply(this, args);
1630
- }
1631
- };
1632
-
1633
- const originalXHROpen = XMLHttpRequest.prototype.open;
1634
- XMLHttpRequest.prototype.open = function (method, xhrUrl) {
1635
- try {
1636
- console.log('[evalOnDoc][xhr]', xhrUrl); // Log XHR requests
1637
-
1638
- // Add error handling for XHR
1639
- this.addEventListener('error', function(event) {
1640
- console.log('[evalOnDoc][xhr-error]', xhrUrl, 'Network error occurred');
1641
- });
1642
-
1643
- return originalXHROpen.apply(this, arguments);
1644
- } catch (xhrOpenErr) {
1645
- console.log('[evalOnDoc][xhr-open-error]', xhrOpenErr.message);
1646
- return originalXHROpen.apply(this, arguments);
1647
- }
1648
- };
1649
- }),
1650
- new Promise((_, reject) =>
1651
- setTimeout(() => reject(new Error('Injection timeout')), 8000)
1652
- )
1653
- ]);
1654
- evalOnDocSuccess = true;
1655
- if (forceDebug) {
1656
- console.log(formatLogMessage('debug', `[evalOnDoc] Full injection successful for ${currentUrl}`));
1657
- }
1658
- } catch (fullInjectionErr) {
1659
- if (forceDebug) {
1660
- console.log(formatLogMessage('debug', `[evalOnDoc] Full injection failed: ${fullInjectionErr.message}, trying simplified fallback`));
1661
- }
1662
-
1663
- // Strategy 3: Fallback - Try minimal injection (just fetch monitoring)
1664
- try {
1665
- await Promise.race([
1666
- page.evaluateOnNewDocument(() => {
1667
- // Minimal injection - just fetch monitoring
1668
- if (window.fetch) {
1669
- const originalFetch = window.fetch;
1670
- window.fetch = (...args) => {
1671
- try {
1672
- console.log('[evalOnDoc][fetch-minimal]', args[0]);
1673
- return originalFetch.apply(this, args);
1674
- } catch (err) {
1675
- return originalFetch.apply(this, args);
1676
- }
1677
- };
1678
- }
1679
- }),
1680
- new Promise((_, reject) =>
1681
- setTimeout(() => reject(new Error('Minimal injection timeout')), 3000)
1682
- )
1683
- ]);
1684
- evalOnDocSuccess = true;
1685
- if (forceDebug) {
1686
- console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection successful for ${currentUrl}`));
1687
- }
1688
- } catch (minimalInjectionErr) {
1689
- if (forceDebug) {
1690
- console.log(formatLogMessage('debug', `[evalOnDoc] Minimal injection also failed: ${minimalInjectionErr.message}`));
1691
- }
1692
- evalOnDocSuccess = false;
1693
- }
1694
- }
1695
- } else {
1696
- if (forceDebug) {
1697
- console.log(formatLogMessage('debug', `[evalOnDoc] Browser unresponsive, skipping injection for ${currentUrl}`));
1698
- }
1699
- evalOnDocSuccess = false;
1700
- }
1701
-
1702
- // Final status logging
1703
- if (!evalOnDocSuccess) {
1704
- console.warn(formatLogMessage('warn', `[evalOnDoc] All injection strategies failed for ${currentUrl} - continuing with standard request monitoring only`));
1705
- }
1706
- }
1558
+ const evalOnDocSuccess = await applyEvaluateOnNewDocument(
1559
+ page, currentUrl, siteConfig, globalEvalOnDoc, forceDebug, formatLogMessage
1560
+ );
1707
1561
  // --- END: evaluateOnNewDocument for Fetch/XHR Interception ---
1708
1562
 
1709
1563
  // --- CSS Element Blocking Setup ---
@@ -1754,7 +1608,7 @@ function setupFrameHandling(page, forceDebug) {
1754
1608
  await Promise.race([
1755
1609
  page.setRequestInterception(true),
1756
1610
  new Promise((_, reject) =>
1757
- setTimeout(() => reject(new Error('Network.enable timeout')), 10000)
1611
+ setTimeout(() => reject(new Error('Network.enable timeout')), 8000)
1758
1612
  )
1759
1613
  ]);
1760
1614
 
@@ -1764,7 +1618,9 @@ function setupFrameHandling(page, forceDebug) {
1764
1618
  } catch (networkErr) {
1765
1619
  if (networkErr.message.includes('timed out') ||
1766
1620
  networkErr.message.includes('Network.enable') ||
1767
- networkErr.message.includes('timeout')) {
1621
+ networkErr.message.includes('timeout') ||
1622
+ networkErr.constructor.name === 'ProtocolError' ||
1623
+ networkErr.name === 'ProtocolError') {
1768
1624
  console.warn(formatLogMessage('warn', `Network setup failed for ${currentUrl}: ${networkErr.message} - triggering browser restart`));
1769
1625
  return {
1770
1626
  url: currentUrl,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fanboynz/network-scanner",
3
- "version": "1.0.89",
3
+ "version": "1.0.90",
4
4
  "description": "A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.",
5
5
  "main": "nwss.js",
6
6
  "scripts": {