@fanboynz/network-scanner 2.0.65 → 3.0.0

package/lib/proxy.js

@@ -18,6 +18,15 @@
  *
  *   SOCKS5 with auth:
  *     "proxy": "socks5://user:pass@127.0.0.1:1080"
+ *     Chromium itself cannot authenticate SOCKS5 (crbug.com/256785), so
+ *     this module auto-starts an in-process no-auth SOCKS5 relay
+ *     (lib/socks-relay.js) that does the upstream RFC 1929 auth. Chromium
+ *     connects to the local relay (no auth — which it CAN do) and the
+ *     relay tunnels to the authenticated upstream. Transparent: keep the
+ *     socks5://user:pass@host form in config. Requires prepareSocksRelays()
+ *     to be awaited once before the scan loop (nwss.js does this).
+ *     NOTE: socks4 with auth is still unsupported (userId-only,
+ *     near-extinct) — use socks5 or an authenticated HTTP proxy.
  *
  *   HTTP proxy (corporate):
  *     "proxy": "http://proxy.corp.com:3128"
@@ -56,8 +65,9 @@
  */
 
 const { formatLogMessage } = require('./colorize');
+const { ensureRelay, getRelayPort } = require('./socks-relay');
 
-const PROXY_MODULE_VERSION = '1.1.0';
+const PROXY_MODULE_VERSION = '1.2.0';
 const SUPPORTED_PROTOCOLS = ['socks5', 'socks4', 'http', 'https'];
 
 const DEFAULT_PORTS = {
@@ -105,8 +115,12 @@ function parseProxyUrl(proxyUrl) {
     if (!host) return null;
 
     const port = parseInt(url.port, 10) || DEFAULT_PORTS[protocol] || 1080;
-    const username = url.username ? decodeURIComponent(url.username) : null;
-    const password = url.password ? decodeURIComponent(url.password) : null;
+    // decodeURIComponent throws URIError on a literal '%' that isn't a valid
+    // escape (e.g. a password containing '%'). Fall back to the raw value so
+    // an otherwise-valid proxy isn't rejected as "Invalid proxy URL".
+    const safeDecode = (v) => { try { return decodeURIComponent(v); } catch (_) { return v; } };
+    const username = url.username ? safeDecode(url.username) : null;
+    const password = url.password ? safeDecode(url.password) : null;
 
     return { protocol, host, port, username, password };
   } catch (_) {
@@ -124,6 +138,41 @@ function needsProxy(siteConfig) {
   return !!getConfiguredProxy(siteConfig);
 }
 
+/**
+ * Pre-start local no-auth SOCKS5 relays for every distinct authenticated
+ * SOCKS5 upstream across the given site configs. Must be awaited ONCE
+ * before the scan loop — getProxyArgs() then does a pure sync lookup of
+ * the relay port, so the fragile per-batch browser-launch path stays
+ * synchronous.
+ *
+ * @param {object[]} siteConfigs
+ * @param {boolean} forceDebug
+ * @returns {Promise<number>} count of relays started
+ */
+async function prepareSocksRelays(siteConfigs, forceDebug = false) {
+  let started = 0;
+  const seen = new Set();
+  for (const cfg of (siteConfigs || [])) {
+    const url = getConfiguredProxy(cfg);
+    if (!url) continue;
+    const parsed = parseProxyUrl(url);
+    // Only socks5 with credentials needs a relay. socks4-auth stays
+    // unsupported (near-extinct, userId-only); http/https auth works
+    // natively via page.authenticate().
+    if (!parsed || parsed.protocol !== 'socks5' || !parsed.username) continue;
+    const key = `${parsed.host}:${parsed.port}:${parsed.username}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    try {
+      await ensureRelay(parsed, forceDebug);
+      started++;
+    } catch (e) {
+      console.warn(formatLogMessage('proxy', `Failed to start SOCKS5 auth relay for ${parsed.host}:${parsed.port}: ${e.message}`));
+    }
+  }
+  return started;
+}
+
 /**
  * Returns Chromium launch arguments for the configured proxy.
  *
@@ -141,15 +190,45 @@ function getProxyArgs(siteConfig, forceDebug = false) {
     return [];
   }
 
+  // Authenticated SOCKS5: Chromium can't auth SOCKS, so point it at the
+  // local no-auth relay (started upfront by prepareSocksRelays) which does
+  // the upstream auth. Credentials never reach Chromium. The relay speaks
+  // SOCKS5 and forwards domain addresses, so the remote-DNS rule below
+  // still applies correctly to the localhost hop.
+  let effectiveHost = parsed.host;
+  let effectivePort = parsed.port;
+  let effectiveProto = parsed.protocol;
+  if (parsed.protocol === 'socks5' && parsed.username) {
+    const relayPort = getRelayPort(parsed);
+    if (relayPort) {
+      effectiveHost = '127.0.0.1';
+      effectivePort = relayPort;
+      const debug = forceDebug || siteConfig.proxy_debug || siteConfig.socks5_debug;
+      if (debug) {
+        console.log(formatLogMessage('proxy', `SOCKS5 auth via local relay 127.0.0.1:${relayPort} -> ${parsed.host}:${parsed.port}`));
+      }
+    } else {
+      // prepareSocksRelays should have started this; defensive only.
+      console.warn(formatLogMessage('proxy', `No SOCKS5 auth relay for ${parsed.host}:${parsed.port} — call prepareSocksRelays() before the scan. Connection will fail (Chromium can't auth SOCKS).`));
+    }
+  }
+
   const args = [
-    `--proxy-server=${parsed.protocol}://${parsed.host}:${parsed.port}`
+    `--proxy-server=${effectiveProto}://${effectiveHost}:${effectivePort}`
   ];
 
-  // Remote DNS: resolve hostnames through the proxy (prevents DNS leaks)
-  // Only meaningful for SOCKS proxies; HTTP proxies resolve remotely by default
+  // Remote DNS: force proxy-side hostname resolution (prevents DNS leaks).
+  // SOCKS5 only — it can carry a hostname to the proxy for remote
+  // resolution. SOCKS4 cannot (the protocol only accepts an IPv4 address;
+  // resolution must happen client-side), so applying MAP * ~NOTFOUND there
+  // makes Chromium's local resolver fail with nothing able to resolve the
+  // hostname — every request breaks. HTTP/HTTPS proxies resolve remotely
+  // by default and need no rule.
   const remoteDns = siteConfig.proxy_remote_dns ?? siteConfig.socks5_remote_dns;
-  if ((parsed.protocol === 'socks5' || parsed.protocol === 'socks4') && remoteDns !== false) {
+  if (parsed.protocol === 'socks5' && remoteDns !== false) {
     args.push('--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1');
+  } else if (parsed.protocol === 'socks4' && remoteDns === true) {
+    console.warn(formatLogMessage('proxy', `proxy_remote_dns ignored: SOCKS4 cannot do proxy-side DNS resolution (use SOCKS5)`));
   }
 
   // Bypass list: domains that skip the proxy
@@ -182,6 +261,20 @@ async function applyProxyAuth(page, siteConfig, forceDebug = false) {
   const parsed = parseProxyUrl(proxyUrl);
   if (!parsed || !parsed.username) return false;
 
+  // Chromium can't authenticate SOCKS proxies, and page.authenticate() is
+  // HTTP-407-only. SOCKS5+creds is handled out-of-band by the local
+  // no-auth relay (prepareSocksRelays + getProxyArgs rewrite) — Chromium
+  // talks no-auth to 127.0.0.1, so there's nothing for page.authenticate
+  // to do here; return quietly. SOCKS4 auth (userId-only, near-extinct)
+  // stays genuinely unsupported.
+  if (parsed.protocol === 'socks5') {
+    return false; // relay handles upstream auth
+  }
+  if (parsed.protocol === 'socks4') {
+    console.warn(formatLogMessage('proxy', `SOCKS4 proxy auth is unsupported (use SOCKS5, which is auto-relayed, or an authenticated HTTP proxy).`));
+    return false;
+  }
+
   try {
     await page.authenticate({
       username: parsed.username,
@@ -265,9 +358,14 @@ function getModuleInfo() {
   return { version: PROXY_MODULE_VERSION, name: 'Proxy Handler' };
 }
 
+// Re-export relay teardown so nwss.js cleanup paths can close listeners.
+const { closeAllRelays: closeAllSocksRelays } = require('./socks-relay');
+
 module.exports = {
   parseProxyUrl,
   needsProxy,
+  prepareSocksRelays,
+  closeAllSocksRelays,
   getProxyArgs,
   applyProxyAuth,
   testProxy,

package/lib/redirect.js

@@ -165,8 +165,14 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
     // Inject JavaScript redirect detection
     await jsRedirectDetector();
 
-    if (forceDebug && Object.keys(gotoOptions).length > 0) {
-      console.log(formatLogMessage('debug', `Using goto options: ${JSON.stringify(gotoOptions)}`));
+    if (forceDebug) {
+      // Avoid Object.keys allocation just to check emptiness — a for...in
+      // early-exit on the first own key is enough.
+      let hasOpts = false;
+      for (const _k in gotoOptions) { hasOpts = true; break; }
+      if (hasOpts) {
+        console.log(formatLogMessage('debug', `Using goto options: ${JSON.stringify(gotoOptions)}`));
+      }
     }
 
     // Initial navigation. Puppeteer's page.goto returns the response for the
@@ -184,7 +190,7 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
       } catch (_) { /* response disposed or detached — fine, stays null */ }
     }
 
-    if (response && response.url() !== currentUrl) {
+    if (response && response.url() !== currentUrl && !response.url().startsWith('chrome-error://')) {
       // Check redirect limit before adding
       if (redirectChain.length >= maxRedirects) {
         if (forceDebug) {
@@ -192,12 +198,12 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
         }
         finalUrl = currentUrl; // Keep original URL
       } else {
-      finalUrl = response.url();
-      redirected = true;
-      if (!redirectChain.includes(finalUrl)) redirectChain.push(finalUrl);
+        finalUrl = response.url();
+        redirected = true;
+        if (!redirectChain.includes(finalUrl)) redirectChain.push(finalUrl);
       }
       if (forceDebug) {
-        console.log(formatLogMessage('debug', `HTTP redirect detected: ${currentUrl} ? ${finalUrl}`));
+        console.log(formatLogMessage('debug', `HTTP redirect detected: ${currentUrl} -> ${finalUrl}`));
       }
     }
 
@@ -223,9 +229,11 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
           };
         });
         
-        // Check if URL changed (either through JS redirect or automatic redirect)
+        // Check if URL changed (either through JS redirect or automatic redirect).
+        // Skip chrome-error://* — it's Puppeteer's landing page on DNS/connection
+        // failure and adding it to the chain produces bogus intermediate hops.
         const currentPageUrl = page.url();
-        if (currentPageUrl && currentPageUrl !== finalUrl && !redirectChain.includes(currentPageUrl)) {
+        if (currentPageUrl && currentPageUrl !== finalUrl && !currentPageUrl.startsWith('chrome-error://') && !redirectChain.includes(currentPageUrl)) {
           // Check redirect limit before adding
           if (redirectChain.length >= maxRedirects) {
             if (forceDebug) {
@@ -275,21 +283,23 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
       await detectCommonJSRedirects(page, forceDebug, formatLogMessage);
     }
 
-    // Final URL check
+    // Final URL check. Same chrome-error://* skip as the earlier branches —
+    // a navigation that ended in a chrome-error landing shouldn't be treated
+    // as the "final" URL of a successful redirect chain.
     const finalPageUrl = page.url();
-    if (finalPageUrl && finalPageUrl !== finalUrl) {
+    if (finalPageUrl && finalPageUrl !== finalUrl && !finalPageUrl.startsWith('chrome-error://')) {
       // Check redirect limit before final update
       if (redirectChain.length >= maxRedirects) {
         if (forceDebug) {
           console.log(formatLogMessage('debug', `Maximum redirects (${maxRedirects}) reached, keeping current finalUrl`));
         }
       } else {
-      finalUrl = finalPageUrl;
-      redirected = true;
-      if (!redirectChain.includes(finalUrl)) {
-        redirectChain.push(finalUrl);
+        finalUrl = finalPageUrl;
+        redirected = true;
+        if (!redirectChain.includes(finalUrl)) {
+          redirectChain.push(finalUrl);
+        }
       }
-     }
     }
 
   } finally {
@@ -298,21 +308,19 @@ async function navigateWithRedirectHandling(page, currentUrl, siteConfig, gotoOp
 
   // Log redirect summary
   if (redirected && forceDebug) {
-    console.log(formatLogMessage('debug', `Redirect chain: ${redirectChain.join(' ? ')}`));
+    console.log(formatLogMessage('debug', `Redirect chain: ${redirectChain.join(' -> ')}`));
   }
 
-  // Extract redirect domains to exclude from matching
-  let redirectDomains = [];
+  // Extract intermediate redirect domains (exclude the final entry). Single
+  // loop instead of slice().map().filter() — three array allocations down to
+  // one push-loop. redirectChain is bounded at maxRedirects (default 10).
+  const redirectDomains = [];
   if (redirected && redirectChain.length > 1) {
-    // Get all intermediate domains (exclude the final domain)
-    const intermediateDomains = redirectChain.slice(0, -1).map(url => {
+    for (let i = 0; i < redirectChain.length - 1; i++) {
       try {
-        return new URL(url).hostname;
-      } catch {
-        return null;
-      }
-    }).filter(Boolean);
-    redirectDomains = intermediateDomains;
+        redirectDomains.push(new URL(redirectChain[i]).hostname);
+      } catch (_) { /* skip malformed entries */ }
+    }
   }
 
   return { finalUrl, redirected, redirectChain, originalUrl: currentUrl, redirectDomains, httpStatus, cfRay };
@@ -410,13 +418,21 @@ async function handleRedirectTimeout(page, originalUrl, error, safeGetDomain, fo
   
   try {
     const currentPageUrl = page.url();
-    if (currentPageUrl && currentPageUrl !== 'about:blank' && currentPageUrl !== originalUrl) {
+    // Skip chrome-error://* the same way navigateWithRedirectHandling does:
+    // a DNS/connection-failure landing isn't a "partial redirect recovery",
+    // and safeGetDomain('chrome-error://chromewebdata/') returns
+    // 'chromewebdata', which would otherwise differ from the original
+    // domain and falsely report success here.
+    if (currentPageUrl
+        && currentPageUrl !== 'about:blank'
+        && !currentPageUrl.startsWith('chrome-error://')
+        && currentPageUrl !== originalUrl) {
       const originalDomain = safeGetDomain(originalUrl);
       const currentDomain = safeGetDomain(currentPageUrl);
-      
+
       if (originalDomain !== currentDomain) {
         if (forceDebug) {
-          console.log(formatLogMessage('debug', `Partial redirect timeout recovered: ${originalDomain} ? ${currentDomain}`));
+          console.log(formatLogMessage('debug', `Partial redirect timeout recovered: ${originalDomain} -> ${currentDomain}`));
         }
         return { success: true, finalUrl: currentPageUrl, redirected: true };
       }