@fanboynz/network-scanner 3.0.1 → 3.0.3

package/lib/proxy.js

@@ -61,13 +61,19 @@
  *   // After page creation, before page.goto()
  *   await applyProxyAuth(page, siteConfig, forceDebug);
  *
- * @version 1.1.0
+ * @version 1.2.0
  */
 
+const net = require('net');
 const { formatLogMessage } = require('./colorize');
-const { ensureRelay, getRelayPort } = require('./socks-relay');
+const { ensureRelay, getRelayPort, closeAllRelays: closeAllSocksRelays } = require('./socks-relay');
+
+// Note: no separate subsystem TAG here — formatLogMessage('proxy', ...)
+// already emits the `[proxy]` prefix from the severity. socks-relay.js's
+// pattern (`[proxy] [socks-relay] ...`) is correct THERE because its
+// module name differs from the severity. For this file the module IS the
+// severity, so a second '[proxy]' would be redundant double-prefix.
 
-const PROXY_MODULE_VERSION = '1.2.0';
 const SUPPORTED_PROTOCOLS = ['socks5', 'socks4', 'http', 'https'];
 
 const DEFAULT_PORTS = {
@@ -115,6 +121,10 @@ function parseProxyUrl(proxyUrl) {
     if (!host) return null;
 
     const port = parseInt(url.port, 10) || DEFAULT_PORTS[protocol] || 1080;
+    // Reject obvious typos at parse time rather than passing a >65535 port
+    // through to Chromium and getting an opaque downstream error. Port 0
+    // is technically OS-assigned but never a valid proxy target.
+    if (port < 1 || port > 65535) return null;
     // decodeURIComponent throws URIError on a literal '%' that isn't a valid
     // escape (e.g. a password containing '%'). Fall back to the raw value so
     // an otherwise-valid proxy isn't rejected as "Invalid proxy URL".
@@ -186,7 +196,19 @@ function getProxyArgs(siteConfig, forceDebug = false) {
 
   const parsed = parseProxyUrl(proxyUrl);
   if (!parsed) {
-    console.warn(formatLogMessage('proxy', `Invalid proxy URL: ${proxyUrl}`));
+    // Strip user:pass before echoing the URL — same redaction policy as
+    // getProxyInfo() / applyProxyAuth / socks-relay logs. Without this, a
+    // proxy URL with embedded creds (`socks5://user:pass@host:port`) that
+    // fails parse (typo in protocol, port out of range, etc.) leaks the
+    // raw creds to stderr. Regex handles both scheme-prefixed
+    // (`socks5://user:pass@`) and bare (`user:pass@`) forms — the latter
+    // because parseProxyUrl normalises bare host:port internally so the
+    // user-supplied string still reaches here unchanged.
+    const safeUrl = String(proxyUrl).replace(
+      /^([a-z0-9+]+:\/\/)?[^@\s]+@/i,
+      (_m, scheme) => `${scheme || ''}[redacted]@`
+    );
+    console.warn(formatLogMessage('proxy', `Invalid proxy URL: ${safeUrl}`));
     return [];
   }
 
@@ -249,10 +271,20 @@ function getProxyArgs(siteConfig, forceDebug = false) {
  * Applies proxy authentication to a page via Puppeteer's authenticate API.
  * Must be called BEFORE page.goto().
  *
+ * Returns `true` only on a successful HTTP/HTTPS page.authenticate() call.
+ * Returns `false` in five distinct scenarios — callers cannot use the
+ * boolean to distinguish them; treat `false` as "no further action needed
+ * from this module" rather than "auth failed":
+ *   - no proxy configured
+ *   - proxy has no username (anonymous)
+ *   - SOCKS5 with creds  -> the local relay handles upstream auth out-of-band
+ *   - SOCKS4 with creds  -> genuinely unsupported (warned)
+ *   - page.authenticate() threw (warned)
+ *
  * @param {object} page - Puppeteer page instance
  * @param {object} siteConfig
  * @param {boolean} forceDebug
- * @returns {Promise<boolean>} True if auth was applied
+ * @returns {Promise<boolean>}
  */
 async function applyProxyAuth(page, siteConfig, forceDebug = false) {
   const proxyUrl = getConfiguredProxy(siteConfig);
@@ -283,7 +315,11 @@ async function applyProxyAuth(page, siteConfig, forceDebug = false) {
 
     const debug = forceDebug || siteConfig.proxy_debug || siteConfig.socks5_debug;
     if (debug) {
-      console.log(formatLogMessage('proxy', `Auth set for ${parsed.username}@${parsed.host}:${parsed.port}`));
+      // Redact the username — same policy as getProxyInfo() and the
+      // socks-relay logs. debug output gets pasted into support tickets /
+      // screenshots / gists; '[redacted]' keeps the "yes, creds were
+      // attached" signal without disclosing what they were.
+      console.log(formatLogMessage('proxy', `Auth set for [redacted]@${parsed.host}:${parsed.port}`));
     }
 
     return true;
@@ -311,7 +347,6 @@ async function testProxy(siteConfig, timeoutMs = 5000) {
     return { reachable: false, latencyMs: 0, error: 'Invalid proxy URL' };
   }
 
-  const net = require('net');
   const start = Date.now();
 
   return new Promise((resolve) => {
@@ -358,15 +393,11 @@ function getProxyInfo(siteConfig) {
   return `${parsed.protocol}://${auth}${parsed.host}:${parsed.port}`;
 }
 
-/**
- * Returns module version information
- */
-function getModuleInfo() {
-  return { version: PROXY_MODULE_VERSION, name: 'Proxy Handler' };
-}
-
-// Re-export relay teardown so nwss.js cleanup paths can close listeners.
-const { closeAllRelays: closeAllSocksRelays } = require('./socks-relay');
+// getModuleInfo() / PROXY_MODULE_VERSION / SUPPORTED_PROTOCOLS / and now
+// getConfiguredProxy removed from exports -- zero external callers (mirrors
+// the same trim done in lib/cloudflare.js). SUPPORTED_PROTOCOLS and
+// getConfiguredProxy stay as module-local since parseProxyUrl /
+// needsProxy / prepareSocksRelays / getProxyArgs use them.
 
 module.exports = {
   parseProxyUrl,
@@ -376,9 +407,5 @@ module.exports = {
   getProxyArgs,
   applyProxyAuth,
   testProxy,
-  getProxyInfo,
-  getModuleInfo,
-  getConfiguredProxy,
-  PROXY_MODULE_VERSION,
-  SUPPORTED_PROTOCOLS
+  getProxyInfo
 };

package/lib/socks-relay.js

@@ -22,9 +22,24 @@ const { SocksClient } = require('socks');
 const { formatLogMessage, messageColors } = require('./colorize');
 const SOCKS_RELAY_TAG = messageColors.processing('[socks-relay]');
 
-// upstreamKey -> { server, port, activeSockets:Set<net.Socket> }
+// upstreamKey -> {
+//   server: net.Server,                  // listening on 127.0.0.1:port
+//   port: number,                        // OS-assigned local port
+//   activeSockets: Set<net.Socket>,      // live client sockets (Chromium side)
+//   errors: number                       // cumulative upstream-connect failures
+// }
 const _relays = new Map();
 
+// upstreamKey -> Promise<port> currently initialising. Singleflight guard for
+// ensureRelay so two concurrent callers for the same upstream share one
+// in-flight init instead of both creating servers and racing to _relays.set,
+// where the loser's server would be orphaned (listening forever, never
+// closed by closeAllRelays). Not triggered by current usage (proxy.js's
+// prepareSocksRelays uses a sequential await loop) but cheap defence
+// against future callers that don't know to serialise. Mirrors the
+// pendingDigLookups / pendingWhoisLookups pattern in lib/nettools.js.
+const _pendingRelays = new Map();
+
 function upstreamKey(u) {
   return `${u.host}:${u.port}:${u.username || ''}`;
 }
@@ -39,7 +54,28 @@ function upstreamKey(u) {
 // notices (default ~2 hours on Linux).
 const HANDSHAKE_TIMEOUT_MS = 10000;
 
-function handleClient(client, upstream, forceDebug) {
+// Cap pre-piping buffer growth. A real SOCKS5 greeting+request is well
+// under 300 bytes; absorbing more before the watchdog fires lets a hostile
+// or buggy local process drip-feed garbage to pin memory for up to 10s.
+// 4096 is the next clean ceiling above any realistic handshake and matches
+// typical TCP receive-buffer batches.
+const MAX_HANDSHAKE_BYTES = 4096;
+
+// Cap simultaneous local connections per relay. If Chromium opens more than
+// this (prefetch-heavy site, fetch-retry loop), excess gets refused at the
+// TCP-accept layer, which Chromium's HTTP retry logic handles cleanly. The
+// alternative (no cap) is excess tunnels opening to the upstream and the
+// provider silently dropping them past its concurrent-tunnel quota — looks
+// to the scan like random missed requests.
+const MAX_LOCAL_CONNECTIONS = 256;
+
+// On closeAllRelays, give in-flight tunnels this long to drain their
+// response data into Chromium before force-destroying. Without it, SIGINT
+// mid-scan loses any upstream bytes that hadn't yet hit Puppeteer's
+// response listener, leaving incomplete entries in results.json.
+const DRAIN_TIMEOUT_MS = 2000;
+
+function handleClient(client, upstream, forceDebug, relay) {
   let phase = 'greeting';
   let buf = Buffer.alloc(0);
   let upstreamSock = null;
@@ -73,6 +109,21 @@ function handleClient(client, upstream, forceDebug) {
 
   const onData = async (chunk) => {
     buf = Buffer.concat([buf, chunk]);
+    // Reject oversize pre-piping buffers before the 10s watchdog. Sends
+    // a protocol-appropriate failure reply per phase so a misbehaving but
+    // RFC-aware client gets a clean signal rather than a raw connection
+    // drop. Skipped once piping starts (buf is nulled then anyway).
+    if (phase !== 'piping' && buf.length > MAX_HANDSHAKE_BYTES) {
+      if (forceDebug) {
+        console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} handshake oversize (${buf.length} bytes, phase=${phase}) — closing`));
+      }
+      if (phase === 'greeting') {
+        try { client.write(Buffer.from([0x05, 0xFF])); } catch (_) {} // no acceptable methods
+      } else if (phase === 'request') {
+        failReply(client, 0x01); // general SOCKS server failure
+      }
+      return cleanup();
+    }
     try {
       if (phase === 'greeting') {
         // [0x05, NMETHODS, METHODS...]
@@ -134,6 +185,17 @@ function handleClient(client, upstream, forceDebug) {
         phase = 'connecting';
         client.pause();
         client.off('data', onData);
+        // Disarm the handshake watchdog now (not later at 'piping'). The
+        // client has completed its SOCKS5 negotiation; the remaining wait
+        // is on SocksClient.createConnection (which has its own 20s
+        // timeout below). Without this, if the upstream connect takes
+        // longer than HANDSHAKE_TIMEOUT_MS (10s) but less than 20s, the
+        // watchdog fires cleanup mid-await — destroying the client and
+        // setting settled=true — then the upstream connect resolves into
+        // a fresh socket that cleanup() can no longer destroy (settled
+        // guard short-circuits), orphaning an open TCP connection until
+        // OS-level timeout or remote close.
+        if (handshakeTimer) { clearTimeout(handshakeTimer); handshakeTimer = null; }
         const early = buf.subarray(hdrLen);   // any bytes after the request header
         buf = null;
 
@@ -152,6 +214,10 @@ function handleClient(client, upstream, forceDebug) {
             timeout: 20000,
           });
         } catch (e) {
+          // Bump the per-relay error counter (exposed via getRelayStats)
+          // so post-scan diagnostics can see "X of N upstream connects
+          // failed" without re-running with forceDebug.
+          relay.errors++;
           if (forceDebug) {
             console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} upstream connect failed (${host}:${port}): ${e.message}`));
           }
@@ -160,7 +226,28 @@ function handleClient(client, upstream, forceDebug) {
         }
 
         upstreamSock = info.socket;
+        // Safety net: if cleanup() ran while we were awaiting the upstream
+        // connect (some path other than the handshake watchdog — e.g. a
+        // 'close' event on the client during pause), settled is true and
+        // cleanup's settled guard would short-circuit a future call,
+        // orphaning this freshly-connected upstream socket. Destroy it
+        // here directly. With Fix #1a moving the watchdog clearTimeout to
+        // the 'connecting' transition this is currently unreachable, but
+        // cheap to keep as defense-in-depth against future code paths.
+        if (settled) {
+          try { upstreamSock.destroy(); } catch (_) {}
+          return;
+        }
         try { upstreamSock.setNoDelay(true); } catch (_) {}
+        // Catch silently-dead upstreams (NAT timeout, mobile-tower drop,
+        // proxy crash without FIN/RST) faster than the default ~2-hour
+        // Linux idle. setKeepAlive(true, 60000) sets TCP_KEEPIDLE only —
+        // the kernel still uses tcp_keepalive_intvl/tcp_keepalive_probes
+        // for the probe phase, so total detection is ~60s idle + N probes
+        // (default 9 × 75s on Linux) ≈ 12 minutes. Big improvement over
+        // 2h, not the 60s the bare argument suggests. Client-side keep-
+        // alive is omitted — same kernel, OS surfaces death immediately.
+        try { upstreamSock.setKeepAlive(true, 60000); } catch (_) {}
         upstreamSock.on('error', cleanup);
         upstreamSock.on('close', cleanup);
         client.on('error', cleanup);
@@ -173,9 +260,8 @@ function handleClient(client, upstream, forceDebug) {
         upstreamSock.pipe(client);
         client.resume();
         phase = 'piping';
-        // Negotiation complete — disarm the handshake watchdog so a
-        // long-running download isn't killed mid-transfer.
-        if (handshakeTimer) { clearTimeout(handshakeTimer); handshakeTimer = null; }
+        // (handshakeTimer was already disarmed at the 'connecting'
+        // transition above; no second clearTimeout needed.)
       }
     } catch (e) {
       if (forceDebug) {
@@ -207,41 +293,86 @@ async function ensureRelay(upstream, forceDebug = false) {
   const existing = _relays.get(key);
   if (existing) return existing.port;
 
-  const activeSockets = new Set();
-  const server = net.createServer((clientSock) => {
-    // Disable Nagle: page scanning is full of small-packet phases (per-origin
-    // TLS handshakes, small XHR/API calls, the SOCKS handshake itself).
-    // Nagle + delayed-ACK adds ~40ms stalls on those; relays should not.
-    try { clientSock.setNoDelay(true); } catch (_) {}
-    activeSockets.add(clientSock);
-    clientSock.on('close', () => activeSockets.delete(clientSock));
-    handleClient(clientSock, upstream, forceDebug);
-  });
+  // Singleflight: if another caller is already initialising this upstream,
+  // ride its promise instead of starting a parallel init. Prevents the
+  // race where two concurrent callers both pass the _relays.get(key) check
+  // above, both create servers, and the second _relays.set(key, ...) below
+  // orphans the first server (listening forever, never closed).
+  if (_pendingRelays.has(key)) return _pendingRelays.get(key);
+
+  // Most authenticated SOCKS5 servers reject empty-password auth at the
+  // RFC 1929 handshake; without this warn, the misconfig surfaces only
+  // per-request inside forceDebug-gated logs (silent in production).
+  // Fire once per unique upstream (after the existing-relay short-circuit
+  // above) so repeated calls don't spam.
+  if (upstream.username && !upstream.password) {
+    console.warn(formatLogMessage('warn', `${SOCKS_RELAY_TAG} upstream ${upstream.host}:${upstream.port} has username but no password — RFC 1929 auth will likely fail`));
+  }
+
+  // .finally() (not try/finally inside the IIFE) so the cleanup is
+  // scheduled in a microtask, guaranteed to run AFTER the _pendingRelays.set
+  // below. If the cleanup were a try/finally inside an async IIFE and the
+  // body threw SYNCHRONOUSLY (before its first await), the finally would
+  // run SYNC before the implicit rejected promise was returned, _pendingRelays
+  // wouldn't be set yet, the delete would no-op, and then the outer .set
+  // would register a permanent rejected entry that future callers would
+  // await forever. The current body has no realistic sync-throw paths
+  // (net.createServer / Set / object literal don't throw), but defensive.
+  const initPromise = (async () => {
+    const activeSockets = new Set();
+    // Single mutable state object referenced by both the connection handler
+    // (writes .errors) and _relays / getRelayStats (read both). Server +
+    // port assigned after listen() completes; declared up-front so the
+    // closure below can close over `relayEntry` and pass it to handleClient.
+    const relayEntry = { server: null, port: null, activeSockets, errors: 0 };
+
+    const server = net.createServer((clientSock) => {
+      // Disable Nagle: page scanning is full of small-packet phases (per-origin
+      // TLS handshakes, small XHR/API calls, the SOCKS handshake itself).
+      // Nagle + delayed-ACK adds ~40ms stalls on those; relays should not.
+      try { clientSock.setNoDelay(true); } catch (_) {}
+      activeSockets.add(clientSock);
+      clientSock.on('close', () => activeSockets.delete(clientSock));
+      handleClient(clientSock, upstream, forceDebug, relayEntry);
+    });
 
-  await new Promise((resolve, reject) => {
-    const onErr = (e) => reject(e);
-    server.once('error', onErr);
-    server.listen(0, '127.0.0.1', () => {
-      server.removeListener('error', onErr);
-      // Keep a listener so a late server error doesn't crash the process.
-      server.on('error', (e) => {
-        if (forceDebug) console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} server error: ${e.message}`));
+    // Shed excess connections at the TCP-accept layer instead of letting them
+    // all proceed to open authenticated tunnels (which the upstream provider
+    // may silently drop past its quota).
+    server.maxConnections = MAX_LOCAL_CONNECTIONS;
+
+    await new Promise((resolve, reject) => {
+      const onErr = (e) => reject(e);
+      server.once('error', onErr);
+      server.listen(0, '127.0.0.1', () => {
+        server.removeListener('error', onErr);
+        // Keep a listener so a late server error doesn't crash the process.
+        server.on('error', (e) => {
+          if (forceDebug) console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} server error: ${e.message}`));
+        });
+        resolve();
       });
-      resolve();
     });
+
+    relayEntry.server = server;
+    relayEntry.port = server.address().port;
+    _relays.set(key, relayEntry);
+    const port = relayEntry.port;
+    if (forceDebug) {
+      // auth status is kept as a presence flag only -- previously printed
+      // the raw username, which leaked into shared debug output (support
+      // tickets, screenshots, gists). Same redaction policy as the
+      // proxy.js getProxyInfo() change.
+      const authTag = upstream.username ? ' (auth: [redacted])' : ' (no auth)';
+      console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} 127.0.0.1:${port} -> ${upstream.host}:${upstream.port}${authTag}`));
+    }
+    return port;
+  })().finally(() => {
+    _pendingRelays.delete(key);
   });
 
-  const port = server.address().port;
-  _relays.set(key, { server, port, activeSockets });
-  if (forceDebug) {
-    // auth status is kept as a presence flag only -- previously printed
-    // the raw username, which leaked into shared debug output (support
-    // tickets, screenshots, gists). Same redaction policy as the
-    // proxy.js getProxyInfo() change.
-    const authTag = upstream.username ? ' (auth: [redacted])' : ' (no auth)';
-    console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} 127.0.0.1:${port} -> ${upstream.host}:${upstream.port}${authTag}`));
-  }
-  return port;
+  _pendingRelays.set(key, initPromise);
+  return initPromise;
 }
 
 /**
@@ -255,26 +386,90 @@ function getRelayPort(upstream) {
 }
 
 /**
- * Tear down every relay: destroy in-flight sockets, close listeners.
- * Safe to call multiple times.
+ * Snapshot of active relays for diagnostics. Returns an array of
+ * { key, port, activeConnections, errors } — the upstream `key` has its
+ * trailing `:username` segment stripped using the same regex as
+ * closeAllRelays' display path (IPv6-safe). `errors` is the cumulative
+ * count of failed upstream-tunnel opens for the relay's lifetime.
+ * Useful for answering "is my proxy slow because the upstream is
+ * saturated, or because the scan is opening too many parallel tunnels?"
+ * without enabling forceDebug.
+ */
+function getRelayStats() {
+  const stats = [];
+  for (const [key, r] of _relays) {
+    stats.push({
+      key: key.replace(/:[^:]*$/, ''),
+      port: r.port,
+      activeConnections: r.activeSockets.size,
+      errors: r.errors
+    });
+  }
+  return stats;
+}
+
+/**
+ * Tear down every relay. Stops accepting new connections, gives in-flight
+ * tunnels up to DRAIN_TIMEOUT_MS (2s) to flush remaining response bytes
+ * into Chromium / Puppeteer, then force-destroys any stragglers. Safe to
+ * call multiple times (subsequent calls iterate an empty _relays Map).
  */
 async function closeAllRelays(forceDebug = false) {
+  // Wait for any in-flight ensureRelay inits to finish before snapshotting
+  // _relays. Without this, a relay whose listen() completes AFTER our
+  // iteration starts would land in _relays unowned by closeAllRelays —
+  // leaked until next call or process exit. allSettled (not all) because
+  // a rejected init has already cleaned up its _pendingRelays entry via
+  // .finally; we just need to not throw here.
+  if (_pendingRelays.size > 0) {
+    await Promise.allSettled(Array.from(_pendingRelays.values()));
+  }
   for (const [key, r] of _relays) {
-    for (const s of r.activeSockets) { try { s.destroy(); } catch (_) {} }
-    await new Promise((res) => {
+    // upstreamKey embeds the username (host:port:username), so the raw
+    // key would leak it in debug output. Strip just the trailing
+    // `:username` segment for display; using a regex (not split-on-':')
+    // so IPv6 hosts with embedded colons (e.g. 2001:db8::1:1080:user)
+    // aren't mangled. The relay identity stays unambiguous from host+port.
+    const displayKey = key.replace(/:[^:]*$/, '');
+    const startedWith = r.activeSockets.size;
+
+    // server.close() stops accepting new connections and resolves only
+    // when all existing sockets have closed naturally. Race that against
+    // DRAIN_TIMEOUT_MS: if active tunnels finish flushing in time,
+    // Chromium / Puppeteer gets the response bytes it was waiting for;
+    // beyond that, force-destroy stragglers (the close callback then
+    // fires immediately). Trade-off chosen so SIGINT mid-scan doesn't
+    // amputate in-flight responses but a hung tunnel can't block exit.
+    const closePromise = new Promise((res) => {
       try { r.server.close(() => res()); } catch (_) { res(); }
     });
+
+    let timer;
+    const drained = await Promise.race([
+      closePromise.then(() => { if (timer) clearTimeout(timer); return true; }),
+      new Promise((res) => {
+        timer = setTimeout(() => res(false), DRAIN_TIMEOUT_MS);
+        // Don't hold the event loop open on a still-pending drain timer
+        // when the close-promise won the race.
+        if (typeof timer.unref === 'function') timer.unref();
+      })
+    ]);
+
+    let forcedCount = 0;
+    if (!drained) {
+      forcedCount = r.activeSockets.size;
+      for (const s of r.activeSockets) { try { s.destroy(); } catch (_) {} }
+      await closePromise; // resolves now that the last socket has closed
+    }
+
     if (forceDebug) {
-      // upstreamKey embeds the username (host:port:username), so the raw
-      // key would leak it in debug output. Strip just the trailing
-      // `:username` segment for display; using a regex (not split-on-':')
-      // so IPv6 hosts with embedded colons (e.g. 2001:db8::1:1080:user)
-      // aren't mangled. The relay identity stays unambiguous from host+port.
-      const displayKey = key.replace(/:[^:]*$/, '');
-      console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} closed relay for ${displayKey}`));
+      const note = forcedCount > 0
+        ? ` (drain timeout — force-closed ${forcedCount}/${startedWith} active socket(s))`
+        : (startedWith > 0 ? ` (drained ${startedWith} active socket(s))` : '');
+      console.log(formatLogMessage('proxy', `${SOCKS_RELAY_TAG} closed relay for ${displayKey}${note}`));
     }
   }
   _relays.clear();
 }
 
-module.exports = { ensureRelay, getRelayPort, closeAllRelays };
+module.exports = { ensureRelay, getRelayPort, getRelayStats, closeAllRelays };