@fanboynz/network-scanner 2.0.66 → 3.0.0

package/lib/flowproxy.js

@@ -10,10 +10,12 @@
  * - Requires specific handling for security scanning tools
  */
 
-/**
- * Module version information
- */
-const FLOWPROXY_MODULE_VERSION = '1.0.0';
+const { formatLogMessage, messageColors } = require('./colorize');
+
+// Precomputed colored '[flowproxy]' subsystem prefix. formatLogMessage only
+// colors the [severity] tag; this constant colors the subsystem prefix so
+// '[debug] [flowproxy] X' has both tags visually distinct.
+const FLOWPROXY_TAG = messageColors.processing('[flowproxy]');
 
 /**
  * Timeout constants for FlowProxy operations (in milliseconds)
@@ -22,25 +24,173 @@ const FLOWPROXY_MODULE_VERSION = '1.0.0';
 const TIMEOUTS = {
   PAGE_EVALUATION_SAFE: 10000,    // Safe page evaluation timeout
   // FlowProxy-specific timeouts
-  PAGE_LOAD_WAIT: 2000,           // Initial wait for page to load
   JS_CHALLENGE_DEFAULT: 15000,    // Default JavaScript challenge timeout
   RATE_LIMIT_DEFAULT: 30000,      // Default rate limit delay
-  ADDITIONAL_DELAY_DEFAULT: 5000, // Default additional processing delay
   PAGE_TIMEOUT_DEFAULT: 45000,    // Default page timeout
-  NAVIGATION_TIMEOUT_DEFAULT: 45000, // Default navigation timeout
+  NAVIGATION_TIMEOUT_DEFAULT: 45000 // Default navigation timeout
 };
 
+// Default-false detection shape returned by the catch paths in
+// safePageEvaluate / analyzeFlowProxyProtection. Hoisted so the two
+// catch branches don't drift if a new detection flag is added.
+const DEFAULT_DETECTION = Object.freeze({
+  isFlowProxyDetected: false,
+  hasSpecificSignal: false,
+  hasFlowProxyDomain: false,
+  hasFlowProxyElements: false,
+  hasFlowProxyScripts: false,
+  hasFlowProxyBrandText: false,
+  hasFlowProxyHeaders: false,
+  hasFlowProxyCookies: false,
+  matchedHeader: null,
+  matchedCookie: null,
+  hasProtectionPage: false,
+  hasChallengeElements: false,
+  isRateLimited: false,
+  hasJSChallenge: false,
+  isProcessing: false
+});
+
+// === HTTP RESPONSE HEADER / COOKIE DETECTION =================================
+// Per-page accumulator for vendor-specific HTTP response signals. Populated
+// by the response listener attached via attachFlowProxyHeaderListener();
+// read by analyzeFlowProxyProtection() to merge with the DOM/text scan.
+//
+// WeakMap so the entry is released when Puppeteer drops the page reference —
+// no manual cleanup needed.
+const pageHeaderState = new WeakMap();
+
+// Header/cookie tokens that uniquely identify FlowProxy/Aurologic. Lowercase
+// for case-insensitive matching (response.headers() returns lowercase keys
+// but values keep their case).
+const VENDOR_TOKENS = ['flowproxy', 'aurologic'];
+
+// Header names where a vendor token in the VALUE is a strong signal.
+// (Server, Via, X-Powered-By, X-Cache, X-CDN are all places a CDN/proxy
+// commonly self-identifies.)
+const VENDOR_VALUE_HEADERS = ['server', 'via', 'x-powered-by', 'x-cache', 'x-cdn'];
+
+// Single source of truth for "is this cookie name vendor-namespaced?"
+// Used both by the Set-Cookie listener (parsing header text) and the jar
+// check in analyzeFlowProxyProtection (cookies()-API results), so the two
+// sides can't disagree about what counts.
+function isVendorCookieName(name) {
+  if (!name) return false;
+  const n = name.toLowerCase();
+  return n === 'flowproxy' || n === 'aurologic' ||
+         n.startsWith('flowproxy_') || n.startsWith('aurologic_') ||
+         n.startsWith('flowproxy-') || n.startsWith('aurologic-');
+}
+
+// Default-empty listener state. One source of truth for the shape, used
+// at attach time (initial state) and at read time (fallback when the
+// listener was never attached). Add a new signal field once, applied
+// everywhere.
+function emptyHeaderState() {
+  return {
+    hasFlowProxyHeaders: false,
+    hasFlowProxyCookies: false,
+    matchedHeader: null,
+    matchedCookie: null
+  };
+}
+
+/**
+ * Attach a response listener to a page that watches for FlowProxy/Aurologic
+ * HTTP response headers + cookies. Idempotent — safe to call multiple times.
+ *
+ * Headers are the most reliable signal: DOM scraping can be fooled by any
+ * "Please wait" / "Loading" string, but a `Server: flowProxy` header is
+ * uniquely the vendor's. Cookies likewise — `flowproxy_*` / `aurologic_*`
+ * names don't collide with anything else in practice.
+ *
+ * Call BEFORE page.goto() so the navigation response itself is observed.
+ * State is read later via analyzeFlowProxyProtection().
+ *
+ * @param {import('puppeteer').Page} page - Puppeteer page instance
+ */
+function attachFlowProxyHeaderListener(page) {
+  if (pageHeaderState.has(page)) return; // idempotent
+
+  const state = emptyHeaderState();
+  pageHeaderState.set(page, state);
+
+  page.on('response', (response) => {
+    // Once both signals are found there's nothing more to learn — bail
+    // immediately to keep per-response overhead near zero on long pages.
+    if (state.hasFlowProxyHeaders && state.hasFlowProxyCookies) return;
+
+    try {
+      const headers = response.headers();
+      if (!headers) return;
+
+      // 1) Vendor-token search across the well-known value-bearing headers.
+      if (!state.hasFlowProxyHeaders) {
+        for (const h of VENDOR_VALUE_HEADERS) {
+          const v = headers[h];
+          if (!v) continue;
+          const vl = v.toLowerCase();
+          for (const tok of VENDOR_TOKENS) {
+            if (vl.includes(tok)) {
+              state.hasFlowProxyHeaders = true;
+              state.matchedHeader = `${h}: ${v}`;
+              break;
+            }
+          }
+          if (state.hasFlowProxyHeaders) break;
+        }
+      }
+
+      // 2) Any X-FlowProxy-* or X-Aurologic-* custom header name — those
+      //    are vendor-namespaced by convention and don't collide.
+      if (!state.hasFlowProxyHeaders) {
+        for (const key of Object.keys(headers)) {
+          // key is already lowercase per Puppeteer's headers() contract
+          if (key.startsWith('x-flowproxy-') || key.startsWith('x-aurologic-')) {
+            state.hasFlowProxyHeaders = true;
+            state.matchedHeader = `${key}: ${headers[key]}`;
+            break;
+          }
+        }
+      }
+
+      // 3) Set-Cookie parsing — extract each cookie's NAME (substring
+      //    before the first '=') and apply the shared vendor-name
+      //    predicate. The old substring-on-value match could false-fire
+      //    on names like `__flowproxy=` or `notaurologic_x=` that contain
+      //    the token without being vendor cookies; the name-level check
+      //    matches the jar inspection in analyzeFlowProxyProtection
+      //    exactly. Puppeteer joins multi-Set-Cookie with '\n'.
+      if (!state.hasFlowProxyCookies) {
+        const setCookie = headers['set-cookie'];
+        if (setCookie) {
+          const lines = setCookie.split('\n');
+          for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            const eq = line.indexOf('=');
+            if (eq <= 0) continue;
+            const name = line.slice(0, eq).trim();
+            if (isVendorCookieName(name)) {
+              state.hasFlowProxyCookies = true;
+              state.matchedCookie = name;
+              break;
+            }
+          }
+        }
+      }
+    } catch (_) {
+      // Observation-only — never let a header read throw into Puppeteer's
+      // event-emitter chain.
+    }
+  });
+}
+
 // Fast timeout constants - optimized for speed while respecting FlowProxy delays
 const FAST_TIMEOUTS = {
   PAGE_LOAD_WAIT: 1500,           // Reduced from 2000ms
-  ADDITIONAL_DELAY_DEFAULT: 3000, // Reduced from 5000ms
-  FALLBACK_TIMEOUT: 3000          // Reduced from 5000ms
+  ADDITIONAL_DELAY_DEFAULT: 3000  // Reduced from 5000ms
 };
 
-/**
- * Gets module version information
- * @returns {object} Version information object
- */
 // Protocols to skip — FlowProxy only protects web traffic
 const SKIP_PATTERNS = [
   'about:', 'chrome:', 'chrome-extension:', 'chrome-error:', 'chrome-search:',
@@ -48,13 +198,6 @@ const SKIP_PATTERNS = [
   'data:', 'blob:', 'javascript:', 'vbscript:', 'file:', 'ftp:', 'ftps:'
 ];
 
-function getModuleInfo() {
-  return {
-    version: FLOWPROXY_MODULE_VERSION,
-    name: 'FlowProxy Protection Handler'
-  };
-}
-
 /**
  * Validates if a URL should be processed by FlowProxy protection
  * Only allows HTTP/HTTPS URLs, skips browser-internal and special protocols
@@ -75,7 +218,7 @@ function getModuleInfo() {
  */
 function shouldProcessUrl(url, forceDebug = false) {
   if (!url || typeof url !== 'string') {
-    if (forceDebug) console.log(`[flowproxy][url-validation] Skipping invalid URL: ${url}`);
+    if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}[url-validation] Skipping invalid URL: ${url}`));
     return false;
   }
 
@@ -84,7 +227,7 @@ function shouldProcessUrl(url, forceDebug = false) {
   for (const pattern of SKIP_PATTERNS) {
     if (urlLower.startsWith(pattern)) {
       if (forceDebug) {
-        console.log(`[flowproxy][url-validation] Skipping ${pattern} URL: ${url.substring(0, 100)}${url.length > 100 ? '...' : ''}`);
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}[url-validation] Skipping ${pattern} URL: ${url.substring(0, 100)}${url.length > 100 ? '...' : ''}`));
       }
       return false;
     }
@@ -93,7 +236,7 @@ function shouldProcessUrl(url, forceDebug = false) {
   // Only process HTTP/HTTPS URLs - FlowProxy only protects web traffic
   if (!urlLower.startsWith('http://') && !urlLower.startsWith('https://')) {
     if (forceDebug) {
-      console.log(`[flowproxy][url-validation] Skipping non-HTTP(S) URL: ${url.substring(0, 100)}${url.length > 100 ? '...' : ''}`);
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}[url-validation] Skipping non-HTTP(S) URL: ${url.substring(0, 100)}${url.length > 100 ? '...' : ''}`));
     }
     return false;
   }
@@ -118,19 +261,22 @@ async function waitForTimeout(page, timeout) {
  * Safe page evaluation with timeout protection for FlowProxy analysis
  */
 async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_SAFE) {
+  let timer;
   try {
     return await Promise.race([
       page.evaluate(func),
-      new Promise((_, reject) => 
-        setTimeout(() => reject(new Error('FlowProxy page evaluation timeout')), timeout)
-      )
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error('FlowProxy page evaluation timeout')), timeout);
+      })
     ]);
   } catch (error) {
-    // Return safe defaults if evaluation fails
-    return {
-      isFlowProxyDetected: false,
-      error: error.message
-    };
+    // Return full default-false shape so downstream `.hasProtectionPage`
+    // etc. read as `false` instead of `undefined` — keeps debug logs
+    // honest and conditional branches in handleFlowProxyProtection
+    // deterministic.
+    return { ...DEFAULT_DETECTION, error: error.message };
+  } finally {
+    if (timer) clearTimeout(timer);
   }
 }
 
@@ -159,9 +305,10 @@ async function safePageEvaluate(page, func, timeout = TIMEOUTS.PAGE_EVALUATION_S
  */
 async function analyzeFlowProxyProtection(page) {
   try {
-    // Get current page URL and validate it first
-    const currentPageUrl = await page.url();
-    
+    // Get current page URL and validate it first.
+    // page.url() is synchronous in Puppeteer 20+; no await needed.
+    const currentPageUrl = page.url();
+
     if (!shouldProcessUrl(currentPageUrl, false)) {
       return {
         isFlowProxyDetected: false,
@@ -170,73 +317,103 @@ async function analyzeFlowProxyProtection(page) {
       };
     }
 
+    // Pull HTTP-layer signals collected by the response listener (populated
+    // by attachFlowProxyHeaderListener if the caller wired it up before
+    // navigation). Falls back to empty-defaults if the listener was never
+    // attached, so DOM-only detection still works.
+    const httpState = pageHeaderState.get(page) || emptyHeaderState();
+
     // Continue with comprehensive FlowProxy detection for valid HTTP(S) URLs
-    return await safePageEvaluate(page, () => {
+    const domResult = await safePageEvaluate(page, () => {
       const title = document.title || '';
       const bodyText = document.body ? document.body.textContent : '';
       const url = window.location.href;
-      
-      // Check for flowProxy/aurologic specific domain indicators
-      // FlowProxy services often redirect to aurologic domains or use flowproxy subdomains
-      const hasFlowProxyDomain = url.includes('aurologic') || 
-                                 url.includes('flowproxy') ||
-                                 url.includes('ddos-protection');
-      
-      // Check for flowProxy challenge page indicators
-      // These are common titles and text patterns used by FlowProxy protection pages
-      const hasProtectionPage = title.includes('DDoS Protection') ||
-                               title.includes('Please wait') ||
-                               title.includes('Checking your browser') ||
-                               bodyText.includes('DDoS protection by aurologic') ||
-                               bodyText.includes('flowProxy') ||
-                               bodyText.includes('Verifying your browser');
-      
-      // Check for specific flowProxy DOM elements
-      // FlowProxy typically adds custom data attributes and CSS classes
+
+      // === VENDOR-SPECIFIC SIGNALS (high-confidence FlowProxy markers) ===
+      // Anything here is unambiguous: it names FlowProxy or its parent
+      // company Aurologic. At least ONE of these must be present for the
+      // primary detection to fire — generic loaders / Cloudflare's
+      // "Checking your browser" / SPA spinners alone do NOT count.
+
+      // URL signals — note: 'ddos-protection' was moved out of this set;
+      // it's too broad (matches docs/blog URLs about DDoS protection).
+      const hasFlowProxyDomain = url.includes('aurologic') ||
+                                 url.includes('flowproxy');
+
+      // DOM signals tied to the vendor's class/id/data-attribute namespace
+      // or its uniquely named challenge input.
       const hasFlowProxyElements = document.querySelector('[data-flowproxy]') !== null ||
-                                  document.querySelector('.flowproxy-challenge') !== null ||
-                                  document.querySelector('#flowproxy-container') !== null ||
-                                  document.querySelector('.aurologic-protection') !== null;
-      
-      // Check for challenge indicators
-      // FlowProxy uses various elements to indicate active challenges
-      const hasChallengeElements = document.querySelector('.challenge-running') !== null ||
-                                  document.querySelector('.verification-container') !== null ||
-                                  document.querySelector('input[name="flowproxy-response"]') !== null;
-      
-      // Check for rate limiting indicators
-      // Rate limiting is a common FlowProxy feature that shows specific messages
+                                   document.querySelector('.flowproxy-challenge') !== null ||
+                                   document.querySelector('#flowproxy-container') !== null ||
+                                   document.querySelector('.aurologic-protection') !== null ||
+                                   document.querySelector('input[name="flowproxy-response"]') !== null;
+
+      // Script src patterns from the vendor.
+      const hasFlowProxyScripts = document.querySelector('script[src*="flowproxy"]') !== null ||
+                                  document.querySelector('script[src*="aurologic"]') !== null;
+
+      // Brand-name strings — "flowProxy" (cased) and the canonical
+      // Aurologic attribution line.
+      const hasFlowProxyBrandText = bodyText.includes('DDoS protection by aurologic') ||
+                                    bodyText.includes('flowProxy');
+
+      // DOM-side specific signals only. The Node caller below merges this
+      // with HTTP-header / cookie signals (which live outside the page
+      // context) to produce the final hasSpecificSignal.
+      const domSpecificSignal = hasFlowProxyDomain ||
+                                hasFlowProxyElements ||
+                                hasFlowProxyScripts ||
+                                hasFlowProxyBrandText;
+
+      // === GENERIC SIGNALS (low-confidence; used for sub-handling only) ===
+      // These flags help the handler decide WHICH delay to apply once
+      // FlowProxy presence is already confirmed by a specific signal.
+      // They are NOT inputs to isFlowProxyDetected — by themselves they
+      // collide with Cloudflare, Sucuri, generic SPA loaders, etc.
+
+      // Generic protection-page text (kept for verification-step semantics
+      // and debug logging — exposed as `hasProtectionPage` for backward
+      // compat with the rest of the module).
+      const hasProtectionPage = hasFlowProxyBrandText ||
+                                title.includes('DDoS Protection') ||
+                                title.includes('Please wait') ||
+                                title.includes('Checking your browser') ||
+                                bodyText.includes('Verifying your browser') ||
+                                url.includes('ddos-protection');
+
+      // Generic challenge-element markers (still exposed for the handler's
+      // sub-decisions; hasFlowProxyElements above is the strong subset).
+      const hasChallengeElements = hasFlowProxyElements ||
+                                   document.querySelector('.challenge-running') !== null ||
+                                   document.querySelector('.verification-container') !== null;
+
       const isRateLimited = bodyText.includes('Rate limited') ||
-                           bodyText.includes('Too many requests') ||
-                           bodyText.includes('Please try again later') ||
-                           title.includes('429') ||
-                           title.includes('Rate Limit');
-      
-      // Check for JavaScript challenge indicators
-      // FlowProxy often requires JavaScript to be enabled and uses specific scripts
-      const hasJSChallenge = document.querySelector('script[src*="flowproxy"]') !== null ||
-                            document.querySelector('script[src*="aurologic"]') !== null ||
-                            bodyText.includes('JavaScript is required') ||
-                            bodyText.includes('Please enable JavaScript');
-      
-      // Check for loading/processing indicators
-      // FlowProxy shows these while performing browser verification
+                            bodyText.includes('Too many requests') ||
+                            bodyText.includes('Please try again later') ||
+                            title.includes('429') ||
+                            title.includes('Rate Limit');
+
+      // hasJSChallenge gates the wait-for-challenge-completion path. The
+      // vendor script-src patterns are strong; the JS-required strings are
+      // generic but only matter when hasSpecificSignal already gated us in.
+      const hasJSChallenge = hasFlowProxyScripts ||
+                             bodyText.includes('JavaScript is required') ||
+                             bodyText.includes('Please enable JavaScript');
+
       const isProcessing = bodyText.includes('Processing') ||
-                          bodyText.includes('Loading') ||
-                          document.querySelector('.loading-spinner') !== null ||
-                          document.querySelector('.processing-indicator') !== null;
-      
-      // Main detection logic - any of these primary indicators suggest FlowProxy presence
-      const isFlowProxyDetected = hasFlowProxyDomain || 
-                                 hasProtectionPage || 
-                                 hasFlowProxyElements || 
-                                 hasChallengeElements;
-      
+                           bodyText.includes('Loading') ||
+                           document.querySelector('.loading-spinner') !== null ||
+                           document.querySelector('.processing-indicator') !== null;
+
+      // The Node-side caller merges this with HTTP signals to compute
+      // the final hasSpecificSignal / isFlowProxyDetected.
       return {
-        isFlowProxyDetected,
+        domSpecificSignal,
         hasFlowProxyDomain,
-        hasProtectionPage,
         hasFlowProxyElements,
+        hasFlowProxyScripts,
+        hasFlowProxyBrandText,
+        hasProtectionPage,
         hasChallengeElements,
         isRateLimited,
         hasJSChallenge,
@@ -246,19 +423,61 @@ async function analyzeFlowProxyProtection(page) {
         bodySnippet: bodyText.substring(0, 200) // First 200 chars for debugging
       };
     });
-  } catch (error) {
-    // Return safe defaults if page evaluation fails
+
+    // Cookie-jar check: complements the Set-Cookie response-header listener
+    // by reading what's ACTUALLY persisted in the browser jar. Catches:
+    //   - cookies set on prior visits (session-reuse scenarios)
+    //   - cookies set via document.cookie = '...' from page JS
+    //   - Set-Cookie emitted before the listener attached (defensive)
+    // Uses isVendorCookieName so the predicate matches the listener.
+    // Try/catch because page.cookies() throws on closed/detached pages.
+    let hasJarCookie = false;
+    let jarMatchedCookie = null;
+    try {
+      const cookies = await page.cookies();
+      if (Array.isArray(cookies)) {
+        for (let i = 0; i < cookies.length; i++) {
+          if (isVendorCookieName(cookies[i].name)) {
+            hasJarCookie = true;
+            jarMatchedCookie = cookies[i].name;
+            break;
+          }
+        }
+      }
+    } catch (_) {
+      // Observation-only — never fail detection because the jar read errored.
+    }
+
+    // Unified merge: works for both the success path AND the DOM-error
+    // path. On error, domResult is `{...DEFAULT_DETECTION, error}` so
+    // domSpecificSignal is undefined; isFlowProxyDetected still becomes
+    // true if HTTP signals fired. Previously the error path returned
+    // early before recomputing primary detection, silently dropping
+    // header/cookie evidence whenever the DOM eval errored.
+    const hasFlowProxyCookies = httpState.hasFlowProxyCookies || hasJarCookie;
+    const hasSpecificSignal = (domResult && domResult.domSpecificSignal) ||
+                              httpState.hasFlowProxyHeaders ||
+                              hasFlowProxyCookies;
+
     return {
-      isFlowProxyDetected: false,
-      hasFlowProxyDomain: false,
-      hasProtectionPage: false,
-      hasFlowProxyElements: false,
-      hasChallengeElements: false,
-      isRateLimited: false,
-      hasJSChallenge: false,
-      isProcessing: false,
-      error: error.message
+      ...domResult, // includes .error on the safePageEvaluate failure path
+      hasFlowProxyHeaders: httpState.hasFlowProxyHeaders,
+      hasFlowProxyCookies,
+      matchedHeader: httpState.matchedHeader,
+      // Listener-captured name wins over jar name when both fire — the
+      // listener saw the cookie at the moment it was set, which is the
+      // more informative time-of-event for debug output.
+      matchedCookie: httpState.matchedCookie || jarMatchedCookie,
+      hasSpecificSignal,
+      // PRIMARY DETECTION: at least one vendor-specific signal across DOM
+      // OR HTTP layer. Headers are the most reliable signal; cookies
+      // close behind. DOM markers remain the fallback for sites where
+      // the listener wasn't wired up before navigation.
+      isFlowProxyDetected: hasSpecificSignal
     };
+  } catch (error) {
+    // Return safe defaults if page evaluation fails
+    return { ...DEFAULT_DETECTION, error: error.message };
   }
 }
 
@@ -314,7 +533,7 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
   // FlowProxy only protects web traffic, so other protocols should be skipped
   if (!shouldProcessUrl(currentUrl, forceDebug)) {
     if (forceDebug) {
-      console.log(`[debug][flowproxy] Skipping protection handling for non-HTTP(S) URL: ${currentUrl}`);
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Skipping protection handling for non-HTTP(S) URL: ${currentUrl}`));
     }
     return {
       flowProxyDetection: { attempted: false, detected: false },
@@ -336,7 +555,7 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
   };
 
   try {
-    if (forceDebug) console.log(`[debug][flowproxy] Checking for flowProxy protection on ${currentUrl}`);
+    if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Checking for flowProxy protection on ${currentUrl}`));
     
     // Wait for initial page load before analyzing
     // FlowProxy protection pages need time to fully render their elements
@@ -355,15 +574,23 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
       result.handlingResult.attempted = true;
       
       if (forceDebug) {
-        console.log(`[debug][flowproxy] FlowProxy protection detected on ${currentUrl}:`);
-        console.log(`[debug][flowproxy]   Page Title: "${detectionInfo.title}"`);
-        console.log(`[debug][flowproxy]   Current URL: ${detectionInfo.url}`);
-        console.log(`[debug][flowproxy]   Has Protection Page: ${detectionInfo.hasProtectionPage}`);
-        console.log(`[debug][flowproxy]   Has Challenge Elements: ${detectionInfo.hasChallengeElements}`);
-        console.log(`[debug][flowproxy]   Is Rate Limited: ${detectionInfo.isRateLimited}`);
-        console.log(`[debug][flowproxy]   Has JS Challenge: ${detectionInfo.hasJSChallenge}`);
-        console.log(`[debug][flowproxy]   Is Processing: ${detectionInfo.isProcessing}`);
-        console.log(`[debug][flowproxy]   Body Snippet: "${detectionInfo.bodySnippet}"`);
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} FlowProxy protection detected on ${currentUrl}:`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Page Title: "${detectionInfo.title}"`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Current URL: ${detectionInfo.url}`));
+        // Specific-signal breakdown — which vendor-specific marker(s) fired
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Specific signals: domain=${detectionInfo.hasFlowProxyDomain} elements=${detectionInfo.hasFlowProxyElements} scripts=${detectionInfo.hasFlowProxyScripts} brandText=${detectionInfo.hasFlowProxyBrandText} headers=${detectionInfo.hasFlowProxyHeaders} cookies=${detectionInfo.hasFlowProxyCookies}`));
+        if (detectionInfo.matchedHeader) {
+          console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Matched header: ${detectionInfo.matchedHeader}`));
+        }
+        if (detectionInfo.matchedCookie) {
+          console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Matched cookie: ${detectionInfo.matchedCookie}`));
+        }
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Has Protection Page: ${detectionInfo.hasProtectionPage}`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Has Challenge Elements: ${detectionInfo.hasChallengeElements}`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Is Rate Limited: ${detectionInfo.isRateLimited}`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Has JS Challenge: ${detectionInfo.hasJSChallenge}`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Is Processing: ${detectionInfo.isProcessing}`));
+        console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Body Snippet: "${detectionInfo.bodySnippet}"`));
       }
 
       // HANDLE RATE LIMITING - Highest priority as it blocks all requests
@@ -371,7 +598,7 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
       if (detectionInfo.isRateLimited) {
         const rateLimitDelay = siteConfig.flowproxy_delay || TIMEOUTS.RATE_LIMIT_DEFAULT;
         result.warnings.push(`Rate limiting detected - implementing ${rateLimitDelay}ms delay`);
-        if (forceDebug) console.log(`[debug][flowproxy] Rate limiting detected, waiting ${rateLimitDelay}ms`);
+        if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Rate limiting detected, waiting ${rateLimitDelay}ms`));
         await waitForTimeout(page, rateLimitDelay);
       }
 
@@ -379,40 +606,39 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
       // FlowProxy uses JS challenges to verify browser legitimacy
       if (detectionInfo.hasJSChallenge || detectionInfo.isProcessing) {
         const jsWaitTime = siteConfig.flowproxy_js_timeout || TIMEOUTS.JS_CHALLENGE_DEFAULT;
-        if (forceDebug) console.log(`[debug][flowproxy] JavaScript challenge detected, waiting up to ${jsWaitTime}ms for completion`);
+        if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} JavaScript challenge detected, waiting up to ${jsWaitTime}ms for completion`));
         
         try {
-          // Wait for challenge completion indicators to disappear
-          // These conditions indicate the JS challenge has finished
-          await Promise.race([
-            page.waitForFunction(
+          // Wait for challenge completion indicators to disappear.
+          // page.waitForFunction has its own { timeout } — the previous
+          // outer Promise.race added a setTimeout that fired 5s LATER,
+          // leaked its timer on the success path, and never won the race
+          // in practice. Dropped: waitForFunction's own timeout is the
+          // single source of truth.
+          await page.waitForFunction(
             () => {
               const bodyText = document.body ? document.body.textContent : '';
-              return !bodyText.includes('Processing') && 
+              return !bodyText.includes('Processing') &&
                      !bodyText.includes('Checking your browser') &&
                      !bodyText.includes('Please wait') &&
                      !document.querySelector('.loading-spinner') &&
                      !document.querySelector('.processing-indicator');
             },
             { timeout: jsWaitTime }
-            ),
-            new Promise((_, reject) => 
-              setTimeout(() => reject(new Error('JS challenge timeout')), jsWaitTime + 5000)
-            )
-          ]);
-          
-          if (forceDebug) console.log(`[debug][flowproxy] JavaScript challenge appears to have completed`);
+          );
+
+          if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} JavaScript challenge appears to have completed`));
         } catch (timeoutErr) {
           // Continue even if timeout occurs - some challenges may take longer
           result.warnings.push(`JavaScript challenge timeout after ${jsWaitTime}ms`);
-          if (forceDebug) console.log(`[debug][flowproxy] JavaScript challenge timeout - continuing anyway`);
+          if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} JavaScript challenge timeout - continuing anyway`));
         }
       }
 
       // IMPLEMENT ADDITIONAL DELAY - Final step to ensure all processing completes
       // FlowProxy may need extra time even after challenges complete
       const additionalDelay = siteConfig.flowproxy_additional_delay || FAST_TIMEOUTS.ADDITIONAL_DELAY_DEFAULT;
-      if (forceDebug) console.log(`[debug][flowproxy] Implementing additional ${additionalDelay}ms delay for flowProxy processing`);
+      if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Implementing additional ${additionalDelay}ms delay for flowProxy processing`));
       await waitForTimeout(page, additionalDelay);
 
       // VERIFY SUCCESSFUL BYPASS - Check if we're still on a protection page
@@ -420,17 +646,17 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
       const finalCheck = await analyzeFlowProxyProtection(page);
       if (finalCheck.isFlowProxyDetected && finalCheck.hasProtectionPage) {
         result.warnings.push('Still on flowProxy protection page after handling attempts');
-        if (forceDebug) console.log(`[debug][flowproxy] Warning: Still appears to be on protection page`);
+        if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Warning: Still appears to be on protection page`));
         // Don't mark as failure - protection page may persist but still allow access
       } else {
         result.handlingResult.success = true;
-        if (forceDebug) console.log(`[debug][flowproxy] Successfully handled flowProxy protection for ${currentUrl}`);
+        if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} Successfully handled flowProxy protection for ${currentUrl}`));
       }
       
     } else {
-      // No FlowProxy protection detected - mark as successful (nothing to handle)
-      if (forceDebug) console.log(`[debug][flowproxy] No flowProxy protection detected on ${currentUrl}`);
-      result.overallSuccess = true;
+      // No FlowProxy protection detected — nothing to handle.
+      // result.overallSuccess is already true from initialization.
+      if (forceDebug) console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} No flowProxy protection detected on ${currentUrl}`));
     }
     
   } catch (error) {
@@ -438,86 +664,63 @@ async function handleFlowProxyProtection(page, currentUrl, siteConfig, forceDebu
     result.errors.push(`FlowProxy handling error: ${error.message}`);
     result.overallSuccess = false;
     if (forceDebug) {
-      console.log(`[debug][flowproxy] FlowProxy handling failed for ${currentUrl}:`);
-      console.log(`[debug][flowproxy]   Error: ${error.message}`);
-      console.log(`[debug][flowproxy]   Stack: ${error.stack}`);
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} FlowProxy handling failed for ${currentUrl}:`));
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Error: ${error.message}`));
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   Stack: ${error.stack}`));
     }
   }
 
   // LOG COMPREHENSIVE RESULTS for debugging and monitoring
   if (result.errors.length > 0 && forceDebug) {
-    console.log(`[debug][flowproxy] FlowProxy handling completed with errors for ${currentUrl}:`);
+    console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} FlowProxy handling completed with errors for ${currentUrl}:`));
     result.errors.forEach(error => {
-      console.log(`[debug][flowproxy]   - ${error}`);
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   - ${error}`));
     });
   } else if (result.warnings.length > 0 && forceDebug) {
-    console.log(`[debug][flowproxy] FlowProxy handling completed with warnings for ${currentUrl}:`);
+    console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} FlowProxy handling completed with warnings for ${currentUrl}:`));
     result.warnings.forEach(warning => {
-      console.log(`[debug][flowproxy]   - ${warning}`);
+      console.log(formatLogMessage('debug', `${FLOWPROXY_TAG}   - ${warning}`));
     });
   } else if (result.flowProxyDetection.attempted && forceDebug) {
-    console.log(`[debug][flowproxy] FlowProxy handling completed successfully for ${currentUrl}`);
+    console.log(formatLogMessage('debug', `${FLOWPROXY_TAG} FlowProxy handling completed successfully for ${currentUrl}`));
   }
 
   return result;
 }
 
 /**
- * Quick check to determine if the current page might be behind flowProxy protection
- * This is a lightweight alternative to full analysis for simple detection needs
- * 
- * @param {import('puppeteer').Page} page - Puppeteer page instance
- * @returns {Promise<boolean>} True if flowProxy protection is suspected
- * 
+ * Gets page-level timeout values for flowProxy-protected sites. Used by
+ * nwss.js to call page.setDefaultTimeout/setDefaultNavigationTimeout
+ * before navigating. The handler itself reads challenge/rate-limit/
+ * additional-delay values directly from siteConfig (with TIMEOUTS
+ * fallbacks), so those don't need to round-trip through this function.
+ *
+ * @param {object} siteConfig - Site configuration object
+ * @returns {{ pageTimeout: number, navigationTimeout: number }}
+ *
  * @example
- * if (await isFlowProxyProtected(page)) {
- *   console.log('FlowProxy protection detected - implementing handling');
- *   await handleFlowProxyProtection(page, url, config);
- * }
- */
-async function isFlowProxyProtected(page) {
-  try {
-    const detection = await analyzeFlowProxyProtection(page);
-    return detection.isFlowProxyDetected;
-  } catch (error) {
-    // Return false if detection fails - assume no protection
-    return false;
-  }
-}
-
-/**
- * Gets recommended timeout values for flowProxy protected sites
- * Provides sensible defaults while allowing site-specific customization
- * 
- * @param {object} siteConfig - Site configuration object with optional FlowProxy settings
- * @returns {object} Recommended timeout values for FlowProxy handling
- * 
- * @example
- * const timeouts = getFlowProxyTimeouts({
- *   flowproxy_delay: 60000,        // Custom rate limit delay
- *   flowproxy_js_timeout: 25000    // Custom JS challenge timeout
- * });
- * 
- * // Use timeouts in page operations
- * await page.goto(url, { timeout: timeouts.pageTimeout });
+ * const { pageTimeout, navigationTimeout } = getFlowProxyTimeouts(siteConfig);
+ * page.setDefaultTimeout(pageTimeout);
+ * page.setDefaultNavigationTimeout(navigationTimeout);
  */
 function getFlowProxyTimeouts(siteConfig) {
   return {
     pageTimeout: siteConfig.flowproxy_page_timeout || TIMEOUTS.PAGE_TIMEOUT_DEFAULT,
-    navigationTimeout: siteConfig.flowproxy_nav_timeout || TIMEOUTS.NAVIGATION_TIMEOUT_DEFAULT,
-    challengeTimeout: siteConfig.flowproxy_js_timeout || TIMEOUTS.JS_CHALLENGE_DEFAULT,
-    rateLimit: siteConfig.flowproxy_delay || TIMEOUTS.RATE_LIMIT_DEFAULT,
-    additionalDelay: siteConfig.flowproxy_additional_delay || TIMEOUTS.ADDITIONAL_DELAY_DEFAULT
+    navigationTimeout: siteConfig.flowproxy_nav_timeout || TIMEOUTS.NAVIGATION_TIMEOUT_DEFAULT
   };
 }
 
-// Export all public functions for use in other modules
+// Public surface used by nwss.js. Internal helpers (waitForTimeout,
+// safePageEvaluate, analyzeFlowProxyProtection, shouldProcessUrl) stay
+// module-private — the old export list included several functions no
+// caller imported.
+//
+// attachFlowProxyHeaderListener should be called by the caller BEFORE
+// navigation so the response listener observes the document response's
+// own headers. Without it, header/cookie detection silently no-ops and
+// the module falls back to DOM-only detection.
 module.exports = {
-  analyzeFlowProxyProtection,
   handleFlowProxyProtection,
-  isFlowProxyProtected,
   getFlowProxyTimeouts,
-  waitForTimeout,
-  getModuleInfo,
-  FLOWPROXY_MODULE_VERSION
+  attachFlowProxyHeaderListener
 };