@fanboynz/network-scanner 3.1.2 → 3.3.0

package/lib/domain-cache.js

@@ -103,54 +103,6 @@ class DomainCache {
     return wasNew;
   }
 
-  /**
-   * Combined check-and-mark in one pass. Functionally equivalent to
-   * isDomainAlreadyDetected() followed by markDomainAsDetected(), but with
-   * one Set.has() call instead of two. (JS is single-threaded so all three
-   * variants are individually atomic; this one is just cheaper.)
-   * @param {string} domain - Domain to check and potentially mark
-   * @returns {boolean} True if domain was ALREADY detected (should skip), false if NEW (should process)
-   */
-  checkAndMark(domain) {
-    if (!domain || typeof domain !== 'string') {
-      return false;
-    }
-
-    const wasAlreadyDetected = this.cache.has(domain);
-    
-    if (wasAlreadyDetected) {
-      // Domain already exists - update skip stats and return true (should skip)
-      this.stats.totalSkipped++;
-      this.stats.cacheHits++;
-      
-      if (this.enableLogging) {
-        console.log(formatLogMessage('debug', `${this.logPrefix} Cache HIT: ${domain} (skipped)`));
-      }
-      return true; // Already detected, should skip
-    }
-    
-    // Domain is NEW - mark it as detected
-    this.stats.cacheMisses++;
-    
-    this.cache.add(domain);
-    this.stats.totalDetected++;
-    
-    if (this.enableLogging) {
-      console.log(formatLogMessage('debug', `${this.logPrefix} Cache MISS: ${domain} (processing and marked, cache size: ${this.cache.size})`));
-    }
-    
-    // Check size after the add so an overflow only fires eviction once per
-    // overflowing call (using targetCacheSize precomputed in the constructor).
-    if (this.cache.size > this.maxCacheSize) {
-      const toRemove = this.cache.size - this.targetCacheSize;
-      if (toRemove > 0) {
-        this.clearOldestEntries(toRemove);
-      }
-    }
-
-    return false; // New domain, should process
-  }
-
   /**
    * Clear oldest entries from cache (FIFO eviction). Set iteration order is
    * guaranteed insertion order per ES2015, so this genuinely evicts oldest-
@@ -208,45 +160,6 @@ class DomainCache {
     return this.cache.has(domain);
   }
 
-  /**
-   * Add multiple domains to cache at once. Uses a single .size delta to
-   * count actually-new entries (skipping per-domain .has() calls), and
-   * runs the size-overflow eviction check once after the batch instead of
-   * per-domain. For a batch of N domains this is N .has() calls saved and
-   * up to N redundant cap checks collapsed to one.
-   * @param {Array<string>} domains - Array of domains to add
-   * @returns {number} Number of domains actually added (excludes duplicates)
-   */
-  markMultipleDomainsAsDetected(domains) {
-    if (!Array.isArray(domains) || domains.length === 0) {
-      return 0;
-    }
-
-    const startSize = this.cache.size;
-    for (let i = 0; i < domains.length; i++) {
-      const d = domains[i];
-      if (d && typeof d === 'string') {
-        this.cache.add(d);
-      }
-    }
-    const addedCount = this.cache.size - startSize;
-    this.stats.totalDetected += addedCount;
-
-    if (this.enableLogging && addedCount > 0) {
-      console.log(formatLogMessage('debug', `${this.logPrefix} Batch added ${addedCount} new domains (cache size: ${this.cache.size})`));
-    }
-
-    // One eviction sweep at the end, mirroring the single-add overflow check.
-    if (this.cache.size > this.maxCacheSize) {
-      const toRemove = this.cache.size - this.targetCacheSize;
-      if (toRemove > 0) {
-        this.clearOldestEntries(toRemove);
-      }
-    }
-
-    return addedCount;
-  }
-
   /**
    * Create bound helper functions for easy integration with existing code
    * @returns {object} Object with bound helper functions
@@ -255,7 +168,6 @@ class DomainCache {
     return {
       isDomainAlreadyDetected: this.isDomainAlreadyDetected.bind(this),
       markDomainAsDetected: this.markDomainAsDetected.bind(this),
-      checkAndMark: this.checkAndMark.bind(this),
       getSkippedCount: () => this.stats.totalSkipped,
       getCacheSize: () => this.cache.size,
       getStats: this.getStats.bind(this)
@@ -273,8 +185,7 @@ let globalDomainCache = null;
  *
  * NOTE: `options` is honored ONLY on the first call (the call that actually
  * constructs the singleton). Subsequent calls return the existing instance
- * regardless of what's passed. If you need different settings, call
- * resetGlobalCache() first or use `new DomainCache(options)` directly.
+ * regardless of what's passed; options are fixed at first construction.
  *
  * Under debug logging, a warning fires if a later caller passes options
  * that don't match the live instance — silent drift is a recurring source
@@ -295,7 +206,7 @@ function getGlobalDomainCache(options = {}) {
       (options.enableLogging !== undefined && options.enableLogging !== globalDomainCache.enableLogging) ||
       (options.logPrefix !== undefined && options.logPrefix !== globalDomainCache.logPrefix);
     if (drifted) {
-      console.log(formatLogMessage('debug', `${globalDomainCache.logPrefix} getGlobalDomainCache called with options that differ from the live singleton; ignored (call resetGlobalCache() first to apply new options)`));
+      console.log(formatLogMessage('debug', `${globalDomainCache.logPrefix} getGlobalDomainCache called with options that differ from the live singleton; ignored (options are fixed at first construction)`));
     }
   }
   return globalDomainCache;
@@ -312,36 +223,17 @@ function createGlobalHelpers(options = {}) {
 }
 
 /**
- * Reset the global cache (useful for testing or manual resets)
- */
-function resetGlobalCache() {
-  if (globalDomainCache) {
-    globalDomainCache.clear();
-  }
-  globalDomainCache = null;
-}
-
-/**
- * Legacy wrapper functions for backward compatibility
- * These match the original function signatures from nwss.js
+ * Legacy wrapper for backward compatibility.
  *
- * NOTE: getTotalDomainsSkipped and getDetectedDomainsCount are the only
- * ones kept — they're used directly by nwss.js for end-of-scan stats.
- * Previously-defined isDomainAlreadyDetected / markDomainAsDetected /
- * checkAndMark wrappers were removed: nwss.js calls those via
- * createGlobalHelpers() now and repo-wide grep confirmed zero remaining
- * external callers of the legacy wrappers.
+ * getDetectedDomainsCount is the only one kept — nwss.js reads it for the
+ * end-of-scan "unique domains cached" stat. getTotalDomainsSkipped was
+ * removed: its value was always 0 because the global cache's skip-check
+ * (isDomainAlreadyDetected) is never called — cross-URL dedup is handled by
+ * nettools' processed-domain sets / smart-cache / the per-URL set — so the
+ * stat was misleading. The isDomainAlreadyDetected / markDomainAsDetected /
+ * checkAndMark wrappers were likewise removed; nwss.js uses createGlobalHelpers().
  */
 
-/**
- * Get total domains skipped (legacy wrapper)
- * @returns {number} Number of domains skipped
- */
-function getTotalDomainsSkipped() {
-  const cache = getGlobalDomainCache();
-  return cache.stats.totalSkipped;
-}
-
 /**
  * Get detected domains cache size (legacy wrapper)
  * @returns {number} Size of the detected domains cache
@@ -352,15 +244,10 @@ function getDetectedDomainsCount() {
 }
 
 module.exports = {
-  // Main class
-  DomainCache,
-  
-  // Global cache functions
-  getGlobalDomainCache,
+  // Global cache helpers — createGlobalHelpers feeds nwss.js's per-domain
+  // marking; getDetectedDomainsCount feeds the end-of-scan "unique domains
+  // cached" stat. (DomainCache / getGlobalDomainCache stay internal — no
+  // external consumer; construct via createGlobalHelpers.)
   createGlobalHelpers,
-  resetGlobalCache,
-  
-  // Legacy wrappers still used by nwss.js for end-of-scan stats
-  getTotalDomainsSkipped,
   getDetectedDomainsCount
 };

package/lib/ghost-cursor.js

@@ -15,6 +15,11 @@
 //   npm install ghost-cursor            (optional dependency)
 
 const { formatLogMessage, messageColors } = require('./colorize');
+// humanClick gives the coordinate-click path the same press realism as the
+// built-in content clicks (hover dwell + mousedown/hold/mouseup, optional
+// hand-tremor + mouseup drift) instead of a 0ms page.mouse.click. One-way
+// require — interaction.js does not depend on ghost-cursor, so no cycle.
+const { humanClick } = require('./interaction');
 const GHOST_CURSOR_TAG = messageColors.processing('[ghost-cursor]');
 
 let ghostCursorModule = null;
@@ -56,7 +61,7 @@ function createGhostCursor(page, options = {}) {
     const cursor = ghostCursorModule.createCursor(page, { x: startX, y: startY });
 
     if (forceDebug) {
-      console.log(formatLogMessage('debug', '[ghost-cursor] Cursor instance created'));
+      console.log(formatLogMessage('debug', `${GHOST_CURSOR_TAG} Cursor instance created`));
     }
 
     return cursor;
@@ -98,7 +103,7 @@ async function ghostMove(cursor, toX, toY, options = {}) {
     const moveOpts = {};
     if (moveSpeed !== undefined) moveOpts.moveSpeed = moveSpeed;
     if (moveDelay > 0) moveOpts.moveDelay = moveDelay;
-    if (randomizeMoveDelay !== undefined) moveOpts.randomizeMoveDelay = randomizeMoveDelay;
+    moveOpts.randomizeMoveDelay = randomizeMoveDelay; // always defined (defaults to true)
     if (overshootThreshold !== undefined) moveOpts.overshootThreshold = overshootThreshold;
 
     await cursor.moveTo({ x: toX, y: toY }, moveOpts);
@@ -126,6 +131,8 @@ async function ghostMove(cursor, toX, toY, options = {}) {
  * @param {number} options.waitForClick - Delay (ms) between mousedown/mouseup (default: auto)
  * @param {number} options.moveDelay - Delay (ms) after moving to target
  * @param {number} options.paddingPercentage - Click point within element (0=edge, 100=center)
+ * @param {import('puppeteer').Page} options.page - Page for coordinate clicks (falls back to cursor.page)
+ * @param {boolean} options.realistic - Coordinate clicks: emit hand-tremor + mouseup drift (default: false)
  * @param {boolean} options.forceDebug - Enable debug logging
  * @returns {Promise<boolean>} true if click succeeded
  */
@@ -137,6 +144,8 @@ async function ghostClick(cursor, target, options = {}) {
     waitForClick,
     moveDelay,
     paddingPercentage,
+    page,
+    realistic = false,
     forceDebug
   } = options;
 
@@ -149,16 +158,25 @@ async function ghostClick(cursor, target, options = {}) {
     if (typeof target === 'string') {
       await cursor.click(target, clickOpts);
     } else {
-      // For coordinate clicks, move first then use page click
+      // Coordinate click: ghost-cursor's bezier moveTo brings the cursor to the
+      // point, then humanClick does the realistic press (hover dwell, mousedown
+      // → hold → mouseup, plus hand-tremor + down≠up drift when realistic). This
+      // replaces a 0ms page.mouse.click, so the ghost path gets the same click
+      // realism as built-in content clicks.
       await cursor.moveTo(target);
-      // Small hesitation before clicking
-      if (hesitate > 0) {
-        await new Promise(resolve => setTimeout(resolve, hesitate));
-      }
-      const page = cursor._page || cursor.page;
-      if (page && typeof page.mouse?.click === 'function') {
-        await page.mouse.click(target.x, target.y);
+      // Prefer the caller-supplied page; fall back to the cursor's own page
+      // (ghost-cursor exposes it as cursor.page) so we don't depend on internals.
+      // Return false (not silent success) if there's no usable page — otherwise
+      // the "Clicked" log + return true below would lie about a click that
+      // never fired.
+      const clickPage = page || cursor.page;
+      if (!clickPage || typeof clickPage.mouse?.down !== 'function') {
+        if (forceDebug) {
+          console.log(formatLogMessage('debug', `${GHOST_CURSOR_TAG} Coordinate click skipped: no usable page`));
+        }
+        return false;
       }
+      await humanClick(clickPage, target.x, target.y, { realistic, forceDebug });
     }
 
     if (forceDebug) {
@@ -189,7 +207,7 @@ async function ghostRandomMove(cursor, options = {}) {
   try {
     await cursor.randomMove();
     if (options.forceDebug) {
-      console.log(formatLogMessage('debug', '[ghost-cursor] Random movement performed'));
+      console.log(formatLogMessage('debug', `${GHOST_CURSOR_TAG} Random movement performed`));
     }
     return true;
   } catch (err) {

package/lib/interaction.js

@@ -1333,5 +1333,9 @@ module.exports = {
   simulateScrolling,
   interactWithElements,
   performContentClicks,
+  // Realistic timed click (hover dwell + mousedown/hold/mouseup, optional
+  // hand-tremor + mouseup drift). Reused by lib/ghost-cursor.js so the ghost
+  // coordinate click gets the same press realism as built-in content clicks.
+  humanClick,
   generateRandomCoordinates
 };

package/lib/nettools.js

@@ -30,7 +30,7 @@ const GLOBAL_DIG_CACHE_MAX = 2000;
 // Global whois result cache — shared across ALL handler instances and processUrl calls
 // Whois data is per root domain and doesn't change based on search terms
 const globalWhoisResultCache = new Map();
-const GLOBAL_WHOIS_CACHE_TTL = 72000000; // 20 hours (persisted to disk between runs)
+const GLOBAL_WHOIS_CACHE_TTL = 129600000; // 36 hours (persisted to disk between runs). Longer than dig's 20h: registrar data is very stable and whois servers rate-limit aggressively, so caching longer cuts repeat queries.
 const GLOBAL_WHOIS_CACHE_MAX = 2000;
 
 // Persistent disk cache file paths
@@ -40,8 +40,8 @@ const WHOIS_CACHE_FILE = path.join(__dirname, '..', '.whoiscache');
 // Index of hostnames known to resolve, populated as a side effect of
 // positive dig/whois cache writes AND cache hits. nwss.js's DNS pre-check
 // reads this via domainKnownToResolve() so it can skip its own resolve4
-// call on hosts that dig or whois have already proven live within the
-// 20-hour TTL window. Populating on cache HITS (not just writes) handles
+// call on hosts that dig or whois have already proven live within their
+// cache TTL window (dig 20h / whois 36h). Populating on cache HITS (not just writes) handles
 // the --dns-cache disk-load case where entries arrive without going
 // through the in-process write path. Stale entries -- hostname in Set but
 // the dig/whois entry has since been evicted -- are harmless: worst case
@@ -124,7 +124,6 @@ function loadDiskCache(filePath, cache, ttl, maxSize) {
     // Surface the event so the user knows they lost their warm cache;
     // previously this was a silent reset, which made "why did my dns
     // cache stop helping?" hard to diagnose.
-    // eslint-disable-next-line no-console
     console.warn(`${messageColors.highlight('[dns-cache]')} ${path.basename(filePath)} was unreadable (${err.message}); starting fresh`);
     try { fs.unlinkSync(filePath); } catch {}
   }
@@ -256,6 +255,38 @@ function getDnsCacheStats() {
 // Disk cache is opt-in via --dns-cache flag
 let diskCacheEnabled = false;
 
+// Optional dig resolver(s), set from --dns. When non-empty, dig queries
+// `@<one of these>` (round-robin) instead of the system resolver — so dig uses
+// the same reliable servers as the pre-check rather than a flaky /etc/resolv.conf
+// (the cause of `dig: Command timeout` drops on Cloudflare-fronted ad domains).
+let digResolvers = [];
+let digResolverCursor = 0;
+// dig's `@server` wants a bare IP; strip any `ipv4:port` / `[ipv6]:port` form.
+function digServerFromSpec(spec) {
+  const s = String(spec);
+  const br = s.match(/^\[([0-9a-fA-F:]+)\]/);
+  if (br) return br[1];
+  const v4p = s.match(/^(\d{1,3}(?:\.\d{1,3}){3}):\d+$/);
+  if (v4p) return v4p[1];
+  return s;
+}
+function setDigResolvers(servers) {
+  digResolvers = (Array.isArray(servers) ? servers : []).filter(Boolean).map(digServerFromSpec);
+}
+// Ordered `@server` attempt list for ONE dig lookup: starts at the round-robin
+// cursor (advanced once per lookup, preserving the old fairness) then falls
+// through the remaining resolvers as failover. Returns [null] when no --dns
+// resolvers are configured — a single attempt via the system resolver.
+function digServerAttemptList() {
+  if (digResolvers.length === 0) return [null];
+  const start = digResolverCursor++ % digResolvers.length;
+  const list = [];
+  for (let i = 0; i < digResolvers.length; i++) {
+    list.push('@' + digResolvers[(start + i) % digResolvers.length]);
+  }
+  return list;
+}
+
 /**
  * Enable persistent disk caching for dig/whois results.
  * Call this when --dns-cache flag is set. Idempotent — repeated calls
@@ -293,7 +324,6 @@ function enableDiskCache() {
   // Debug log only if anything was actually warmed; silent on fresh
   // installs / empty disk caches.
   if (digWarm > 0 || whoisWarm > 0) {
-    // eslint-disable-next-line no-console
     console.log(`${messageColors.highlight('[dns-cache]')} Warmed resolved-hostnames index from disk: ${digWarm} dig + ${whoisWarm} whois entries`);
   }
 
@@ -994,50 +1024,103 @@ async function whoisLookupWithRetry(domain = '', timeout = 10000, whoisServer =
  * @returns {Promise<Object>} Object with success status and output/error
  */
 async function digLookup(domain = '', recordType = 'A', timeout = 5000) {
-  try {
-    // Clean domain
-    const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
+  // Clean domain (defensive — callers usually pass an already-clean digDomain).
+  const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
+
+  // dig argv-injection guard. dig parses @/-/+ -leading tokens as options
+  // (`@host` redirects the query to an arbitrary server, `-f path` reads a
+  // file as a query batch) and has no `--` end-of-options marker like whois.
+  // Reject anything not hostname-shaped before shelling out — success:false so
+  // it's treated as no-match and not cached. (Charset blocks @ + / space etc;
+  // the leading-`-` check blocks `-f` and friends, since `-` is valid mid-host.)
+  if (!cleanDomain || /[^a-zA-Z0-9._-]/.test(cleanDomain) || cleanDomain.startsWith('-')) {
+    return { success: false, error: `invalid domain shape: ${cleanDomain}`, domain: cleanDomain, recordType };
+  }
+
+  // Resolver failover: try the round-robin resolver first, then fall through
+  // the remaining --dns resolvers on timeout / no-reply / REFUSED / SERVFAIL —
+  // the same resilience the whois path already has via whoisLookupWithRetry,
+  // and the DNS pre-check has via its rotation. Capped at 3 attempts (matches
+  // whois maxRetries default) so a host that's dead on every resolver can't
+  // burn the whole nettools budget.
+  const attempts = digServerAttemptList();
+  // Only do JS-level failover when --dns gave us pinned resolvers. Without it,
+  // attempts is [null]: a SINGLE system-resolver invocation that keeps dig's
+  // native resolv.conf rotation + retries (forcing +tries=1 there would strip
+  // that built-in resilience — the whole point is to be MORE resilient).
+  const usingResolvers = attempts[0] !== null;
+  const maxAttempts = usingResolvers ? Math.min(3, attempts.length) : 1;
+  // Pinned-resolver attempts use +time=2 +tries=1 (the JS loop owns failover)
+  // under a 4s SIGTERM ceiling. The system-resolver path keeps the full budget
+  // and dig's own retry behaviour, matching the pre-failover semantics exactly.
+  const perAttemptTimeout = usingResolvers ? Math.min(timeout, 4000) : timeout;
+
+  let lastError = 'no resolver attempts made';
+
+  for (let i = 0; i < maxAttempts; i++) {
+    const digServerArg = attempts[i];
+    // With a pinned resolver: one fast try (+time=2 +tries=1), then the JS loop
+    // moves to the next resolver. Without --dns: bare `dig name type` so dig
+    // applies its native resolv.conf rotation. execFile (no shell) => args
+    // can't be injected.
+    const digArgs = digServerArg
+      ? [digServerArg, '+time=2', '+tries=1', cleanDomain, recordType]
+      : [cleanDomain, recordType];
+    const resolverLabel = digServerArg ? digServerArg.slice(1) : 'system resolver';
+
+    try {
+      const { stdout: fullOutput } = await execFileWithTimeout('dig', digArgs, perAttemptTimeout);
+
+      // Judge success by RCODE, not by stderr. dig exits 0 for ANY server
+      // response, so non-zero exit (timeout / no-reply) already rejected above.
+      // REFUSED/SERVFAIL are resolver-SIDE failures another resolver may not
+      // share — fail over instead of accepting an answerless response (the
+      // EREFUSED-storm case). NOERROR/NXDOMAIN are definitive => accept.
+      const statusMatch = fullOutput.match(/status:\s*([A-Z]+)/i);
+      const rcode = statusMatch ? statusMatch[1].toUpperCase() : 'NOERROR';
+      if (rcode === 'REFUSED' || rcode === 'SERVFAIL') {
+        lastError = `dig ${rcode} from ${resolverLabel}`;
+        continue; // try next resolver in the failover list
+      }
+
+      // Non-empty stderr is intentionally NOT treated as failure here: dig
+      // prints `;; communications error ... timed out` warnings to stderr while
+      // still returning a valid ANSWER SECTION and exit 0. The old code failed
+      // the whole lookup on any stderr, discarding good answers — the exact
+      // missed-match pattern under flaky resolvers.
+      const answerMatch = fullOutput.match(/;; ANSWER SECTION:\n([\s\S]*?)(?:\n;;|\n*$)/);
+      let shortOutput = '';
+      if (answerMatch) {
+        shortOutput = answerMatch[1]
+          .split('\n')
+          .map(line => line.split(/\s+/).pop())
+          .filter(Boolean)
+          .join('\n');
+      }
 
-    // Single dig command — full output contains everything including short
-    // answers. execFile (no shell) so cleanDomain / recordType can contain
-    // any chars without injection risk.
-    const { stdout: fullOutput, stderr } = await execFileWithTimeout('dig', [cleanDomain, recordType], timeout);
-    
-    if (stderr && stderr.trim()) {
       return {
-        success: false,
-        error: stderr.trim(),
+        success: true,
+        output: fullOutput,
+        shortOutput,
         domain: cleanDomain,
-        recordType
+        recordType,
+        resolver: resolverLabel
       };
+    } catch (error) {
+      // Timeout or non-zero exit (e.g. dig exit 9 = no reply from this server).
+      // Record and fall through to the next resolver.
+      lastError = error.message;
     }
-    
-    // Extract short output from ANSWER SECTION of full dig output
-    const answerMatch = fullOutput.match(/;; ANSWER SECTION:\n([\s\S]*?)(?:\n;;|\n*$)/);
-    let shortOutput = '';
-    if (answerMatch) {
-      shortOutput = answerMatch[1]
-        .split('\n')
-        .map(line => line.split(/\s+/).pop())
-        .filter(Boolean)
-        .join('\n');
-    }
-    
-    return {
-      success: true,
-      output: fullOutput,
-      shortOutput,
-      domain: cleanDomain,
-      recordType
-    };
-  } catch (error) {
-    return {
-      success: false,
-      error: error.message,
-      domain: domain,
-      recordType
-    };
   }
+
+  // Every attempt timed out / was refused. success:false so the handler does
+  // NOT cache it (transient — caching would poison the domain for the TTL).
+  return {
+    success: false,
+    error: lastError,
+    domain: cleanDomain,
+    recordType
+  };
 }
 
 /**
@@ -1170,15 +1253,20 @@ function createNetToolsHandler(config) {
     // Determine which domain will be used for dig lookup
     const digDomain = digSubdomain && originalDomain ? originalDomain : domain;
 
-    // For whois: use root domain only (whois data is consistent for entire domain)
-    const whoisRootDomain = getRootDomain ? getRootDomain(`http://${domain}`) : domain;
-    
+    // For whois: use root domain only (whois data is consistent for entire
+    // domain). Only compute it when whois is actually configured — getRootDomain
+    // does a domain parse, so on a dig-only config (no whois/whois-or) this skips
+    // a parse + string build on every single request. whoisRootDomain is only
+    // ever read inside the whois branch, so the `domain` fallback is never used.
+    const wantWhois = hasWhois || hasWhoisOr;
+    const whoisRootDomain = wantWhois ? (getRootDomain ? getRootDomain(`http://${domain}`) : domain) : domain;
+
     // Check if we need to perform any lookups with appropriate deduplication
     // Whois: root domain + config (whois data same for sub.example.com and example.com)
-    const whoisDedupeKey = `${whoisRootDomain}:${whoisConfigKey}`;
+    const whoisDedupeKey = wantWhois ? `${whoisRootDomain}:${whoisConfigKey}` : '';
     // Dig: specific subdomain + config (DNS records can differ between subdomains)
     const digDedupeKey = `${digDomain}:${digConfigKey}`;
-    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(whoisDedupeKey);
+    const needsWhoisLookup = wantWhois && !processedWhoisDomains.has(whoisDedupeKey);
     const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDedupeKey);
 
     // Claim the dedupe keys NOW, synchronously, before executeNetToolsLookup
@@ -1606,11 +1694,20 @@ function createNetToolsHandler(config) {
               // backwards-compat additive: old code reading new cache
               // ignores it; new code reading old cache (no field) falls
               // back to lazy on-hit population in the cache-hit branch.
-              globalDigResultCache.set(digCacheKey, {
-                result: digResult,
-                timestamp: now,
-                hostname: digDomain
-              });
+              //
+              // Only cache a SUCCESSFUL dig. A timeout/error (success:false) is
+              // transient — caching it would poison the domain for the full
+              // cache TTL (20h when persisted via --dns-cache), so a host that
+              // resolves fine on the next attempt keeps getting dropped. (An
+              // NXDOMAIN is success:true with NXDOMAIN in the body — a real
+              // answer — so it's correctly still cached.)
+              if (digResult.success) {
+                globalDigResultCache.set(digCacheKey, {
+                  result: digResult,
+                  timestamp: now,
+                  hostname: digDomain
+                });
+              }
               dnsCacheStats.digMisses++;
               pushFreshSample(dnsCacheStats.freshDig, `${digDomain} (${digRecordType})`);
               // Index hostname IF dig actually proved resolution -- NXDOMAIN
@@ -1662,7 +1759,7 @@ function createNetToolsHandler(config) {
                 if (hasDig) logToConsoleAndFile(`${messageColors.highlight('[dig-and]')} Terms checked: ${digTerms.join(' AND ')}, matched: ${digMatched}`);
                 if (hasDigOr) logToConsoleAndFile(`${messageColors.highlight('[dig-or]')} Terms checked: ${digOrTerms.join(' OR ')}, matched: ${digOrMatched}`);
               }
-              logToConsoleAndFile(`${messageColors.highlight('[dig]')} Lookup completed for ${digDomain}, dig-and: ${digMatched}, dig-or: ${digOrMatched}`);
+              logToConsoleAndFile(`${messageColors.highlight('[dig]')} Lookup completed for ${digDomain}${digResult.resolver ? ` via ${digResult.resolver}` : ''}, dig-and: ${digMatched}, dig-or: ${digOrMatched}`);
               if (siteConfig.verbose === 1) {
                 if (hasDig) logToConsoleAndFile(`${messageColors.highlight('[dig]')} AND terms: ${digTerms.join(', ')}`);
                 if (hasDigOr) logToConsoleAndFile(`${messageColors.highlight('[dig]')} OR terms: ${digOrTerms.join(', ')}`);
@@ -1813,6 +1910,12 @@ module.exports = {
   validateDigAvailability,
   enableDiskCache,
   getDnsCacheStats,
+  // Route dig through the --dns resolver(s) instead of the system resolver.
+  setDigResolvers,
+  // Generic disk-cache primitives (atomic write, TTL/size-bounded) — reused by
+  // nwss.js to persist the DNS pre-check negative cache under --dns-cache.
+  loadDiskCache,
+  saveDiskCache,
   // Resolved-hostnames index for the DNS pre-check optimization.
   // nwss.js's per-task pre-check consults this BEFORE calling resolve4
   // so hosts already proven live by dig or whois (within their 20h

package/lib/openvpn_vpn.js

@@ -778,6 +778,14 @@ function validateOvpnConfig(ovpnConfig) {
  * @returns {Promise<Object>} { success, connection, tunDevice, error }
  */
 async function connectForSite(siteConfig, forceDebug = false) {
+  // Platform guard: OpenVPN routing here reads /proc and uses the iproute2 `ip`
+  // command, both Linux-only. Fail clearly instead of a cryptic /proc or `ip`
+  // error on macOS/Windows. WSL2 reports 'linux' and passes (TUN is checked
+  // separately below via isWSL/checkTunDevice).
+  if (process.platform !== 'linux') {
+    return { success: false, error: `OpenVPN routing is currently Linux-only (needs /proc + the iproute2 'ip' command; not available on ${process.platform}). Run on Linux/WSL2, or remove the 'openvpn' option from the site config.` };
+  }
+
   const ovpnConfig = normalizeOvpnConfig(siteConfig.openvpn);
   if (!ovpnConfig) {
     return { success: false, error: 'Invalid OpenVPN configuration' };

package/lib/output.js

@@ -133,32 +133,43 @@ function formatDomain(domain, options = {}) {
   if (!domain || domain.length <= 6 || !domain.includes('.')) {
     return null;
   }
-  
-  // If plain is true, always return just the domain regardless of other options
+
+  // Path-prefix rules (from output_regex) are stored as "host/path/" — they
+  // contain a '/'. Only adblock can express a path; every domain-only format
+  // (dnsmasq/unbound/pihole/hosts/privoxy/plain) falls back to the bare host
+  // (everything before the first '/') so output stays valid in all formats.
+  const slash = domain.indexOf('/');
+  const isPathRule = slash !== -1;
+  const host = isPathRule ? domain.slice(0, slash) : domain;
+
+  // If plain is true, always return just the host regardless of other options
   if (plain) {
-    return domain;
+    return host;
   }
-  
+
   // Apply specific format based on output mode
   if (pihole) {
     // Escape dots for regex and use Pi-hole format: (^|\.)domain\.com$
-    const escapedDomain = domain.replace(/\./g, '\\.');
+    const escapedDomain = host.replace(/\./g, '\\.');
     return `(^|\\.)${escapedDomain}$`;
   } else if (privoxy) {
-    return `{ +block } .${domain}`;
+    return `{ +block } .${host}`;
   } else if (dnsmasq) {
-    return `local=/${domain}/`;
+    return `local=/${host}/`;
   } else if (dnsmasqOld) {
-    return `server=/${domain}/`;
+    return `server=/${host}/`;
   } else if (unbound) {
-    return `local-zone: "${domain}." always_null`;
+    return `local-zone: "${host}." always_null`;
   } else if (localhostIP) {
-    return `${localhostIP} ${domain}`;
+    return `${localhostIP} ${host}`;
   } else if (adblockRules && resourceType) {
-    // Generate adblock filter rules with resource type modifiers
-    return `||${domain}^${resourceType}`;
+    // Adblock with resource-type modifier. A path rule self-anchors via its
+    // trailing '/', so it takes no '^' separator; a domain rule needs '^'.
+    return isPathRule ? `||${domain}${resourceType}` : `||${domain}^${resourceType}`;
   } else {
-    return `||${domain}^`;
+    // Default adblock: ||host^ for a domain, ||host/path/ for a path rule
+    // (the path already anchors, so no trailing '^').
+    return isPathRule ? `||${domain}` : `||${domain}^`;
   }
 }