@fanboynz/network-scanner 3.0.3 → 3.1.0

package/lib/interaction.js

@@ -67,6 +67,97 @@ function fastTimeout(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+/**
+ * Human-timed click — page.mouse.click() fires mousedown+mouseup ~10-30ms
+ * apart, which many ad-network popunder loaders (AdsCore/PropellerAds
+ * family) specifically filter as a bot signal: real users hold the
+ * button 50-150ms. This helper splits the click into explicit
+ * mousedown / hold / mouseup with realistic hold timing, plus optional
+ * hover-before-click pause and small click-offset jitter so clicks
+ * don't land pixel-perfect at the same (x,y) every time.
+ *
+ * Drop-in replacement for `page.mouse.click(x, y)` at popunder-trigger
+ * sites; bounded per-call cost is ~300-700ms (hover pause + hold + jitter)
+ * vs ~30ms for plain .click().
+ *
+ * @param {object} page - Puppeteer page
+ * @param {number} x - target X
+ * @param {number} y - target Y
+ * @param {object} options
+ * @param {number} options.offsetRange - ± px jitter from (x,y); default 5
+ * @param {number} options.hoverMin - min hover pause ms; default 150
+ * @param {number} options.hoverMax - max hover pause ms; default 450
+ * @param {number} options.holdMin - min mouse-down hold ms; default 50
+ * @param {number} options.holdMax - max mouse-down hold ms; default 150
+ * @param {boolean} options.realistic - emit hold-tremor + mouseup drift;
+ *   default false. Opt-in for sites that score mouse-click realism
+ *   (DataDome, Akamai BM, PerimeterX). Adds ~0ms latency (events fit
+ *   inside the existing hold) but generates 1–3 extra mousemove events
+ *   between mousedown and mouseup at ±1px tremor, plus a final ±1.5px
+ *   drift before mouseup so mousedown.x !== mouseup.x. Pure event-stream
+ *   change — no behavioral difference for the click itself.
+ */
+async function humanClick(page, x, y, options = {}) {
+  const {
+    offsetRange = 5,
+    hoverMin = 150, hoverMax = 450,
+    holdMin = 50,   holdMax = 150,
+    forceDebug = false,
+    realistic = false
+  } = options;
+  // ±offsetRange-px jitter so we don't click pixel-perfect (x,y) every
+  // time -- real users have spatial scatter even when aiming for the
+  // 'same' visible button.
+  const jx = x + (Math.random() - 0.5) * 2 * offsetRange;
+  const jy = y + (Math.random() - 0.5) * 2 * offsetRange;
+  try {
+    // Hover/move first -- many bot detectors check that mouse position
+    // matches the click point at mousedown time (browser fires mousemove
+    // before mousedown for real cursor hardware).
+    await page.mouse.move(jx, jy);
+    await fastTimeout(hoverMin + Math.random() * (hoverMax - hoverMin));
+    await page.mouse.down();
+
+    if (realistic) {
+      // Split the hold into (tremorCount + 1) chunks; emit a ±1px micromove
+      // between each chunk so the page sees mousemove events during the
+      // press window (real human hand tremor). Then drift ±MOUSEUP_DRIFT_PX
+      // before firing mouseup so mousedown.x/y !== mouseup.x/y.
+      const holdMs = holdMin + Math.random() * (holdMax - holdMin);
+      const tremorCount = CONTENT_CLICK.TREMOR_COUNT_MIN +
+        Math.floor(Math.random() * (CONTENT_CLICK.TREMOR_COUNT_MAX - CONTENT_CLICK.TREMOR_COUNT_MIN + 1));
+      const chunkMs = holdMs / (tremorCount + 1);
+      for (let i = 0; i < tremorCount; i++) {
+        await fastTimeout(chunkMs);
+        const tjx = jx + (Math.random() - 0.5) * 2 * CONTENT_CLICK.TREMOR_RANGE_PX;
+        const tjy = jy + (Math.random() - 0.5) * 2 * CONTENT_CLICK.TREMOR_RANGE_PX;
+        await page.mouse.move(tjx, tjy);
+      }
+      await fastTimeout(chunkMs);
+      // Final drift before mouseup. Move first (mouseup fires at current
+      // position) so the up event lands at slightly different coords than
+      // the down event — real humans almost always drift during the hold.
+      const ux = jx + (Math.random() - 0.5) * 2 * CONTENT_CLICK.MOUSEUP_DRIFT_PX;
+      const uy = jy + (Math.random() - 0.5) * 2 * CONTENT_CLICK.MOUSEUP_DRIFT_PX;
+      await page.mouse.move(ux, uy);
+      await page.mouse.up();
+    } else {
+      await fastTimeout(holdMin + Math.random() * (holdMax - holdMin));
+      await page.mouse.up();
+    }
+  } catch (err) {
+    // Page closed / target detached mid-click is the expected non-fatal
+    // path; everything else is unusual enough to surface in debug mode so
+    // a site silently failing 100% of clicks (CSP, broken input pipeline,
+    // CDP session collapse) is at least visible without --headful.
+    if (forceDebug && !/closed|detached|Target|Session closed|Protocol error/i.test(err.message || '')) {
+      try {
+        console.log(formatLogMessage('debug', `${INTERACTION_TAG} humanClick failed at (${jx.toFixed(0)}, ${jy.toFixed(0)}): ${err.message}`));
+      } catch (_) { /* logging itself shouldn't throw, but belt-and-braces */ }
+    }
+  }
+}
+
 // === VIEWPORT AND COORDINATE CONSTANTS ===
 // These control the default viewport assumptions and coordinate generation
 const DEFAULT_VIEWPORT = {
@@ -138,7 +229,8 @@ const ELEMENT_INTERACTION = {
 // NOTE: No preDelay needed — mouse movements + scrolling already provide ~1s
 // of activity before clicks fire, which is enough for async ad script registration
 const CONTENT_CLICK = {
-  CLICK_COUNT: 2,            // Two attempts (primary + backup if first suppressed)
+  CLICK_COUNT: 3,            // Three attempts (primary + 2 backup; ad SDKs sometimes suppress first OR second click as warmup before firing)
+  CLICK_COUNT_MAX: 20,       // Hard cap when overridden via siteConfig.interact_click_count — a typo of 500 shouldn't run for minutes
   INTER_CLICK_MIN: 300,      // Minimum ms between clicks (above Monetag 250ms cooldown)
   INTER_CLICK_MAX: 500,      // Maximum ms between clicks
   // PRE_CLICK_DELAY: most ad scripts register document-level listeners
@@ -147,7 +239,22 @@ const CONTENT_CLICK = {
   // 300ms buffer here was mostly defensive. Reduced to 100ms.
   PRE_CLICK_DELAY: 100,
   VIEWPORT_INSET: 0.2,       // Avoid outer 20% of viewport (menus, overlays)
-  MOUSE_APPROACH_STEPS: 3    // Minimal steps — just enough for non-instant movement
+  MOUSE_APPROACH_STEPS: 3,   // Minimal steps — just enough for non-instant movement
+  // Realistic-mode opt-in (siteConfig.realistic_click). Higher step count
+  // raises the mousemove event rate to ~80–125Hz (real mouse minimum is
+  // 125Hz USB default) so per-event movementX/Y deltas land in the 5–30px
+  // range a real cursor produces — fixes the strongest movement tell.
+  // Cost: +~80–120ms per click over the approach. Off by default.
+  MOUSE_APPROACH_STEPS_REALISTIC: 15,
+  // Realistic-mode hold tremor: 1–3 ±1px micromoves spread across the
+  // mousedown→mouseup hold to defeat the "zero events during hold" tell.
+  TREMOR_COUNT_MIN: 1,
+  TREMOR_COUNT_MAX: 3,
+  TREMOR_RANGE_PX: 1,
+  // Realistic-mode mouseup drift: real human clicks drift 0–2px between
+  // mousedown and mouseup, especially with longer holds. Without this,
+  // mousedown.x === mouseup.x is a robotic signal.
+  MOUSEUP_DRIFT_PX: 1.5
 };
 
 // === INTENSITY SETTINGS ===
@@ -358,16 +465,19 @@ async function humanLikeMouseMove(page, fromX, fromY, toX, toY, options = {}) {
     minDelay = MOUSE_MOVEMENT.MIN_DELAY,
     maxDelay = MOUSE_MOVEMENT.MAX_DELAY,
     curve = MOUSE_MOVEMENT.DEFAULT_CURVE,
-    jitter = MOUSE_MOVEMENT.DEFAULT_JITTER
+    jitter = MOUSE_MOVEMENT.DEFAULT_JITTER,
+    realistic = false  // bypass MAX_STEPS / MAX_TOTAL_MS caps for high-cadence approach
   } = options;
 
   const distance = Math.sqrt((toX - fromX) ** 2 + (toY - fromY) ** 2);
 
   // Step count: caller-provided value capped at MAX_STEPS, otherwise derived
-  // from distance and clamped to [MIN_STEPS, DEFAULT_STEPS].
+  // from distance and clamped to [MIN_STEPS, DEFAULT_STEPS]. Realistic mode
+  // skips the MAX_STEPS cap so callers can push 12–15 steps to match real
+  // mouse hardware event rates (~125Hz vs the default's ~30–60Hz).
   let actualSteps;
   if (options.steps) {
-    actualSteps = Math.min(options.steps, MOUSE_MOVEMENT.MAX_STEPS);
+    actualSteps = realistic ? options.steps : Math.min(options.steps, MOUSE_MOVEMENT.MAX_STEPS);
   } else {
     const calculatedSteps = Math.floor(distance / MOUSE_MOVEMENT.DISTANCE_STEP_RATIO);
     actualSteps = Math.max(
@@ -377,10 +487,16 @@ async function humanLikeMouseMove(page, fromX, fromY, toX, toY, options = {}) {
   }
 
   // Emergency cap on total movement time — if step count × max-per-step delay
-  // would exceed the budget, reduce step count to fit.
+  // would exceed the budget, reduce step count to fit. Realistic mode raises
+  // the cap to 600ms so the higher step count survives the trim.
+  // Floor-clamp to MIN_STEPS: if a caller passes a maxDelay larger than
+  // totalMsLimit (e.g. maxDelay: 1000), the floor division yields 0, and the
+  // i=0 iteration then computes progress = 0/0 = NaN, propagating into
+  // page.mouse.move(NaN, NaN). Clamping preserves at least MIN_STEPS moves.
+  const totalMsLimit = realistic ? 600 : MOUSE_MOVEMENT.MAX_TOTAL_MS;
   const estimatedTime = actualSteps * maxDelay;
-  if (estimatedTime > MOUSE_MOVEMENT.MAX_TOTAL_MS) {
-    actualSteps = Math.floor(MOUSE_MOVEMENT.MAX_TOTAL_MS / maxDelay);
+  if (estimatedTime > totalMsLimit) {
+    actualSteps = Math.max(MOUSE_MOVEMENT.MIN_STEPS, Math.floor(totalMsLimit / maxDelay));
   }
 
   for (let i = 0; i <= actualSteps; i++) {
@@ -398,8 +514,12 @@ async function humanLikeMouseMove(page, fromX, fromY, toX, toY, options = {}) {
     let currentX = fromX + (toX - fromX) * easedProgress;
     let currentY = fromY + (toY - fromY) * easedProgress;
 
-    // Add slight curve to movement (more human-like)
-    if (curve > 0 && i > 0 && i < actualSteps) {
+    // Add slight curve to movement (more human-like).
+    // distance > 0 guard: when fromX === toX AND fromY === toY (integer-quantized
+    // random targets in performContentClicks can collide; or external caller passes
+    // from === to deliberately) the perpX/perpY divisions become -0/0 = NaN and
+    // poison currentX/currentY, causing page.mouse.move(NaN, NaN) to reject via CDP.
+    if (curve > 0 && distance > 0 && i > 0 && i < actualSteps) {
       const curveIntensity = Math.sin((i / actualSteps) * Math.PI) * curve * distance * MOUSE_MOVEMENT.CURVE_INTENSITY_RATIO;
       const perpX = -(toY - fromY) / distance;
       const perpY = (toX - fromX) / distance;
@@ -547,7 +667,9 @@ async function interactWithElements(page, options = {}) {
     maxAttempts = ELEMENT_INTERACTION.MAX_ATTEMPTS,
     elementTypes = ['button', 'a', '[role="button"]'],
     avoidDestructive = true,
-    timeout = ELEMENT_INTERACTION.TIMEOUT
+    timeout = ELEMENT_INTERACTION.TIMEOUT,
+    forceDebug = false,
+    realistic = false
   } = options;
 
   try {
@@ -555,22 +677,23 @@ async function interactWithElements(page, options = {}) {
     try {
       // Check if page is closed before attempting interaction
       if (page.isClosed()) {
-        if (options.forceDebug) {
+        if (forceDebug) {
           console.log(formatLogMessage('debug', `${INTERACTION_TAG} Page is closed, skipping element interaction`));
         }
         return;
       }
 
-      // Very short timeout since page should already be loaded.
-      // Explicitly dispose the returned handle rather than relying on
-      // Puppeteer's FinalizationRegistry — matches the dispose pattern
-      // already used in performPageInteraction's final-hover block.
-      const bodyHandle = await page.waitForSelector('body', { timeout: 1000 });
+      // Body wait honors the caller-provided timeout option (default 2000ms
+      // via ELEMENT_INTERACTION.TIMEOUT) -- was previously hardcoded to 1000
+      // and silently ignored the option. Explicitly dispose the returned handle
+      // rather than relying on Puppeteer's FinalizationRegistry -- matches the
+      // dispose pattern already used in performPageInteraction's final-hover block.
+      const bodyHandle = await page.waitForSelector('body', { timeout });
       if (bodyHandle) { try { await bodyHandle.dispose(); } catch (_) {} }
       // Re-check after async wait — page may have closed during selector wait
       if (page.isClosed()) return;
     } catch (bodyWaitErr) {
-      if (options.forceDebug) {
+      if (forceDebug) {
         console.log(formatLogMessage('debug', `${INTERACTION_TAG} Page not ready for element interaction: ${bodyWaitErr.message}`));
       }
       return;
@@ -598,7 +721,12 @@ async function interactWithElements(page, options = {}) {
               
               if (isVisible) {
                 const text = (el.textContent || el.alt || el.title || '').toLowerCase();
-                const shouldAvoid = avoidWords && avoidWords.some(word => text.includes(word));
+                // Word-boundary regex match -- prior `text.includes(word)`
+                // produced false positives like 'submit' matching
+                // 'resubmit'/'submitter', filtering out legitimate
+                // clickables. \b ensures whole-word matches only.
+                const shouldAvoid = avoidWords && avoidWords.length > 0 &&
+                  new RegExp('\\b(' + avoidWords.join('|') + ')\\b').test(text);
                 
                 if (!shouldAvoid) {
                   clickableElements.push({
@@ -627,7 +755,7 @@ async function interactWithElements(page, options = {}) {
           // Brief pause before clicking
           await fastTimeout(TIMING.CLICK_PAUSE_MIN + Math.random() * (TIMING.CLICK_PAUSE_MAX - TIMING.CLICK_PAUSE_MIN));
           
-          await page.mouse.click(element.x, element.y);
+          await humanClick(page, element.x, element.y, { forceDebug, realistic });
           
           // Brief pause after clicking
           await fastTimeout(TIMING.POST_CLICK_MIN + Math.random() * (TIMING.POST_CLICK_MAX - TIMING.POST_CLICK_MIN));
@@ -679,8 +807,12 @@ async function performContentClicks(page, options = {}) {
     preDelay = CONTENT_CLICK.PRE_CLICK_DELAY,
     interClickMin = CONTENT_CLICK.INTER_CLICK_MIN,
     interClickMax = CONTENT_CLICK.INTER_CLICK_MAX,
-    forceDebug = false
+    forceDebug = false,
+    realistic = false   // siteConfig.realistic_click — denser approach + hold tremor + mouseup drift
   } = options;
+  const approachSteps = realistic
+    ? CONTENT_CLICK.MOUSE_APPROACH_STEPS_REALISTIC
+    : CONTENT_CLICK.MOUSE_APPROACH_STEPS;
 
   try {
     if (page.isClosed()) return;
@@ -707,14 +839,15 @@ async function performContentClicks(page, options = {}) {
 
       // Natural mouse approach (few steps, no need for elaborate curves)
       await humanLikeMouseMove(page, lastX, lastY, targetX, targetY, {
-        steps: CONTENT_CLICK.MOUSE_APPROACH_STEPS,
+        steps: approachSteps,
         curve: 0.03 + Math.random() * 0.04,
-        jitter: 1
+        jitter: 1,
+        realistic
       });
 
       // Brief human-like pause, then click
       await fastTimeout(TIMING.CLICK_PAUSE_MIN + Math.random() * (TIMING.CLICK_PAUSE_MAX - TIMING.CLICK_PAUSE_MIN));
-      await page.mouse.click(targetX, targetY);
+      await humanClick(page, targetX, targetY, { forceDebug, realistic });
 
       if (forceDebug) {
         console.log(formatLogMessage('debug', `${INTERACTION_TAG} Content click ${i + 1}/${clicks} at (${targetX}, ${targetY})`));
@@ -792,7 +925,82 @@ async function performContentClicks(page, options = {}) {
  *   includeElementClicks: false
  * });
  */
+/**
+ * Work-aware ceiling (ms) for a single interaction pass.
+ *
+ * Interaction is a sequence of awaited steps (mouse moves, scrolls, content
+ * clicks); under event-loop/CDP contention from many concurrent URLs each step
+ * stretches well past its intrinsic cost (a default 3-click pass measured ~4s
+ * solo but ~22s at the default concurrency of 6). A FLAT ceiling therefore
+ * either truncates legitimate high interact_click_count / realistic_click
+ * configs — dropping the very popunder clicks the pass exists to fire — or sits
+ * loosely over light runs. Scale by the actual work envelope instead, same
+ * philosophy as nwss's per-URL timeout. Per-unit allowances are sized to absorb
+ * up to ~default-concurrency contention; the result is a SAFETY ceiling, not a
+ * target — interaction returns as soon as its work is done, so a generous
+ * ceiling never slows a fast pass, it only bounds a stuck one.
+ *
+ * @param {Object} options - same shape performPageInteraction receives
+ * @returns {number} ceiling in ms (floored at 15000, the prior flat budget)
+ */
+function computeInteractionCeilingMs(options = {}) {
+  const {
+    intensity = 'medium',
+    mouseMovements,
+    includeScrolling = true,
+    includeElementClicks = false,
+    clickCount,
+    realistic = false
+  } = options;
+
+  const settings = INTENSITY_SETTINGS[String(intensity).toUpperCase()] || INTENSITY_SETTINGS.MEDIUM;
+  const movements = mouseMovements !== undefined ? mouseMovements : settings.movements;
+  const scrolls = includeScrolling ? settings.scrolls : 0;
+  const clicks = includeElementClicks
+    ? (clickCount ? Math.min(Math.floor(clickCount), CONTENT_CLICK.CLICK_COUNT_MAX) : CONTENT_CLICK.CLICK_COUNT)
+    : 0;
+
+  const BASE_MS = 6000;        // setup, viewport, final move, slack
+  const PER_MOVE_MS = 700;
+  const PER_SCROLL_MS = 800;
+  const PER_CLICK_MS = realistic ? 7000 : 4000;  // realistic clicks are denser (15-step approach + tremor)
+
+  return Math.max(
+    15000,  // floor = the prior flat budget, so light/default configs are unchanged
+    BASE_MS + movements * PER_MOVE_MS + scrolls * PER_SCROLL_MS + clicks * PER_CLICK_MS
+  );
+}
+
 async function performPageInteraction(page, currentUrl, options = {}, forceDebug = false) {
+  // Hard wall-clock ceiling on the whole interaction. The impl's internal
+  // checkTimeout() is cooperative — only evaluated BETWEEN steps — so a single
+  // blocking await (a CDP round-trip, or a fastTimeout that fires late once
+  // many URLs saturate the one event loop / CDP pipe) sails right past the 15s
+  // soft budget; that's how interaction was clocking 21-22s under concurrency.
+  // Racing the work against a real timer enforces the ceiling no matter where
+  // the time actually goes. The timer RESOLVES (never rejects) — interaction
+  // failures must not break the scan — and the impl is .catch()'d so the
+  // orphaned run can't surface an unhandled rejection after the race settles.
+  // Keeps nwss's per-URL INTERACTION_OVERHEAD_MS budget honest: one cycle now
+  // stays <= the ceiling even under heavy contention.
+  const HARD_CAP_MS = computeInteractionCeilingMs(options); // work-aware: scales with clicks/realistic/intensity
+  let capTimer;
+  let capped = false;
+  const work = _performPageInteractionImpl(page, currentUrl, options, forceDebug).catch(() => {});
+  try {
+    await Promise.race([
+      work,
+      new Promise(resolve => { capTimer = setTimeout(() => { capped = true; resolve(); }, HARD_CAP_MS); })
+    ]);
+  } finally {
+    if (capTimer) clearTimeout(capTimer);
+  }
+  if (capped && forceDebug) {
+    console.log(formatLogMessage('debug', `${INTERACTION_TAG} Interaction hard-capped at ${HARD_CAP_MS}ms for ${currentUrl} (event-loop/CDP contention)`));
+  }
+}
+
+async function _performPageInteractionImpl(page, currentUrl, options = {}, forceDebug = false) {
   // mouseMovements deliberately has no default in the destructure: we want
   // to distinguish 'caller didn't pass it' from 'caller explicitly passed 3'
   // so the actualMovements calculation below can let intensity drive the
@@ -803,7 +1011,9 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
     includeScrolling = true,
     includeElementClicks = false,
     duration = TIMING.DEFAULT_INTERACTION_DURATION,
-    intensity = 'medium'
+    intensity = 'medium',
+    clickCount,       // optional override; undefined -> performContentClicks uses CONTENT_CLICK.CLICK_COUNT default
+    realistic = false // siteConfig.realistic_click — propagated to performContentClicks
   } = options;
 
   try {
@@ -910,7 +1120,13 @@ async function performPageInteraction(page, currentUrl, options = {}, forceDebug
     // interactWithElements is still exported for callers that want it.
     if (includeElementClicks) {
       if (checkTimeout()) return; // Emergency timeout check
-      await performContentClicks(page, { forceDebug });
+      // Pass clickCount only when caller set it (via siteConfig.interact_click_count)
+      // -- omit otherwise so performContentClicks's default destructure
+      // falls through to CONTENT_CLICK.CLICK_COUNT. realistic is always
+      // forwarded (defaults to false at every layer).
+      const ccOpts = { forceDebug, realistic };
+      if (clickCount) ccOpts.clicks = clickCount;
+      await performContentClicks(page, ccOpts);
     }
 
     // Final resting position — single mouse.move instead of the previous
@@ -1037,6 +1253,25 @@ function createInteractionConfig(url, siteConfig = {}) {
     if (siteConfig.interact_clicks !== undefined) {
       config.includeElementClicks = siteConfig.interact_clicks;
     }
+    // interact_click_count: per-site override of how many random
+    // content-zone clicks performContentClicks fires. Cap at
+    // CLICK_COUNT_MAX to prevent runaway from typos. Coerce to integer
+    // and clamp >= 1 (count of 0 should be expressed via
+    // interact_clicks: false, not interact_click_count: 0).
+    if (typeof siteConfig.interact_click_count === 'number' && siteConfig.interact_click_count > 0) {
+      config.clickCount = Math.min(
+        Math.floor(siteConfig.interact_click_count),
+        CONTENT_CLICK.CLICK_COUNT_MAX
+      );
+    }
+    // realistic_click: opt-in for sites that score click realism
+    // (DataDome, Akamai BM, PerimeterX). Adds ~80–120ms per click for the
+    // denser approach; hold-tremor and mouseup-drift fit inside the
+    // existing hold window so they're free. Off by default since ad-network
+    // popunder discovery doesn't need it and we'd rather keep scans fast.
+    if (siteConfig.realistic_click === true) {
+      config.realistic = true;
+    }
 
     return config;
   } catch (urlErr) {
@@ -1091,6 +1326,7 @@ module.exports = {
   // Main interaction functions
   performPageInteraction,
   createInteractionConfig,
+  computeInteractionCeilingMs,
   getViewport,
   // Component functions for custom implementations
   humanLikeMouseMove,

package/lib/nettools.js

@@ -561,8 +561,19 @@ async function whoisLookup(domain = '', timeout = 10000, whoisServer = '', debug
 
     const { stdout, stderr } = await execFileWithTimeout('whois', whoisArgs, timeout);
     const duration = Date.now() - startTime;
-    
-    if (stderr && stderr.trim()) {
+
+    // Treat stderr as failure ONLY when there's no usable stdout. Many whois
+    // servers/clients emit non-fatal notices to stderr — referral/redirect
+    // lines, rate-limit and GDPR "data redacted" banners, registry
+    // disclaimers — while returning the real record on stdout. The old "any
+    // stderr -> fail" path discarded those valid records (and triggered
+    // needless fallback retries in whoisLookupWithRetry). When stdout has
+    // content we prefer it and treat the lookup as successful; the downstream
+    // term match is the real gate, so this can't manufacture a match. stderr
+    // is still surfaced in debug logging below for visibility.
+    const hasUsableStdout = !!(stdout && stdout.trim());
+
+    if (!hasUsableStdout && stderr && stderr.trim()) {
       if (debugMode) {
         if (logFunc) {
           logFunc(`${messageColors.highlight('[whois]')} Lookup failed for ${cleanDomain} after ${duration}ms`);
@@ -604,6 +615,14 @@ async function whoisLookup(domain = '', timeout = 10000, whoisServer = '', debug
         console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`));
         console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Output length: ${stdout.length} characters`));
       }
+      // Non-fatal stderr alongside usable stdout — kept visible so a real
+      // problem (truncation, partial referral) is still diagnosable even
+      // though we're treating the lookup as a success.
+      if (stderr && stderr.trim()) {
+        const note = `${messageColors.highlight('[whois]')} Non-fatal stderr (stdout used): ${stderr.trim()}`;
+        if (logFunc) logFunc(note);
+        else console.log(formatLogMessage('debug', note));
+      }
     }
     
     return {
@@ -1021,66 +1040,6 @@ async function digLookup(domain = '', recordType = 'A', timeout = 5000) {
   }
 }
 
-/**
- * Checks if whois output contains all specified search terms (AND logic)
- * @param {string} whoisOutput - The whois lookup output
- * @param {Array<string>} searchTerms - Array of terms that must all be present
- * @returns {boolean} True if all terms are found
- */
-function checkWhoisTerms(whoisOutput, searchTerms) {
-  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
-    return false;
-  }
-  
-  const lowerOutput = whoisOutput.toLowerCase();
-  return searchTerms.every(term => lowerOutput.includes(term.toLowerCase()));
-}
-
-/**
- * Checks if whois output contains any of the specified search terms (OR logic)
- * @param {string} whoisOutput - The whois lookup output
- * @param {Array<string>} searchTerms - Array of terms where at least one must be present
- * @returns {boolean} True if any term is found
- */
-function checkWhoisTermsOr(whoisOutput, searchTerms) {
-  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
-    return false;
-  }
-  
-  const lowerOutput = whoisOutput.toLowerCase();
-  return searchTerms.some(term => lowerOutput.includes(term.toLowerCase()));
-}
-
-/**
- * Checks if dig output contains all specified search terms (AND logic)
- * @param {string} digOutput - The dig lookup output
- * @param {Array<string>} searchTerms - Array of terms that must all be present
- * @returns {boolean} True if all terms are found
- */
-function checkDigTerms(digOutput, searchTerms) {
-  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
-    return false;
-  }
-  
-  const lowerOutput = digOutput.toLowerCase();
-  return searchTerms.every(term => lowerOutput.includes(term.toLowerCase()));
-}
-
-/**
- * Checks if dig output contains any of the specified search terms (OR logic)
- * @param {string} digOutput - The dig lookup output
- * @param {Array<string>} searchTerms - Array of terms where at least one must be present
- * @returns {boolean} True if any term is found
- */
-function checkDigTermsOr(digOutput, searchTerms) {
-  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
-    return false;
-  }
-  
-  const lowerOutput = digOutput.toLowerCase();
-  return searchTerms.some(term => lowerOutput.includes(term.toLowerCase()));
-}
-
 /**
  * Enhanced dry run callback factory for better nettools reporting
  * @param {Map} matchedDomains - The matched domains collection
@@ -1221,7 +1180,20 @@ function createNetToolsHandler(config) {
     const digDedupeKey = `${digDomain}:${digConfigKey}`;
     const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(whoisDedupeKey);
     const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDedupeKey);
-    
+
+    // Claim the dedupe keys NOW, synchronously, before executeNetToolsLookup
+    // hits its first await. These Sets are shared across all concurrent URL
+    // handlers, so the .has() checks above and these .add() claims must sit in
+    // the same uninterrupted synchronous span to be atomic. The whois claim
+    // was already safe (its add ran before any await), but the dig claim used
+    // to live AFTER the whois lookup's await — opening a window where a second
+    // handler for the same domain passed the dig check before the first
+    // claimed it, running dig twice. Claiming both here closes that window.
+    // Matches the existing claim-before-lookup semantics (no rollback on
+    // failure: a failed lookup still suppresses retries for the TTL).
+    if (needsWhoisLookup) processedWhoisDomains.add(whoisDedupeKey);
+    if (needsDigLookup) processedDigDomains.add(digDedupeKey);
+
     // Skip if we don't need to perform any lookups
     if (!needsWhoisLookup && !needsDigLookup) {
       if (forceDebug) {
@@ -1320,9 +1292,9 @@ function createNetToolsHandler(config) {
       
       // Perform whois lookup if either whois or whois-or is configured
       if (needsWhoisLookup) {
-        // Mark whois root domain+config as being processed
-        processedWhoisDomains.add(whoisDedupeKey);
-        
+        // Dedupe key already claimed up-front (before the first await) to keep
+        // the has()/add() span atomic across concurrent handlers.
+
         // Check whois cache first - cache key includes server for accuracy
         const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
         const whoisCacheKey = `${whoisRootDomain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
@@ -1438,11 +1410,10 @@ function createNetToolsHandler(config) {
         if (whoisResult) {
 
           if (whoisResult.success) {
-            // Lowercase the output ONCE — checkWhoisTerms / checkWhoisTermsOr
-            // each call .toLowerCase() on their input independently, which
-            // re-allocates a multi-KB lowercased string per call. Pre-lowering
-            // here lets the AND check, OR check, and matched-term find share
-            // a single allocation.
+            // Lowercase the output ONCE. The AND check, OR check, and
+            // matched-term find below each need a lowercased copy; doing it
+            // per-check would re-allocate a multi-KB string each time, so
+            // pre-lower here and let all three share a single allocation.
             const whoisOutputLower = whoisResult.output.toLowerCase();
 
             // Check AND terms if configured
@@ -1578,9 +1549,10 @@ function createNetToolsHandler(config) {
       
       // Perform dig lookup if configured
       if (needsDigLookup) {
-        // Mark dig domain+config as being processed (includes specific subdomain)
-        processedDigDomains.add(digDedupeKey);
-        
+        // Dedupe key already claimed up-front (before the first await) to keep
+        // the has()/add() span atomic across concurrent handlers — this used
+        // to claim here, after the whois await, which left the race window.
+
         if (forceDebug) {
           const digTypes = [];
           if (hasDig) digTypes.push('dig-and');
@@ -1826,8 +1798,7 @@ function createNetToolsHandler(config) {
 
 // Public surface kept narrow on purpose -- only what nwss.js actually
 // imports (verified via repo-wide grep). Internal helpers
-// (whoisLookup, whoisLookupWithRetry, digLookup, checkWhoisTerms,
-// checkWhoisTermsOr, checkDigTerms, checkDigTermsOr, selectWhoisServer,
+// (whoisLookup, whoisLookupWithRetry, digLookup, selectWhoisServer,
 // getCommonWhoisServers, suggestWhoisServers, execFileWithTimeout,
 // markResolved, digOutputIndicatesResolution, loadDiskCache,
 // saveDiskCache, enforceCacheCap, stripAnsiColors) stay as module-local