@fanboynz/network-scanner 2.0.66 → 3.0.1

package/.github/workflows/npm-publish.yml

@@ -1,41 +1,170 @@
 name: Publish to NPM
+
 on:
   workflow_dispatch:
+    inputs:
+      version_bump:
+        description: 'Version bump type'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+          - explicit
+      explicit_version:
+        description: 'Explicit version (required if version_bump = explicit, e.g. 3.0.0)'
+        required: false
+        default: ''
+      release_notes_source:
+        description: 'GitHub Release body source'
+        required: true
+        default: 'changelog'
+        type: choice
+        options:
+          - changelog
+          - manual
+      manual_release_notes:
+        description: 'Manual release notes (used if release_notes_source = manual; supports markdown)'
+        required: false
+        default: ''
+      prerelease:
+        description: 'Mark GitHub Release as pre-release'
+        required: false
+        default: false
+        type: boolean
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-      
+
     steps:
       - uses: actions/checkout@v5
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
-          
+
       - name: Setup Node.js
+        # Must be >=22.12.0 to satisfy package.json engines.node and to
+        # support require()-of-ESM for puppeteer 25 (the project's
+        # current floor). Bumping below this re-introduces the EBADENGINE
+        # warnings seen on 20.x runners and will fail npm publish for
+        # consumers on the same Node version.
         uses: actions/setup-node@v5
         with:
-          node-version: '20'
+          node-version: '22'
           registry-url: 'https://registry.npmjs.org'
-          
+
       - run: npm ci
       - run: npm run lint
-      
+
       - name: Configure git
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
-          
-      - name: Version and publish
+
+      - name: Validate explicit-version input
+        if: inputs.version_bump == 'explicit'
+        run: |
+          if [ -z "${{ inputs.explicit_version }}" ]; then
+            echo "::error::version_bump=explicit but explicit_version is empty"
+            exit 1
+          fi
+          # Loose semver shape check -- npm version will do the strict validation
+          if ! echo "${{ inputs.explicit_version }}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+([+-].+)?$'; then
+            echo "::error::explicit_version '${{ inputs.explicit_version }}' does not look like semver (e.g. 3.0.0)"
+            exit 1
+          fi
+
+      - name: Bump version + promote CHANGELOG Unreleased
+        id: version
+        run: |
+          # Use --no-git-tag-version so we can fold the CHANGELOG promotion
+          # into the same commit + tag. Otherwise the auto-commit npm creates
+          # would not include the changelog edit (and amending afterwards
+          # would orphan the tag).
+          if [ "${{ inputs.version_bump }}" = "explicit" ]; then
+            npm version "${{ inputs.explicit_version }}" --no-git-tag-version
+          else
+            npm version "${{ inputs.version_bump }}" --no-git-tag-version
+          fi
+          NEW_VERSION=$(node -p "require('./package.json').version")
+          echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
+          DATE=$(date -u +%Y-%m-%d)
+          echo "date=$DATE" >> "$GITHUB_OUTPUT"
+
+          # If CHANGELOG has an [Unreleased] section, rename its header to
+          # the new version + today's date. Idempotent: skips if the header
+          # is already a versioned one (e.g. for re-runs after a failure).
+          if grep -q '^## \[Unreleased\]$' CHANGELOG.md; then
+            sed -i "s/^## \[Unreleased\]\$/## [$NEW_VERSION] - $DATE/" CHANGELOG.md
+            echo "Promoted [Unreleased] -> [$NEW_VERSION] - $DATE"
+          else
+            echo "::warning::No [Unreleased] section in CHANGELOG.md; skipping promotion. Release notes will use whatever already exists at [$NEW_VERSION]."
+          fi
+
+          # Single combined commit + tag so the tag points at HEAD containing
+          # BOTH the package.json bump AND the changelog promotion.
+          git add package.json package-lock.json CHANGELOG.md
+          git commit -m "$NEW_VERSION"
+          git tag "v$NEW_VERSION"
+
+      - name: Extract release notes from CHANGELOG
+        id: changelog_notes
+        if: inputs.release_notes_source == 'changelog'
+        run: |
+          NEW_VERSION="${{ steps.version.outputs.version }}"
+          # Pull every line between `## [VERSION]` and the next `## [` header.
+          # awk: flag=1 after the start line; flag=0 at the next ## [ header.
+          NOTES=$(awk -v v="$NEW_VERSION" '
+            $0 ~ "^## \\[" v "\\]" { flag = 1; next }
+            $0 ~ "^## \\["          { flag = 0 }
+            flag                    { print }
+          ' CHANGELOG.md)
+
+          if [ -z "$(echo "$NOTES" | tr -d '[:space:]')" ]; then
+            echo "::warning::No CHANGELOG section found for $NEW_VERSION -- release body will be empty"
+          fi
+
+          # Multi-line GITHUB_OUTPUT requires a delimited heredoc
+          {
+            echo "notes<<NOTES_EOF"
+            echo "$NOTES"
+            echo "NOTES_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Use manual release notes
+        id: manual_notes
+        if: inputs.release_notes_source == 'manual'
         run: |
-          npm version patch
-          npm publish
+          {
+            echo "notes<<NOTES_EOF"
+            echo "${{ inputs.manual_release_notes }}"
+            echo "NOTES_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Publish to npm
+        # Publish BEFORE pushing to GitHub: if npm rejects (auth/network/
+        # name-collision), nothing reaches GitHub and a re-run is clean.
+        run: npm publish
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-          
-      - name: Push changes
+
+      - name: Push commits and tag
         run: git push --follow-tags
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.version.outputs.version }}
+          name: v${{ steps.version.outputs.version }}
+          body: ${{ inputs.release_notes_source == 'changelog' && steps.changelog_notes.outputs.notes || steps.manual_notes.outputs.notes }}
+          draft: false
+          prerelease: ${{ inputs.prerelease }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/CHANGELOG.md

@@ -2,6 +2,170 @@
 
 All notable changes to the Network Scanner (nwss.js) project.
 
+## [3.0.1] - 2026-05-24
+
+### Security
+- **Proxy credentials redacted in debug logs** — `lib/proxy.js` `getProxyInfo()` now replaces the `username:password@` segment with `[redacted]@` before logging; `lib/socks-relay.js` strips the username from both the relay-startup log (`auth: [redacted]` / `no auth`) and the close log (regex-trims the `:username` suffix from the relay key, IPv6-safe). Prior output exposed SOCKS5 credentials to anyone the user shared a debug dump, screenshot, or support ticket with.
+
+### Added
+- `scripts/test-stealth.js` — stealth smoke-test harness. Launches Puppeteer with `applyAllFingerprintSpoofing` applied and reports what bot.sannysoft.com / creepjs / browserleaks.com/javascript concluded. Flags: `--headful`, `--no-spoof` (baseline), `--ua=<family>` (validated against `USER_AGENT_COLLECTIONS`), `--format=json` (stable schema for diff/jq A/B), `--help`, positional target filtering. `PUPPETEER_NO_SANDBOX=1` env-var opt-in for CI/root containers (sandbox is on by default). Caught 3 real bugs that 5 rounds of static review missed.
+- `USER_AGENT_COLLECTIONS` exported from `lib/fingerprint.js` — single source of truth for valid UA families, consumed by the test harness so the list isn't duplicated.
+
+### Fixed
+- **Puppeteer 25 compatibility** — `browser.isConnected()` (removed in Puppeteer 25 per [puppeteer#14910](https://github.com/puppeteer/puppeteer/pull/14910)) replaced with the `browser.connected` property at 14 call sites across 6 files. Compatible with both Puppeteer 24 and 25.
+- **Fingerprint own-goal — PHANTOM_PROPERTIES + SELENIUM_DRIVER** — spoofing did `delete window[prop]` followed by `defineProperty(prop, { get: () => undefined })`. The undefined-returning getters left the properties detectable via the `in` operator, defeating the delete. Now only deletes. (caught by `scripts/test-stealth.js` sannysoft)
+- **`navigator.plugins instanceof PluginArray` failed** — the spoof returned a plain array. Now `Object.setPrototypeOf(pluginsArray, PluginArray.prototype)` with fallback to `Object.getPrototypeOf(navigator.plugins)` for environments where `PluginArray` isn't a global.
+- **`navigator.plugins[0].toString() === '[object Plugin]'` failed** — plain plugin objects returned `[object Object]`. Each plugin now wraps via `Object.create(Plugin.prototype)` with `Symbol.toStringTag` fallback.
+- **`window.chrome` descriptor was a fingerprinting tell** — had `writable: false, enumerable: false`; real Chrome has both `true`. Aligned.
+- **`_fingerprintCache` cross-UA poisoning** — was keyed by domain only, so the same domain visited under a different UA returned cached values from the wrong OS. Now keyed by `${domain}|${userAgent}`.
+- **7 broken regex patterns** in the fingerprint error-suppression list — double backslashes (`\\.X`) parsed as literal-backslash + wildcard and never matched real errors. All 7 repaired.
+- Constructor `.name` / `.length` preserved through 5 wrapper sites (Error, Image, RTCPeerConnection, PointerEvent, WheelEvent) — wrapped ctors had `.name = ''` and `.length = 0`, a fingerprinting tell.
+- `Error` static properties (`stackTraceLimit`, `captureStackTrace`, `prepareStackTrace`) forward to the OriginalError via live getter/setter instead of snapshot-copy (snapshot diverged once any caller mutated the wrapped Error).
+- `navigator.connection` fallback returns a closure-captured stable object — was re-allocating per call, so object identity changed every access.
+- `chrome.runtime.getManifest()` derives version from the spoofed UA instead of returning a hardcoded older version.
+
+### Improved
+- `isBrowserDead` helper extracted — deduped 3 spoof sites that hand-rolled the same `isConnected`/`closed` check.
+- `preserveCtorIdentity` helper added — applied at the 5 wrapper sites above.
+- GPU pool seeded by `domain + ':gpu'` (was just `domain`) — keeps per-domain GPU stable while decoupling it from any other per-domain seed we might add.
+- 10 dead module-level exports trimmed from `lib/fingerprint.js`.
+- `safeDefinePropertyLocal` forces `configurable: true` instead of merging it from the caller's descriptor (caller-side opt-in was unreliable).
+
+## [3.0.0] - 2026-05-23
+
+### Changed
+- **Engines floor bumped**: `engines.node` from `>=22.0.0` to `>=22.12.0` to match Puppeteer 25's stable `require()`-of-ESM requirement. Anyone running on Node 22.0–22.11 will see an npm engine warning and should upgrade.
+- **Puppeteer dependency floor bumped**: `puppeteer` and `puppeteer-core` from `>=20.0.0` to `>=24.0.0`. Range still permits both v24 and v25 — pick via `npm install puppeteer@24` or `npm install puppeteer@25` according to taste. Dev lockfile moved to `puppeteer@25.0.4`.
+- Audit confirms no breaking-change impact from Puppeteer 25's `executablePath`/`defaultArgs` Promise return — neither is called in this codebase. `require('puppeteer')` continues to work on the now-ESM-only package thanks to Node 22.12+'s stable require-of-ESM.
+
+### Added
+- `blockDomainsByUrl` config key (top-level) — regex patterns mirroring `ignoreDomainsByUrl` but for active blocking. A matching request URL triggers Puppeteer `request.abort()` on the triggering request, the request's root domain, and all subsequent requests to that domain or its subdomains for the rest of the scan
+- Cloudflare aggregate stats accessible via `getAggregateStats({reset})` — returns `byOutcome`, `bySolveMethod`, `maxDurationMs`, `avgDurationMs`, `failures`, `timedOut` counts; bumped on every URL regardless of debug mode
+- Cloudflare per-stage timing breakdown in outcome lines: `q=Xms p=Xms c=Xms` (zero-stage suffixes omitted)
+- Production-level Cloudflare outcome logs: `warn` severity for `!overallSuccess || timedOut`, `info` for 5xx origin-error pages, debug-only on success
+- DNS pre-check positive-resolution shortcut — hosts already proven live by dig or whois within the cache TTL skip the c-ares pre-check via a `knownResolvedHostnames` index (also warmed at startup from disk-loaded dig/whois caches)
+- DNS pre-check skip summary now reports both NXDOMAIN-cache and positive-cache savings: `DNS pre-check skipped: N URL(s) via M unresolvable host(s), N URL(s) via M resolved host(s)`
+- `[blocked-stats]` per-pattern hit counters reported at scan end — surfaces which `blocked` patterns are doing work vs. which are stale
+- `disable_adblock` per-site config flag to escape global ad-blocking layers
+- `capture_popups` now runs whois/dig validation on matched popup URLs
+- `lib/spawn-async.js` shared async-spawn helper module — consolidates 4 near-identical Promise wrappers across curl/grep/searchstring
+
+### Fixed
+- **Security**: nettools shell-injection vector closed — `exec(string)` replaced with `execFile(cmd, args)` (no shell); config-supplied `whois_server` and `recordType` values can no longer execute commands via `$()`/backticks/etc.
+- Cloudflare `detectChallengeLoop` off-by-one bug — counted the current URL against itself, tripping `>= 2` threshold one iteration early
+- Cloudflare `detectChallengeLoop` threshold was unreachable with default `cloudflare_max_retries = 2`; new exact-match path catches reload-to-same-URL loops at attempt 2
+- Cloudflare outcome cache namespace collision — now stored in a separate Map (was sharing keys with the detection cache, getting evicted by detection-cache pressure)
+- `ignoreDomains` dynamic Set didn't cascade to subdomains — `ignoreDomainsByUrl` dynamic adds now apply parent-walk just like static config (e.g. dynamically-ignored `example.com` now also catches `cdn.example.com`)
+- `blocked` / `blockDomainsByUrl` / `ignoreDomainsByUrl` regex compile failures unified — was silent-drop for *byUrl and hard-throw for blocked; now all warn loudly with `[config] X pattern dropped (compile error): "..." -- regex msg` and continue
+- adblock pattern-cache key mismatch — anchored patterns (`||example.com`) were missing their own cache because get/set used different keys
+- grep AND-logic silently dropped non-matching rules; ENOBUFS silently truncated output on large pages
+- Cloudflare debug logs rendered literal `"undefined"` when detection short-circuited on non-HTTP pages (popup → about:blank case)
+- Outcome label `no_indicators` was lying when detection short-circuited on non-HTTP page URL; now correctly reports `skipped(non-http)`
+- Cloudflare `handleLegacyCheckbox` selector list aligned with detection — dropped orphan `.cf-turnstile input[type="checkbox"]` selector that had no matching detection entry
+- Cloudflare `safeWaitForNavigation` warn was unconditional; now `forceDebug`-gated (was spamming stderr on phishing-bypass nav failures in production)
+- Cloudflare `enhancedParallelChallengeDetection` had zero callers — deleted
+- `analyzeCloudflareChallenge` ignored managed-challenge signals (`.cf-managed-challenge`, `[data-cf-managed]`); now folded into `isChallengePresent`
+- `isChallengeCompleted` double-queried the same DOM element; cached once
+- Various correctness fixes across compare (inline hosts-comment stripping), curl, dry-run, flowproxy (error-path bug, cookie parsing), referrer, searchstring, validate_rules modules
+- 30+ dead exports trimmed across nettools (11), cloudflare (18 → then re-trimmed after refactor), adblock, adblock-rust, compare, dry-run
+
+### Improved
+- Dig/whois cache TTL 14h → 20h, capacity 1000 → 2000 entries each — covers overnight scan-then-rescan cadence without forcing fresh lookups
+- nettools disk-cache writes now atomic (tmp + rename) — surviving SIGKILL/OOM/power-loss mid-write no longer leaves a truncated file that wipes the cache on next load
+- Corrupt `.digcache`/`.whoiscache` files surface a `[dns-cache] X was unreadable (...); starting fresh` warn instead of silently resetting
+- `dnsCacheStats.freshDig`/`freshWhois` arrays capped at 1000 entries (FIFO) — no more unbounded growth on scans with thousands of unique fresh lookups
+- nettools `enableDiskCache` made idempotent (uses the previously-dead `diskCacheEnabled` flag); also warms the resolved-hostnames index from loaded entries
+- 200+ log sites unified through `formatLogMessage` + subsystem tags across cloudflare, adblock, adblock-rust, compare, ignore_similar, validate_rules, wireguard_vpn, dry-run, smart-cache, flowproxy, browserexit, redirect, post-processing, cdp, output, interaction modules
+- Cloudflare `runWithRetries` helper extracted — verification-challenge and phishing-warning retry harnesses collapsed from ~150 lines of duplication to thin hook-driven wrappers
+- Cloudflare 14-line debug block in `handleVerificationChallenge` collapsed to one structured line: `Challenge detected: turnstile=t js=f ... title="..."`
+- Cloudflare timing constants pruned (4 dead, 1 dead local var); `waitForTimeout(page, ms)` renamed to `fastTimeout(ms)`, unused `page` arg dropped
+- Cloudflare `attemptChallengeSolve` post-failure diagnostic + `JS challenge` body.textContent now capped (2KB) per poll — was materializing MB on content-heavy pages
+- adblock-rust: zero-copy deserialize, eager buffer release, FIFOCache rename for honest naming
+- `interaction.js` performance: ~350ms saved per no-click interaction, ~750ms per with-click
+- nwss per-URL timeout 120s → 75s for faster hang recovery
+- Popup handler honors both `ignoreDomainsByUrl` and `blockDomainsByUrl`
+- Early `ignoreDomains` gate added at main request handler — skips dig/whois/regex cycles on ignored hostnames
+- `--dns-cache` help text refreshed (was stale "3hr/4hr TTL"; now "20h TTL, 2000-entry cap each")
+
+## [2.0.66] - 2026-05-20
+
+### Added
+- DNS pre-check before `page.goto()` to skip unresolvable hosts fast — `--no-dns-precheck` to disable
+- In-process SOCKS5 auth relay so `socks5://user:pass@host` URLs work end-to-end
+- socks-relay handshake-phase watchdog so stalled clients can't sit forever
+- DNS pre-check EAI_AGAIN retry-once + FIFO cap on negative cache
+
+### Fixed
+- proxy.js: SOCKS auth false-success + SOCKS4 remote-DNS footgun
+- DNS pre-check was starving under scan load (`dns.lookup` queued behind Puppeteer's libuv threadpool); switched to `dns.resolve` (c-ares, no threadpool contention)
+- DNS pre-check: clear the timeout timer when lookup wins the race
+- Bumped `ws` override to >=8.20.1 (CVE-2026-45736, GHSA-58qx-3vcg-4xpx)
+
+### Improved
+- Neutralize Fullscreen API so sites can't hijack the window in `--headful` mode
+- socks-relay: disable Nagle + reject unoffered no-auth selection
+
+## [2.0.65] - 2026-05-15
+
+### Added
+- Cloudflare 5xx origin-error page detection — recognizes `<domain> | 5xx: <reason>` titles, marks as `error_page(522)` etc. instead of treating as a bypass target
+- Per-URL Cloudflare outcome summary log with cookie state + error-code signal
+- HTTP status + cf-ray captured at `page.goto()` time and threaded through to the Cloudflare outcome line
+- Surface Cloudflare 5xx origin-error page count in scan stats
+- HANG CHECK: per-URL progress counter + per-URL timeout + short-circuit queued URLs on restart flag
+- Surface adblock-rust engine stats in debug exit output
+
+### Fixed
+- HANG CHECK detection logic was debug-gated and never fired in production
+- `--validate-config` TDZ crash by moving block below config load
+- Scan-exit hang: cleanups now run on normal completion (was relying on `process.exit(0)` to skip them)
+- nettools: pending-lookup leak + signal-handler conflict with nwss.js cleanup
+- cloudflare: null-safe error categorization, unref'd cache timer, body.textContent reuse
+- Suppressed contradictory "no indicators / error page detected" log pair
+
+### Improved
+- cloudflare: precompile skip-proto regex, combine within-category selectors, rename outcome key
+- redirect.js: skip `detectCommonJSRedirects` in production, cap `outerHTML`, filter `chrome-error://`
+- Cloudflare module banner + "no indicators" log deduped (was firing once per URL)
+- npm update: adblock-rs, lru-cache, puppeteer patch bumps
+- Removed dead `scanner-script-org.js` prototype
+
+## [2.0.64] - 2026-05-02
+
+### Added
+- `--adblock-engine=rust` option using Brave's adblock-rs (faster on large filter lists; requires `npm install adblock-rs`)
+- Cache hygiene: atomic write, version key, 30-day prune, JSDoc
+
+### Fixed
+- adblock-rs always returning `no_match` (4th arg to `engine.check` was missing — caused silent total-block-failure)
+- Drop existsSync before readFileSync in cache load path (avoids redundant stat + TOCTOU)
+
+### Improved
+- Reduce wrapper memory: zero-copy deserialize, eager buffer release
+- Bumped `engines.node` floor to >=22
+- npm update: `p-limit` 4.0 → 7.x (ESM API unchanged), `lru-cache` 10.4 → 11.3 (drop-in), `globals` 16.5 → 17.6 (dev-dep), `eslint` patch bump
+- V8 micro-opts in adblock-rs hot path (null-proto resource-type map, bound engine.check)
+
+## [2.0.63] - 2026-04-25
+
+### Added
+- `ignoreDomainsByUrl` config (top-level) — regex patterns; if any request URL matches, the request's root domain is dynamically ignored for the rest of the scan
+- Redirect source and matching regex now included in `adblock_rules` log titles
+
+### Fixed
+- Positional `.json` arg was ignored by config loader (always defaulted to `config.json`)
+- ReferenceError on `allowedResourceTypes` in debug log
+- ReferenceError on `matchedRegexPattern` in even_blocked path
+
+### Improved
+- Convert resourceTypes filter to Set for O(1) lookups in hot path
+- Sample `config.json` filterRegex values updated
+
+## [2.0.62] - 2026-04-25
+
+### Fixed
+- TypeError in `SmartCache.getStats` when `requestCache` fails to initialize
+
 ## [2.0.61] - 2026-03-17
 
 ### Added

package/CLAUDE.md

@@ -4,46 +4,57 @@ Puppeteer-based network scanner for analyzing web traffic, generating adblock fi
 
 ## Project Structure
 
-- `nwss.js` — Main entry point (~4,600 lines). CLI args, URL processing, orchestration.
+- `nwss.js` — Main entry point (~5,800 lines). CLI args, URL processing, orchestration.
 - `config.json` — Default scan configuration (sites, filters, options).
-- `lib/` — 28 focused, single-purpose modules:
+- `lib/` — 32 focused, single-purpose modules:
   - `fingerprint.js` — Bot detection evasion (device/GPU/timezone spoofing)
   - `cloudflare.js` — Cloudflare challenge detection and solving
   - `browserhealth.js` — Memory management and browser lifecycle
   - `interaction.js` — Human-like mouse/scroll/typing simulation
+  - `ghost-cursor.js` — Bezier-curve cursor pathing for human-like mouse movement
   - `smart-cache.js` — Multi-layer caching with persistence
   - `nettools.js` — WHOIS/dig integration
   - `output.js` — Multi-format rule output (adblock, dnsmasq, unbound, pihole, etc.)
   - `proxy.js` — SOCKS5/HTTP proxy support
+  - `socks-relay.js` — Local SOCKS proxy relay/chain helper
   - `wireguard_vpn.js` / `openvpn_vpn.js` — VPN routing
-  - `adblock.js` — Adblock filter parsing and validation
+  - `adblock.js` — Adblock filter parsing and validation (native JS engine)
+  - `adblock-rust.js` — Drop-in adblock.js replacement backed by Brave's `adblock-rs` Rust engine; same matcher shape (`shouldBlock`, `getStats`, `rules`) so callers swap with one `require()`
   - `validate_rules.js` — Domain and rule format validation
   - `colorize.js` — Console output formatting and colors
   - `domain-cache.js` — Domain detection cache for performance
   - `post-processing.js` — Result cleanup and deduplication
+  - `spawn-async.js` — Shared `runProcess(cmd, args, opts)` helper used by curl/grep/searchstring; resolves (never rejects) with `{code, signal, stdout, stderr, truncated, error}`, enforces timeout + stdout caps
   - `redirect.js`, `referrer.js`, `cdp.js`, `curl.js`, `grep.js`, `compare.js`, `compress.js`, `dry-run.js`, `browserexit.js`, `clear_sitedata.js`, `flowproxy.js`, `ignore_similar.js`, `searchstring.js`
 - `.github/workflows/npm-publish.yml` — Automated npm publishing
 - `nwss.1` — Man page
 
 ## Tech Stack
 
-- **Node.js** >=22.0.0
-- **puppeteer** >=20.0.0 — Headless browser automation
-- **psl** — Public Suffix List for domain parsing
+- **Node.js** >=22.12.0 (required for stable `require()` of ESM-only puppeteer 25)
+- **puppeteer** >=24.0.0 — Headless browser automation. Range permits both v24 and v25; dev lockfile is on v25.
+- **psl** — Public Suffix List for domain parsing (prefer this over hand-curated TLD lists)
 - **lru-cache** — LRU cache implementation
 - **p-limit** — Concurrency limiting (dynamically imported)
+- **adblock-rs** — Optional native Rust filter engine, used by `lib/adblock-rust.js`. Install with `npm install adblock-rs` (requires Rust toolchain). Not a hard dep — `lib/adblock.js` is the default.
 - **eslint** — Linting (`npm run lint`)
 
 ## Conventions
 
 - Store modular functionality in `./lib/` with focused, single-purpose modules
 - Use `messageColors` and `formatLogMessage` from `./lib/colorize` for consistent console output
+- Prefix every log line with a subsystem tag, e.g. `const TAG = messageColors.processing('[adblock]');` then `formatLogMessage('warn', `${TAG} ...`)`. Keeps mixed-module output attributable; every module in `lib/` follows this — match it when adding new ones.
+- Pick severities deliberately: `warn` for actual errors/failures (cache write fail, native exception), `debug` for diagnostic chatter (cache misses, parse summaries, per-match traces)
 - Implement timeout protection for all Puppeteer operations using `Promise.race` patterns
 - Handle browser lifecycle with comprehensive cleanup in try-finally blocks
 - Validate all external tool availability before use (grep, curl, whois, dig)
 - Use `forceDebug` flag for detailed logging, `silentMode` for minimal output
 - Use `Object.freeze` for constant configuration objects (TIMEOUTS, CACHE_LIMITS, CONCURRENCY_LIMITS)
-- Use `fastTimeout(ms)` helper instead of `node:timers/promises` for Puppeteer 22.x compatibility
+- Use `fastTimeout(ms)` helper instead of `node:timers/promises` for delays — project convention since the Puppeteer 22.x `page.waitForTimeout` removal, retained as the standard for all Promise-based sleeps
+- Prefer `runProcess` from `./lib/spawn-async` over bare `child_process.spawn`/`spawnSync` for new external-tool calls. It resolves (never rejects), enforces a SIGKILL timeout + stdout cap, and returns a uniform result object. `lib/wireguard_vpn.js` intentionally stays on `spawnSync` — startup-only validation paths where sync is simpler. Don't follow that exception unless you have the same justification.
+- Prefer `net.isIP()` over hand-rolled IPv4/IPv6 regexes for IP validation
+- For disk-cache writes use the atomic `tmpPath = path + '.' + pid + '.tmp'` + `fs.renameSync` pattern (see `lib/adblock-rust.js`) so a killed process never leaves a half-written cache file
+- Keep `module.exports` minimal — trim helpers that have no external consumers (grep the repo before deciding); internal-only functions stay as functions but leave the exports surface
 
 ## Running
 
@@ -55,6 +66,28 @@ node nwss.js --dry-run                # Preview without network calls
 node nwss.js --headful                # Launch with browser GUI
 ```
 
+## Stealth Testing
+
+`scripts/test-stealth.js` is a smoke-test harness for the fingerprint spoofing
+stack. Launches Puppeteer with `applyAllFingerprintSpoofing` applied (same
+call shape nwss.js uses), navigates to public bot-detection pages, and
+reports what they concluded. Use it to A/B a stealth change — run before the
+edit, run after, diff. Found 3 real bugs that 5 rounds of static review
+missed (PHANTOM/SELENIUM own-goal, PluginArray instanceof, Plugin toString).
+
+```bash
+node scripts/test-stealth.js                  # all targets, human-readable
+node scripts/test-stealth.js sannysoft        # one target
+node scripts/test-stealth.js --no-spoof       # baseline (spoof disabled)
+node scripts/test-stealth.js --format=json    # machine-readable for diff/jq
+node scripts/test-stealth.js --help           # full flag list
+```
+
+Set `PUPPETEER_NO_SANDBOX=1` when running as root (CI containers). Off by
+default so local dev doesn't silently drop the sandbox. The harness depends
+on `USER_AGENT_COLLECTIONS` exported from `lib/fingerprint.js` — keep that
+export in sync if the UA list changes.
+
 ## Files to Ignore
 
 - `node_modules/**`

package/README.md

@@ -66,7 +66,8 @@ A Puppeteer-based tool for scanning websites to find third-party (or optionally
 | `--use-puppeteer-core`      | Use `puppeteer-core` with system Chrome instead of bundled Chromium |
 | `--use-obscura`             | Connect to running Obscura CDP server (`ws://127.0.0.1:9222` or `OBSCURA_WS` env). Skips fingerprint injection — Obscura provides built-in stealth |
 | `--load-extension <path>`   | Load unpacked Chrome extension from directory (can be used multiple times) |
-| `--dns-cache`               | Persist dig/whois results to disk between runs (14hr TTL, `.digcache`/`.whoiscache`) |
+| `--dns-cache`               | Persist dig/whois results to disk between runs (20hr TTL, 2000-entry cap each, `.digcache`/`.whoiscache`). Disk writes are atomic (tmp + rename); corrupt cache files are detected on load with a `[dns-cache]` warn line and reset cleanly. |
+| `--no-dns-precheck`         | Disable per-URL DNS resolution check before page navigation. By default, hosts that dig/whois have already proven live (within the 20hr cache TTL) skip their c-ares pre-check via a positive-resolution index. |
 | `--block-ads=<files>`       | Block ads using EasyList format rules (comma-separated: `easylist.txt,easyprivacy.txt`) |
 | `--cdp`                     | Enable Chrome DevTools Protocol logging (now per-page if enabled) |
 | `--remove-dupes`            | Remove duplicate domains from output (only with `-o`) |
@@ -101,6 +102,12 @@ Example:
     "googleapis.com",
     "googletagmanager.com"
   ],
+  "ignoreDomainsByUrl": [
+    "\\/jwplayer\\/"
+  ],
+  "blockDomainsByUrl": [
+    "\\/tracker\\/"
+  ],
   "sites": [
     {
       "url": "https://example.com/",
@@ -461,9 +468,10 @@ These options go at the root level of your config.json:
 
 | Field                | Values | Default | Description |
 |:---------------------|:-------|:-------:|:------------|
-| `ignoreDomains`      | Array | - | Domains to completely ignore (supports wildcards like `*.ads.com`) |
-| `ignoreDomainsByUrl` | Array | - | Regex patterns; if a request URL matches, the request's root domain is dynamically ignored for the rest of the scan (e.g. `["\\/jwplayer\\/", "\\/build\\/assets\\/"]`) |
-| `blocked`            | Array | - | Global regex patterns to block requests (combined with per-site blocked) |
+| `ignoreDomains`      | Array | - | Domains to completely ignore (supports wildcards like `*.ads.com`). Subdomains of any listed entry are also ignored via parent-walk (e.g. `example.com` ignores `cdn.example.com` and `a.b.example.com`). |
+| `ignoreDomainsByUrl` | Array | - | Regex patterns; if a request URL matches, the request's root domain is dynamically ignored for the rest of the scan AND any subsequent request to its subdomains (cascade matches the static `ignoreDomains` semantic). Example: `["\\/jwplayer\\/", "\\/build\\/assets\\/"]` |
+| `blockDomainsByUrl` | Array | - | Symmetric to `ignoreDomainsByUrl` but for active blocking. Regex patterns; if a request URL matches, the request's root domain is added to a dynamic block set and ALL subsequent requests on that root (and subdomains) are aborted via Puppeteer for the rest of the scan. The triggering request itself is also aborted. Use when seeing a trigger URL is sufficient evidence the whole host is hostile. |
+| `blocked`            | Array | - | Global regex patterns to block requests (combined with per-site blocked). Patterns that fail to compile are warned about at scan start (`[config] blocked (global) pattern dropped (compile error): ...`) instead of crashing startup or silently disappearing. Per-pattern hit counts are reported at scan end via `[blocked-stats]` lines so stale patterns are easy to spot. |
 | `whois_server_mode`  | String | `"random"` | Default server selection mode for all sites |
 | `ignore_similar`     | Boolean | `true` | Ignore domains similar to already found domains |
 | `ignore_similar_threshold` | Integer | `80` | Similarity threshold percentage for ignore_similar |
@@ -782,4 +790,21 @@ your_username ALL=(root) NOPASSWD: /usr/bin/wg-quick, /usr/bin/wg
 - Ghost-cursor (`cursor_mode: "ghost"`) is optional — install with `npm i ghost-cursor`. Falls back to built-in mouse if not installed
 - Ghost-cursor duration defaults to `interact_duration` (or 2000ms), capped by the 15s hard timeout
 
+## Stealth Testing
+
+`scripts/test-stealth.js` is a developer-facing smoke test for the fingerprint spoofing stack in `lib/fingerprint.js`. It launches Puppeteer with the same `applyAllFingerprintSpoofing` call that `nwss.js` uses, navigates to public bot-detection pages, and reports what they concluded — pass/warn/fail counts from sannysoft, trust score from creepjs, raw navigator values from browserleaks.
+
+Use it to A/B a stealth change: run before the edit, run after, diff the output. Not a unit test — it doesn't assert; it reports.
+
+```bash
+node scripts/test-stealth.js                  # all targets, human-readable
+node scripts/test-stealth.js sannysoft        # one target only
+node scripts/test-stealth.js --no-spoof       # baseline (spoof disabled)
+node scripts/test-stealth.js --ua=firefox     # spoof a different UA family
+node scripts/test-stealth.js --format=json    # machine-readable for diff/jq
+node scripts/test-stealth.js --help           # full flag list
+```
+
+Set `PUPPETEER_NO_SANDBOX=1` when running as root (CI containers, some Docker setups). Off by default so local dev doesn't silently drop the Chromium sandbox.
+
 ---

package/lib/adblock-rust.js

@@ -10,6 +10,10 @@ const fs = require('fs');
 const path = require('path');
 const os = require('os');
 const crypto = require('crypto');
+const { formatLogMessage, messageColors } = require('./colorize');
+// Subsystem tag matches the project convention used by other modules
+// (lib/adblock.js, flowproxy, cloudflare, curl, grep, etc.).
+const ADBLOCK_RUST_TAG = messageColors.processing('[adblock-rust]');
 
 let adblockRust = null;
 let adblockRustVersion = null;
@@ -77,18 +81,19 @@ const RESOURCE_TYPE_MAP = Object.assign(Object.create(null), {
   '':                    ''
 });
 
-function normalizeResourceType(type) {
-  if (!type) return '';
-  return RESOURCE_TYPE_MAP[type] || 'other';
-}
+// Removed: normalizeResourceType() helper. The hot path in shouldBlock
+// inlines the (RESOURCE_TYPE_MAP[rt] || 'other') lookup directly to skip
+// the function-call frame; the standalone helper had zero callers.
 
-// Small FIFO cache keyed on (url \0 sourceUrl \0 resourceType). Despite the
-// class name, eviction is insertion-order, not access-order — `get()` does not
-// promote. For this workload (per-page request bursts whose working set fits
-// in maxSize) FIFO and true LRU produce the same evictions, so the simpler
-// path wins. If cache effectiveness becomes a concern with larger working
-// sets, promote on hit by re-inserting (delete + set).
-class ResultLRU {
+// Small FIFO cache keyed on (url \0 sourceUrl \0 resourceType). Eviction
+// is insertion-order — `get()` does not promote. For this workload
+// (per-page request bursts whose working set fits in maxSize) FIFO and
+// true LRU produce the same evictions, so the simpler path wins. If
+// cache effectiveness becomes a concern with larger working sets,
+// promote on hit by re-inserting (delete + set). Renamed from ResultLRU
+// since the previous name lied about the eviction policy — matches
+// the FIFOCache rename in lib/adblock.js.
+class FIFOCache {
   constructor(maxSize) {
     this.cache = new Map();
     this.maxSize = maxSize;
@@ -172,7 +177,7 @@ function parseAdblockRules(filePathOrArray, options = {}) {
       compiled = fs.readFileSync(cachePath);
     } catch (err) {
       if (err.code !== 'ENOENT' && enableLogging) {
-        console.log(`[Adblock-Rust] Cache read failed (${err.message}); reparsing`);
+        console.log(formatLogMessage('debug', `${ADBLOCK_RUST_TAG} Cache read failed (${err.message}); reparsing`));
       }
     }
     if (compiled) {
@@ -196,7 +201,7 @@ function parseAdblockRules(filePathOrArray, options = {}) {
         // Corrupt cache or version mismatch — fall through to a fresh parse.
         engine = null;
         if (enableLogging) {
-          console.log(`[Adblock-Rust] Cache deserialize failed (${err.message}); reparsing`);
+          console.log(formatLogMessage('debug', `${ADBLOCK_RUST_TAG} Cache deserialize failed (${err.message}); reparsing`));
         }
       }
     }
@@ -240,7 +245,7 @@ function parseAdblockRules(filePathOrArray, options = {}) {
         pruneOldCacheFiles(cacheDir, cacheTtlMs);
       } catch (err) {
         if (enableLogging) {
-          console.log(`[Adblock-Rust] Cache write failed (${err.message}); continuing`);
+          console.log(formatLogMessage('warn', `${ADBLOCK_RUST_TAG} Cache write failed (${err.message}); continuing`));
         }
       }
     }
@@ -262,7 +267,7 @@ function parseAdblockRules(filePathOrArray, options = {}) {
     cacheMisses: 0
   };
 
-  const resultCache = new ResultLRU(resultCacheSize);
+  const resultCache = new FIFOCache(resultCacheSize);
   // Hot-path optimization: shared "no_match" object — most checks return this,
   // skip per-call object allocation. Safe because callers only read fields.
   const NO_MATCH = Object.freeze({ blocked: false, rule: null, reason: 'no_match' });
@@ -275,9 +280,9 @@ function parseAdblockRules(filePathOrArray, options = {}) {
 
   if (enableLogging) {
     if (cacheHit) {
-      console.log(`[Adblock-Rust] Restored compiled engine from ${cachePath} (${(totalBytes/1024/1024).toFixed(2)}MB source, ${filePaths.length} list${filePaths.length>1?'s':''})`);
+      console.log(formatLogMessage('debug', `${ADBLOCK_RUST_TAG} Restored compiled engine from ${cachePath} (${(totalBytes/1024/1024).toFixed(2)}MB source, ${filePaths.length} list${filePaths.length>1?'s':''})`));
     } else {
-      console.log(`[Adblock-Rust] Compiled ${ruleCount} rules from ${filePaths.length} list${filePaths.length>1?'s':''} (${(totalBytes/1024/1024).toFixed(2)}MB)`);
+      console.log(formatLogMessage('debug', `${ADBLOCK_RUST_TAG} Compiled ${ruleCount} rules from ${filePaths.length} list${filePaths.length>1?'s':''} (${(totalBytes/1024/1024).toFixed(2)}MB)`));
     }
   }
 
@@ -315,7 +320,7 @@ function parseAdblockRules(filePathOrArray, options = {}) {
       } catch (err) {
         stats.errors++;
         if (enableLogging) {
-          console.log(`[Adblock-Rust] Error checking ${url}: ${err.message}`);
+          console.log(formatLogMessage('warn', `${ADBLOCK_RUST_TAG} Error checking ${url}: ${err.message}`));
         }
         // Don't cache errors — next call may succeed (transient native panic).
         return { blocked: false, rule: null, reason: 'error' };