@paywalls-net/filter 1.3.11 → 1.3.13

package/package.json

@@ -3,7 +3,7 @@
   "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
   "author": "paywalls.net",
   "license": "MIT",
-  "version": "1.3.11",
+  "version": "1.3.13",
   "publishConfig": {
     "access": "public"
   },

package/src/index.js

@@ -180,6 +180,10 @@ async function proxyVAIRequest(cfg, request) {
         setIfPresent(forwardHeaders, 'X-PW-Net',    extractNetFeatures(cf.asn));
         setIfPresent(forwardHeaders, 'X-PW-CH',     extractCHFeatures(headers['sec-ch-ua'], headers['user-agent']));
 
+        // Geo context: forward CF edge geo data for logging/investigation (paywalls-site-60rp)
+        setIfPresent(forwardHeaders, 'X-PW-Geo',
+            cf.country ? `co=${cf.country}, re=${cf.region || ''}, ci=${cf.city || ''}, asn=${cf.asn || ''}` : null);
+
         // Tier 3: UA features + HMAC (§6.3)
         setIfPresent(forwardHeaders, 'X-PW-UA', extractUAFeatures(headers['user-agent']));
         setIfPresent(forwardHeaders, 'X-PW-UA-HMAC', await computeUAHMAC(headers['user-agent'], cfg.vaiUAHmacKey));

package/src/signal-extraction.js

@@ -446,6 +446,50 @@ function bucketVersion(ver) {
 }
 
 // ── §6.3.1  extractUAFeatures ──────────────────────────────────────────────
+
+/**
+ * Detect structurally impossible or fabricated browser version strings.
+ *
+ * Chrome frozen UA policy (since Chrome 107, late 2022):
+ *   Real Chrome reports Chrome/[major].0.0.0 — minor, build, and patch are
+ *   always zero.  Any major >= 107 with non-zero build or patch is fabricated.
+ *
+ * Legacy Chrome with 4-digit patch (e.g. Chrome/48.0.1025.1402):
+ *   Chrome patch numbers are 1-4 digits (max ~6367 in historical builds).
+ *   A 4+ digit patch on an old Chrome version is structurally fabricated.
+ *
+ * Fabricated Edge (e.g. Edge/18.19582):
+ *   Edge/18 was EdgeHTML-era; real minor versions were at most 3 digits.
+ *   A 5-digit minor on EdgeHTML is structurally impossible.
+ *
+ * @param {string} ua
+ * @returns {boolean}
+ */
+function isFabricatedVersion(ua) {
+  // Chrome / HeadlessChrome / Chromium: full version parse
+  const chromeMatch = ua.match(/(?:\b|Headless)Chrom(?:e|ium)\/(\d+)\.(\d+)\.(\d+)\.(\d+)/);
+  if (chromeMatch) {
+    const major = parseInt(chromeMatch[1], 10);
+    const build = parseInt(chromeMatch[3], 10);
+    const patch = parseInt(chromeMatch[4], 10);
+
+    // Frozen UA policy: Chrome >= 107 must be major.0.0.0
+    if (major >= 107 && (build !== 0 || patch !== 0)) return true;
+
+    // 4-digit patch on any Chrome version is structurally impossible
+    if (chromeMatch[4].length >= 4) return true;
+  }
+
+  // EdgeHTML-era (Edge/12-18): minor version should be ≤ 3 digits
+  const edgeMatch = ua.match(/\bEdge\/(\d+)\.(\d+)/);
+  if (edgeMatch) {
+    const major = parseInt(edgeMatch[1], 10);
+    if (major <= 18 && edgeMatch[2].length >= 5) return true;
+  }
+
+  return false;
+}
+
 /**
  * Parse a User-Agent string into an SF-Dictionary of derived features.
  *
@@ -468,6 +512,7 @@ export function extractUAFeatures(userAgent) {
 
   if (HEADLESS_MARKERS.some(re => re.test(ua))) parts.push('headless');
   if (AUTOMATION_MARKERS.some(re => re.test(ua))) parts.push('automation');
+  if (isFabricatedVersion(ua)) parts.push('fabricated');
 
   parts.push(`entropy=${computeUAEntropy(ua)}`);
 

package/tests/bot-signal-extraction.test.js

@@ -0,0 +1,204 @@
+/**
+ * Unit tests for bot/automation signal extraction via extractUAFeatures
+ *
+ * Tests the Tier 3 UA feature extractor against real-world bot signals
+ * discovered from Cloudflare production logs (2026-03-14).
+ *
+ * Fixture: paywalls-site/tests/fixtures/cloudflare-prod-paywalls-2026-03-14.csv
+ * Issue: (to be assigned)
+ *
+ * These tests verify that extractUAFeatures correctly identifies:
+ *  - headless markers (HeadlessChrome)
+ *  - automation markers (Puppeteer, Selenium, etc.)
+ *  - bot family detection (Googlebot, Applebot, Bytespider, etc.)
+ *  - device/platform/family parsing for suspicious UAs
+ *  - fabricated version patterns
+ */
+import {
+  extractUAFeatures,
+  _resetVAIMetadata,
+} from '../src/signal-extraction.js';
+
+beforeEach(() => {
+  _resetVAIMetadata();
+});
+
+// ── 1. HeadlessChrome detection ────────────────────────────────────────────
+
+describe('HeadlessChrome signal extraction', () => {
+  test('HeadlessChrome/145 should have headless marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/145.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/\bheadless\b/);
+    expect(result).toMatch(/dpf=desktop\/linux\/chrome/);
+    expect(result).toMatch(/browser/);
+  });
+
+  test('HeadlessChrome/143 should have headless and fabricated markers', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/143.0.7499.4 Safari/537.36'
+    );
+    expect(result).toMatch(/\bheadless\b/);
+    expect(result).toMatch(/dpf=desktop\/linux\/chrome/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 2. Self-identified bots ────────────────────────────────────────────────
+
+describe('Self-identified bot signal extraction', () => {
+  test('Applebot should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Googlebot mobile should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.7632.116 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Googlebot desktop should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.7632.116 Safari/537.36'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Bytespider should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('amazon-Quick non-browser UA should have low entropy', () => {
+    const result = extractUAFeatures('amazon-Quick-on-behalf-of-20e61c5a');
+    expect(result).not.toMatch(/browser/);
+    expect(result).toMatch(/entropy=low/);
+  });
+});
+
+// ── 3. Fabricated Chrome versions (4-digit patch) ──────────────────────────
+
+describe('Fabricated Chrome version UAs — signal extraction', () => {
+  // These have impossible 4-digit patch numbers. extractUAFeatures doesn't
+  // currently detect version fabrication, but it should at minimum:
+  // - Parse the very old major version into the low bucket (0-79)
+  // - Have medium entropy (looks like a browser UA)
+
+  const fabricatedUAs = [
+    { ua: 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.1025.1402 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.5596.1136 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2714.1709 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.5974.1013 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.1957.1646 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.4130.1795 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.7842.1119 Mobile Safari/537.36', ver: '0-79' },
+  ];
+
+  test.each(fabricatedUAs)('$ver Chrome with 4-digit patch → fabricated marker', ({ ua, ver }) => {
+    const result = extractUAFeatures(ua);
+    expect(result).toMatch(new RegExp(`ver=${ver}`));
+    expect(result).toMatch(/browser/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 4. Fabricated Edge version ─────────────────────────────────────────────
+
+describe('Fabricated Edge version — signal extraction', () => {
+  test('Edge/18.19582 should parse as edge family and be fabricated', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.7680.71 Safari/537.36 Edge/18.19582'
+    );
+    expect(result).toMatch(/\/edge/);
+    expect(result).toMatch(/dpf=desktop\/windows\/edge/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 5. Outdated browser UAs from bot farm ──────────────────────────────────
+
+describe('Outdated browser UAs from bot farm — version bucketing', () => {
+  test('Chrome/59 (2017) should bucket to 0-79', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+    expect(result).toMatch(/dpf=mobile\/android\/chrome/);
+  });
+
+  test('Chrome/117 with non-zero build (frozen UA violation) → fabricated', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=100-119/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+
+  test('Chrome/83 should bucket to 80-99', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=80-99/);
+  });
+
+  test('Chrome/79 should bucket to 0-79', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+  });
+});
+
+// ── 6. Legitimate browser UAs ──────────────────────────────────────────────
+
+describe('Legitimate browser UAs — correct feature extraction', () => {
+  test('Chrome/145 on macOS → desktop/mac/chrome, current version bucket', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/dpf=desktop\/mac\/chrome/);
+    expect(result).toMatch(/ver=140-159/);
+    expect(result).toMatch(/browser/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Chrome/146 on Windows → desktop/windows/chrome, current version bucket', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/dpf=desktop\/windows\/chrome/);
+    expect(result).toMatch(/ver=140-159/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Safari/17.4.1 on macOS → desktop/mac/safari', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15'
+    );
+    expect(result).toMatch(/dpf=desktop\/mac\/safari/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Edge/122 on Windows → desktop/windows/edge', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0'
+    );
+    expect(result).toMatch(/dpf=desktop\/windows\/edge/);
+    expect(result).toMatch(/ver=120-139/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+});