@paywalls-net/filter 1.3.12 → 1.3.14

package/package.json

@@ -3,7 +3,7 @@
   "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
   "author": "paywalls.net",
   "license": "MIT",
-  "version": "1.3.12",
+  "version": "1.3.14",
   "publishConfig": {
     "access": "public"
   },

package/src/index.js

@@ -185,7 +185,18 @@ async function proxyVAIRequest(cfg, request) {
             cf.country ? `co=${cf.country}, re=${cf.region || ''}, ci=${cf.city || ''}, asn=${cf.asn || ''}` : null);
 
         // Tier 3: UA features + HMAC (§6.3)
-        setIfPresent(forwardHeaders, 'X-PW-UA', extractUAFeatures(headers['user-agent']));
+        let uaFeatures = extractUAFeatures(headers['user-agent']);
+
+        // Sec-Fetch context mismatch detection (paywalls-site-dz23):
+        // vai.json is fetched via sync XHR — browsers set Sec-Fetch-Dest: empty,
+        // Mode: cors. If we see document/navigate, the headless browser is leaking
+        // page-level Sec-Fetch onto the XHR. Emit marker so cloud-api can classify.
+        if (uaFeatures && cloudApiPath.endsWith('/vai.json') &&
+            headers['sec-fetch-dest'] === 'document' && headers['sec-fetch-mode'] === 'navigate') {
+            uaFeatures += ', sec-fetch-mismatch';
+        }
+
+        setIfPresent(forwardHeaders, 'X-PW-UA', uaFeatures);
         setIfPresent(forwardHeaders, 'X-PW-UA-HMAC', await computeUAHMAC(headers['user-agent'], cfg.vaiUAHmacKey));
         setIfPresent(forwardHeaders, 'X-PW-CT-FP',   await computeConfidenceToken(headers['user-agent'], headers['accept-language'], headers['sec-ch-ua']));
 

package/src/signal-extraction.js

@@ -446,6 +446,50 @@ function bucketVersion(ver) {
 }
 
 // ── §6.3.1  extractUAFeatures ──────────────────────────────────────────────
+
+/**
+ * Detect structurally impossible or fabricated browser version strings.
+ *
+ * Chrome frozen UA policy (since Chrome 107, late 2022):
+ *   Real Chrome reports Chrome/[major].0.0.0 — minor, build, and patch are
+ *   always zero.  Any major >= 107 with non-zero build or patch is fabricated.
+ *
+ * Legacy Chrome with 4-digit patch (e.g. Chrome/48.0.1025.1402):
+ *   Chrome patch numbers are 1-4 digits (max ~6367 in historical builds).
+ *   A 4+ digit patch on an old Chrome version is structurally fabricated.
+ *
+ * Fabricated Edge (e.g. Edge/18.19582):
+ *   Edge/18 was EdgeHTML-era; real minor versions were at most 3 digits.
+ *   A 5-digit minor on EdgeHTML is structurally impossible.
+ *
+ * @param {string} ua
+ * @returns {boolean}
+ */
+function isFabricatedVersion(ua) {
+  // Chrome / HeadlessChrome / Chromium: full version parse
+  const chromeMatch = ua.match(/(?:\b|Headless)Chrom(?:e|ium)\/(\d+)\.(\d+)\.(\d+)\.(\d+)/);
+  if (chromeMatch) {
+    const major = parseInt(chromeMatch[1], 10);
+    const build = parseInt(chromeMatch[3], 10);
+    const patch = parseInt(chromeMatch[4], 10);
+
+    // Frozen UA policy: Chrome >= 107 must be major.0.0.0
+    if (major >= 107 && (build !== 0 || patch !== 0)) return true;
+
+    // 4-digit patch on any Chrome version is structurally impossible
+    if (chromeMatch[4].length >= 4) return true;
+  }
+
+  // EdgeHTML-era (Edge/12-18): minor version should be ≤ 3 digits
+  const edgeMatch = ua.match(/\bEdge\/(\d+)\.(\d+)/);
+  if (edgeMatch) {
+    const major = parseInt(edgeMatch[1], 10);
+    if (major <= 18 && edgeMatch[2].length >= 5) return true;
+  }
+
+  return false;
+}
+
 /**
  * Parse a User-Agent string into an SF-Dictionary of derived features.
  *
@@ -468,6 +512,12 @@ export function extractUAFeatures(userAgent) {
 
   if (HEADLESS_MARKERS.some(re => re.test(ua))) parts.push('headless');
   if (AUTOMATION_MARKERS.some(re => re.test(ua))) parts.push('automation');
+  if (isFabricatedVersion(ua)) parts.push('fabricated');
+
+  // Stale version: Chrome family with ver=0-79 is 6+ years old (pre-2020)
+  if (family === 'chrome' && extractMajorVersion(ua) !== null && extractMajorVersion(ua) < 80) {
+    parts.push('stale');
+  }
 
   parts.push(`entropy=${computeUAEntropy(ua)}`);
 

package/tests/bot-signal-extraction.test.js

@@ -0,0 +1,263 @@
+/**
+ * Unit tests for bot/automation signal extraction via extractUAFeatures
+ *
+ * Tests the Tier 3 UA feature extractor against real-world bot signals
+ * discovered from Cloudflare production logs (2026-03-14).
+ *
+ * Fixture: paywalls-site/tests/fixtures/cloudflare-prod-paywalls-2026-03-14.csv
+ * Issue: (to be assigned)
+ *
+ * These tests verify that extractUAFeatures correctly identifies:
+ *  - headless markers (HeadlessChrome)
+ *  - automation markers (Puppeteer, Selenium, etc.)
+ *  - bot family detection (Googlebot, Applebot, Bytespider, etc.)
+ *  - device/platform/family parsing for suspicious UAs
+ *  - fabricated version patterns
+ */
+import {
+  extractUAFeatures,
+  _resetVAIMetadata,
+} from '../src/signal-extraction.js';
+
+beforeEach(() => {
+  _resetVAIMetadata();
+});
+
+// ── 1. HeadlessChrome detection ────────────────────────────────────────────
+
+describe('HeadlessChrome signal extraction', () => {
+  test('HeadlessChrome/145 should have headless marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/145.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/\bheadless\b/);
+    expect(result).toMatch(/dpf=desktop\/linux\/chrome/);
+    expect(result).toMatch(/browser/);
+  });
+
+  test('HeadlessChrome/143 should have headless and fabricated markers', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/143.0.7499.4 Safari/537.36'
+    );
+    expect(result).toMatch(/\bheadless\b/);
+    expect(result).toMatch(/dpf=desktop\/linux\/chrome/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 2. Self-identified bots ────────────────────────────────────────────────
+
+describe('Self-identified bot signal extraction', () => {
+  test('Applebot should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Googlebot mobile should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.7632.116 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Googlebot desktop should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/145.0.7632.116 Safari/537.36'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('Bytespider should be detected as bot family', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)'
+    );
+    expect(result).toMatch(/\/bot/);
+  });
+
+  test('amazon-Quick non-browser UA should have low entropy', () => {
+    const result = extractUAFeatures('amazon-Quick-on-behalf-of-20e61c5a');
+    expect(result).not.toMatch(/browser/);
+    expect(result).toMatch(/entropy=low/);
+  });
+});
+
+// ── 3. Fabricated Chrome versions (4-digit patch) ──────────────────────────
+
+describe('Fabricated Chrome version UAs — signal extraction', () => {
+  // These have impossible 4-digit patch numbers. extractUAFeatures doesn't
+  // currently detect version fabrication, but it should at minimum:
+  // - Parse the very old major version into the low bucket (0-79)
+  // - Have medium entropy (looks like a browser UA)
+
+  const fabricatedUAs = [
+    { ua: 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.1025.1402 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.5596.1136 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2714.1709 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.5974.1013 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.1957.1646 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.4130.1795 Mobile Safari/537.36', ver: '0-79' },
+    { ua: 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.7842.1119 Mobile Safari/537.36', ver: '0-79' },
+  ];
+
+  test.each(fabricatedUAs)('$ver Chrome with 4-digit patch → fabricated marker', ({ ua, ver }) => {
+    const result = extractUAFeatures(ua);
+    expect(result).toMatch(new RegExp(`ver=${ver}`));
+    expect(result).toMatch(/browser/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 4. Fabricated Edge version ─────────────────────────────────────────────
+
+describe('Fabricated Edge version — signal extraction', () => {
+  test('Edge/18.19582 should parse as edge family and be fabricated', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.7680.71 Safari/537.36 Edge/18.19582'
+    );
+    expect(result).toMatch(/\/edge/);
+    expect(result).toMatch(/dpf=desktop\/windows\/edge/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+});
+
+// ── 5. Outdated browser UAs from bot farm ──────────────────────────────────
+
+describe('Outdated browser UAs from bot farm — version bucketing', () => {
+  test('Chrome/59 (2017) should bucket to 0-79', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+    expect(result).toMatch(/dpf=mobile\/android\/chrome/);
+  });
+
+  test('Chrome/117 with non-zero build (frozen UA violation) → fabricated', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=100-119/);
+    expect(result).toMatch(/\bfabricated\b/);
+  });
+
+  test('Chrome/83 should bucket to 80-99', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=80-99/);
+  });
+
+  test('Chrome/79 should bucket to 0-79', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+  });
+});
+
+// ── 6. Legitimate browser UAs ──────────────────────────────────────────────
+
+describe('Legitimate browser UAs — correct feature extraction', () => {
+  test('Chrome/145 on macOS → desktop/mac/chrome, current version bucket', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/dpf=desktop\/mac\/chrome/);
+    expect(result).toMatch(/ver=140-159/);
+    expect(result).toMatch(/browser/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Chrome/146 on Windows → desktop/windows/chrome, current version bucket', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/dpf=desktop\/windows\/chrome/);
+    expect(result).toMatch(/ver=140-159/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Safari/17.4.1 on macOS → desktop/mac/safari', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15'
+    );
+    expect(result).toMatch(/dpf=desktop\/mac\/safari/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/automation/);
+    expect(result).not.toMatch(/fabricated/);
+  });
+
+  test('Edge/122 on Windows → desktop/windows/edge', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0'
+    );
+    expect(result).toMatch(/dpf=desktop\/windows\/edge/);
+    expect(result).toMatch(/ver=120-139/);
+    expect(result).not.toMatch(/headless/);
+    expect(result).not.toMatch(/fabricated/);
+    expect(result).not.toMatch(/stale/);
+  });
+});
+
+// ── 7. Stale browser version marker ───────────────────────────────────────
+
+describe('Stale browser version — signal extraction', () => {
+  test('Chrome/59 (2017, ver=0-79) should have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+    expect(result).toMatch(/\bstale\b/);
+  });
+
+  test('Chrome/79 (2019, ver=0-79) should have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=0-79/);
+    expect(result).toMatch(/\bstale\b/);
+  });
+
+  test('Chrome/83 (80-99 bucket, borderline old) should NOT have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=80-99/);
+    expect(result).not.toMatch(/\bstale\b/);
+  });
+
+  test('Chrome/145 (current) should NOT have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'
+    );
+    expect(result).toMatch(/ver=140-159/);
+    expect(result).not.toMatch(/\bstale\b/);
+  });
+
+  test('Safari/17 should NOT have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15'
+    );
+    expect(result).not.toMatch(/\bstale\b/);
+  });
+
+  test('Firefox/130 should NOT have stale marker', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0'
+    );
+    expect(result).not.toMatch(/\bstale\b/);
+  });
+
+  test('Fabricated Chrome/48.0.1025.1402 also gets stale (both markers)', () => {
+    const result = extractUAFeatures(
+      'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.1025.1402 Mobile Safari/537.36'
+    );
+    expect(result).toMatch(/\bfabricated\b/);
+    expect(result).toMatch(/\bstale\b/);
+  });
+});

package/tests/proxy-vai-request.test.js

@@ -291,6 +291,56 @@ describe('proxyVAIRequest — signal extraction pipeline', () => {
   });
 });
 
+// ── sec-fetch-mismatch CDN marker emission ──────────────────────────────────
+//
+// When the CDN sees sec-fetch-dest: document + sec-fetch-mode: navigate on a
+// vai.json request, it appends sec-fetch-mismatch to X-PW-UA. This marker is
+// used by the cloud-api to reclassify the request as OTHER without inspecting
+// raw Sec-Fetch headers in the classification path.
+
+describe('CDN sec-fetch-mismatch marker emission', () => {
+  let handler;
+
+  beforeAll(async () => {
+    handler = await init('cloudflare');
+  });
+
+  async function proxyVAI(url, headerMap, cf) {
+    const request = makeRequest(url, headerMap, cf);
+    capturedFetchArgs = null;
+    await handler(request, ENV, CTX);
+    return capturedFetchArgs;
+  }
+
+  test('document/navigate on vai.json → appends sec-fetch-mismatch', async () => {
+    const mismatchHeaders = {
+      ...CHROME_MAC_HEADERS,
+      'sec-fetch-dest': 'document',
+      'sec-fetch-mode': 'navigate',
+    };
+    const args = await proxyVAI('https://pub.example.com/pw/vai.json', mismatchHeaders, CF_PROPS);
+    expect(args.headers['X-PW-UA']).toContain('sec-fetch-mismatch');
+  });
+
+  test('empty/cors on vai.json (normal XHR) → no sec-fetch-mismatch', async () => {
+    const args = await proxyVAI('https://pub.example.com/pw/vai.json', CHROME_MAC_HEADERS, CF_PROPS);
+    expect(args.headers['X-PW-UA']).not.toContain('sec-fetch-mismatch');
+  });
+
+  test('document/navigate on non-vai path → no sec-fetch-mismatch', async () => {
+    const mismatchHeaders = {
+      ...CHROME_MAC_HEADERS,
+      'sec-fetch-dest': 'document',
+      'sec-fetch-mode': 'navigate',
+    };
+    const args = await proxyVAI('https://pub.example.com/pw/access/check', mismatchHeaders, CF_PROPS);
+    // sec-fetch-mismatch only applies to vai.json requests
+    if (args && args.headers) {
+      expect(args.headers['X-PW-UA'] || '').not.toContain('sec-fetch-mismatch');
+    }
+  });
+});
+
 // ── logAccess — non-VAI path raw header forwarding ───────────────────────────
 //
 // Verifies that logAccess() does NOT transform headers like proxyVAIRequest().