@paywalls-net/filter 1.3.10 → 1.3.11

package/package.json

@@ -3,7 +3,7 @@
   "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
   "author": "paywalls.net",
   "license": "MIT",
-  "version": "1.3.10",
+  "version": "1.3.11",
   "publishConfig": {
     "access": "public"
   },

package/src/index.js

@@ -8,6 +8,7 @@ import {
     extractAcceptFeatures, extractEncodingFeatures, extractLanguageFeatures,
     extractNetFeatures, extractCHFeatures, extractUAFeatures,
     computeUAHMAC, computeConfidenceToken,
+    loadVAIMetadata,
 } from './signal-extraction.js';
 
 const PAYWALLS_CLOUD_API_HOST = "https://cloud-api.paywalls.net";
@@ -428,7 +429,11 @@ async function cloudflare(config = null) {
             return await proxyVAIRequest(paywallsConfig, request);
         }
         
-        await loadAgentPatterns(paywallsConfig);
+        // Load agent patterns + VAI metadata in parallel (both self-cache for 1 hour)
+        await Promise.all([
+            loadAgentPatterns(paywallsConfig),
+            loadVAIMetadata(paywallsConfig),
+        ]);
 
         if (await isRecognizedBot(paywallsConfig, request)) {
             const authz = await checkAgentStatus(paywallsConfig, request);
@@ -461,7 +466,11 @@ async function fastly() {
             return await proxyVAIRequest(paywallsConfig, request);
         }
 
-        await loadAgentPatterns(paywallsConfig);
+        // Load agent patterns + VAI metadata in parallel (both self-cache for 1 hour)
+        await Promise.all([
+            loadAgentPatterns(paywallsConfig),
+            loadVAIMetadata(paywallsConfig),
+        ]);
 
         if (await isRecognizedBot(paywallsConfig, request)) {
             const authz = await checkAgentStatus(paywallsConfig, request);
@@ -538,7 +547,11 @@ async function cloudfront(config) {
         vaiPath: config.PAYWALLS_VAI_PATH || '/pw',
         vaiUAHmacKey: config.VAI_UA_HMAC_KEY || null,
     };
-    await loadAgentPatterns(paywallsConfig);
+    // Load agent patterns + VAI metadata in parallel (both self-cache for 1 hour)
+    await Promise.all([
+        loadAgentPatterns(paywallsConfig),
+        loadVAIMetadata(paywallsConfig),
+    ]);
 
     return async function handle(event, ctx) {
         let request = event.Records[0].cf.request;

package/src/signal-extraction.js

@@ -11,11 +11,17 @@
  * omit the header entirely (not send an empty value).
  */
 
-// ── §6.2.4 / Appendix A: Data-center ASN set ──────────────────────────────
-// Comprehensive cloud/hosting provider ASNs for DC classification.
-// Kept in sync with cloud-api DC_ASN_LIST (cloudflare/vai.js).
-// Source: public ASN registries (PeeringDB, RIPE, ARIN).
-const DC_ASN_SET = new Set([
+// ── VAI Metadata: dynamic loading with hardcoded fallbacks ──────────────────
+// (paywalls-site-fc4)
+//
+// These module-level vars are initialized from hardcoded defaults below.
+// When loadVAIMetadata() is called, they are updated from the cloud-api
+// /pw/vai/metadata endpoint.  If the fetch fails, the hardcoded defaults
+// remain in effect — no data loss, no crash.
+
+// ── Hardcoded defaults (bootstrap / fallback) ──────────────────────────────
+
+const DEFAULT_DC_ASNS = [
   // ── Major IaaS ───────────────────────────────────────────────────────────
   16509, 14618,          // Amazon AWS (primary + secondary)
   396982, 36492, 15169,  // Google Cloud + Google infra
@@ -34,7 +40,126 @@ const DC_ASN_SET = new Set([
   12876,                 // Scaleway
   51167,                 // Contabo
   60781, 28753,          // Leaseweb (NL + global)
-]);
+];
+
+const DEFAULT_AUTOMATION_PATTERNS = [
+  'Puppeteer', 'Playwright', 'Selenium', 'WebDriver',
+  'PhantomJS', 'CasperJS',
+  'python-requests', 'python-urllib', 'Go-http-client',
+  'okhttp', 'Apache-HttpClient', 'libcurl',
+  '\\bcurl\\/', '\\bwget\\/', 'HTTPie',
+  'node-fetch', 'undici', 'axios\\/', '\\bgot\\/', 'superagent',
+  'Cypress', 'TestCafe', 'Nightwatch', 'WebdriverIO',
+  'Scrapy', 'Java\\/|Java HttpURLConnection', 'PostmanRuntime\\/',
+  '\\bDeno\\/', '\\bhttpx\\b|python-httpx',
+];
+
+const DEFAULT_HEADLESS_PATTERNS = [
+  'HeadlessChrome', '\\bHeadless\\b',
+];
+
+const DEFAULT_BOT_PATTERNS = [
+  'Googlebot', 'bingbot', 'Baiduspider', 'YandexBot', 'DuckDuckBot',
+  'Slurp', 'ia_archiver', 'GPTBot', 'ClaudeBot', 'CCBot', 'Bytespider',
+  'Applebot', 'PetalBot', 'SemrushBot', 'AhrefsBot', 'DotBot',
+];
+
+// ── Mutable state: updated by loadVAIMetadata() ────────────────────────────
+
+/** @type {Set<number>} */
+let DC_ASN_SET = new Set(DEFAULT_DC_ASNS);
+
+/** @type {RegExp[]} */
+let AUTOMATION_MARKERS = DEFAULT_AUTOMATION_PATTERNS.map(p => new RegExp(p, 'i'));
+
+/** @type {RegExp[]} */
+let HEADLESS_MARKERS = DEFAULT_HEADLESS_PATTERNS.map(p => new RegExp(p, 'i'));
+
+/** @type {RegExp} — single combined regex for bot family detection */
+let BOT_FAMILY_RE = new RegExp('\\b(' + DEFAULT_BOT_PATTERNS.join('|') + ')\\b', 'i');
+
+// ── Metadata cache ─────────────────────────────────────────────────────────
+
+let _vaiMetadataCache = null;     // { data, ts }
+const VAI_METADATA_TTL = 60 * 60 * 1000; // 1 hour
+
+/**
+ * Compile pattern strings (from metadata JSON) into RegExp objects.
+ * Each string is treated as a regex source with case-insensitive flag.
+ * @param {string[]} patterns
+ * @returns {RegExp[]}
+ */
+function compilePatterns(patterns) {
+  return patterns.map(p => new RegExp(p, 'i'));
+}
+
+/**
+ * Fetch VAI metadata from cloud-api and update mutable module state.
+ * Caches for 1 hour.  Falls back to hardcoded defaults on failure.
+ *
+ * Pattern: matches loadAgentPatterns() in user-agent-classification.js.
+ *
+ * @param {Object} cfg  Config with paywallsAPIHost (cloud-api base URL)
+ * @returns {Promise<void>}
+ */
+export async function loadVAIMetadata(cfg) {
+  const now = Date.now();
+
+  // Return early if cache is still valid
+  if (_vaiMetadataCache && (now - _vaiMetadataCache.ts) < VAI_METADATA_TTL) {
+    return;
+  }
+
+  try {
+    const response = await fetch(`${cfg.paywallsAPIHost}/pw/vai/metadata`, {
+      method: 'GET',
+      headers: { 'Accept': 'application/json' },
+    });
+
+    if (!response.ok) {
+      throw new Error(`VAI metadata fetch failed: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json();
+
+    // Validate minimal schema
+    if (!data || typeof data.version !== 'number') {
+      throw new Error('VAI metadata: invalid schema (missing version)');
+    }
+
+    // Update mutable state from fetched data
+    if (Array.isArray(data.dc_asns) && data.dc_asns.length > 0) {
+      DC_ASN_SET = new Set(data.dc_asns);
+    }
+    if (Array.isArray(data.automation_patterns) && data.automation_patterns.length > 0) {
+      AUTOMATION_MARKERS = compilePatterns(data.automation_patterns);
+    }
+    if (Array.isArray(data.headless_patterns) && data.headless_patterns.length > 0) {
+      HEADLESS_MARKERS = compilePatterns(data.headless_patterns);
+    }
+    if (Array.isArray(data.bot_patterns) && data.bot_patterns.length > 0) {
+      BOT_FAMILY_RE = new RegExp('\\b(' + data.bot_patterns.join('|') + ')\\b', 'i');
+    }
+
+    _vaiMetadataCache = { data, ts: now };
+  } catch (error) {
+    console.error('loadVAIMetadata: fetch failed, using hardcoded defaults.', error.message || error);
+    // Mark cache so we don't retry immediately (back off for 5 minutes)
+    _vaiMetadataCache = { data: null, ts: now - VAI_METADATA_TTL + (5 * 60 * 1000) };
+  }
+}
+
+/**
+ * Reset metadata state to hardcoded defaults and clear cache.
+ * Exposed for testing only.
+ */
+export function _resetVAIMetadata() {
+  DC_ASN_SET = new Set(DEFAULT_DC_ASNS);
+  AUTOMATION_MARKERS = DEFAULT_AUTOMATION_PATTERNS.map(p => new RegExp(p, 'i'));
+  HEADLESS_MARKERS = DEFAULT_HEADLESS_PATTERNS.map(p => new RegExp(p, 'i'));
+  BOT_FAMILY_RE = new RegExp('\\b(' + DEFAULT_BOT_PATTERNS.join('|') + ')\\b', 'i');
+  _vaiMetadataCache = null;
+}
 
 // ── §6.2.1  Accept → X-PW-Accept ──────────────────────────────────────────
 /**
@@ -191,17 +316,9 @@ export function extractCHFeatures(secChUA, userAgent) {
 // ── §6.3.3  Automation marker detection ────────────────────────────────────
 // HeadlessChrome triggers 'headless' only (via HEADLESS_MARKERS).
 // Explicit automation tools (Puppeteer, Selenium, etc.) trigger 'automation'.
-const AUTOMATION_MARKERS = [
-  /Puppeteer/i, /Playwright/i, /Selenium/i, /WebDriver/i,
-  /PhantomJS/i, /CasperJS/i,
-  /python-requests/i, /python-urllib/i, /Go-http-client/i,
-  /okhttp/i, /Apache-HttpClient/i, /libcurl/i,
-  /\bcurl\//i, /\bwget\//i, /HTTPie/i,
-  /node-fetch/i, /undici/i, /axios\//i, /\bgot\//i, /superagent/i,
-  /Cypress/i, /TestCafe/i, /Nightwatch/i, /WebdriverIO/i,
-];
-
-const HEADLESS_MARKERS = [/HeadlessChrome/i, /\bHeadless\b/i];
+// AUTOMATION_MARKERS and HEADLESS_MARKERS are now module-level mutable vars
+// initialized from hardcoded defaults (top of file) and updated dynamically
+// by loadVAIMetadata().  See paywalls-site-fc4.
 
 // ── §6.3.4  Entropy bucketing ──────────────────────────────────────────────
 /**
@@ -237,8 +354,18 @@ function computeUAEntropy(userAgent) {
 
 // ── §6.3.1  UA dpf/version parsing ─────────────────────────────────────────
 
-/** @returns {'desktop'|'mobile'|'tablet'|'server'|'unknown'} */
+/** @returns {'desktop'|'mobile'|'tablet'|'smarttv'|'console'|'car'|'wearable'|'vr'|'server'|'unknown'} */
 function detectDevice(ua) {
+  // Smart TV: check before tablet/mobile (some TVs include Android)
+  if (/SmartTV|SMART-TV|\bTizen\b|\bWebOS\b|\bBRAVIA\b|\bVizio\b|\bRoku\b|\bAppleTV\b|\bFire TV\b|\bAndroidTV\b|\btvOS\b|\bHBBTV\b/i.test(ua)) return 'smarttv';
+  // Gaming consoles
+  if (/\b(PlayStation|PLAYSTATION|Xbox|Nintendo)\b/i.test(ua)) return 'console';
+  // VR headsets (Meta Quest / Oculus)
+  if (/OculusBrowser|\bQuest\b/i.test(ua)) return 'vr';
+  // Wearables (Apple Watch, etc.)
+  if (/\bWatch\b|\bwearable\b/i.test(ua)) return 'wearable';
+  // Automotive
+  if (/\bTesla\b|\bCarPlay\b/i.test(ua)) return 'car';
   if (/\b(iPad|Tablet|PlayBook|Silk|Kindle)\b/i.test(ua)) return 'tablet';
   if (/\b(iPhone|iPod|Android.*Mobile|Mobile.*Android|webOS|BlackBerry|Opera Mini|IEMobile|Windows Phone)\b/i.test(ua)) return 'mobile';
   if (/\b(Android)\b/i.test(ua) && !/Mobile/i.test(ua)) return 'tablet';
@@ -247,24 +374,31 @@ function detectDevice(ua) {
   return 'unknown';
 }
 
-/** @returns {'windows'|'mac'|'ios'|'android'|'linux'|'other'} */
+/** @returns {'windows'|'mac'|'ios'|'android'|'linux'|'chromeos'|'freebsd'|'other'} */
 function detectPlatform(ua) {
   if (/\b(iPhone|iPad|iPod)\b/i.test(ua))          return 'ios';
   if (/\bAndroid\b/i.test(ua))                      return 'android';
+  if (/\bCrOS\b/i.test(ua))                         return 'chromeos';
   if (/\bMacintosh\b/i.test(ua))                    return 'mac';
   if (/\bWindows\b/i.test(ua))                      return 'windows';
+  if (/\bFreeBSD\b/i.test(ua))                      return 'freebsd';
   if (/\bLinux\b/i.test(ua) || /\bX11\b/i.test(ua)) return 'linux';
   return 'other';
 }
 
-/** @returns {'chrome'|'safari'|'firefox'|'edge'|'other'|'bot'} */
+/** @returns {'chrome'|'safari'|'firefox'|'edge'|'ucbrowser'|'other'|'bot'} */
 function detectFamily(ua) {
-  if (/\b(Googlebot|bingbot|Baiduspider|YandexBot|DuckDuckBot|Slurp|ia_archiver)\b/i.test(ua)) return 'bot';
+  // Bots: search engine crawlers + AI/SEO crawlers (dynamic via loadVAIMetadata)
+  if (BOT_FAMILY_RE.test(ua)) return 'bot';
+  // UC Browser: mobile-heavy, no Client Hints — check before Chrome
+  if (/UCBrowser|UCWEB/i.test(ua)) return 'ucbrowser';
   // Order matters: Edge before Chrome (Edge UA contains "Chrome")
   if (/\bEdg(?:e|A)?\/\d/i.test(ua))  return 'edge';
   if (/\bFirefox\//i.test(ua))         return 'firefox';
   // Safari check: has "Safari/" but NOT "Chrome/" or "Chromium/" or "HeadlessChrome/"
   if (/\bSafari\//i.test(ua) && !/Chrome|Chromium|HeadlessChrome/i.test(ua)) return 'safari';
+  // Opera (OPR/) and Brave share Chromium engine; keep as 'chrome' family
+  // since they support Client Hints and score the same.
   if (/(?:\b|Headless)Chrom(?:e|ium)\//i.test(ua))  return 'chrome';
   return 'other';
 }
@@ -294,16 +428,21 @@ function extractMajorVersion(ua) {
 
 /**
  * Bucket a major version number into a range token.
+ * Uses math-based 20-version spans starting at 80, capped at 420+.
+ * Legacy range: 0-79. Then 80-99, 100-119, …, 400-419, 420+.
  * @param {number|null} ver
  * @returns {string}
  */
 function bucketVersion(ver) {
-  if (ver == null) return '0-79';
-  if (ver < 80)  return '0-79';
-  if (ver < 100) return '80-99';
-  if (ver < 120) return '100-119';
-  if (ver < 140) return '120-139';
-  return '140+';
+  if (ver == null || ver < 80)  return '0-79';
+  if (ver >= 420) return '420+';
+  // 20-version spans starting at 80: floor((ver - 80) / 20) gives bucket index
+  const base = 80;
+  const span = 20;
+  const bucketIndex = Math.floor((ver - base) / span);
+  const lo = base + bucketIndex * span;
+  const hi = lo + span - 1;
+  return `${lo}-${hi}`;
 }
 
 // ── §6.3.1  extractUAFeatures ──────────────────────────────────────────────

package/tests/signal-extraction.test.js

@@ -2,7 +2,7 @@
  * Unit tests for signal extraction functions (Tier 2 + Tier 3)
  *
  * Spec: specs/vai-privacy-v2.spec.md §6.2–§6.4
- * Issue: paywalls-site-drk
+ * Issue: paywalls-site-drk, paywalls-site-fc4
  */
 import {
   extractAcceptFeatures,
@@ -13,6 +13,8 @@ import {
   extractUAFeatures,
   computeUAHMAC,
   computeConfidenceToken,
+  loadVAIMetadata,
+  _resetVAIMetadata,
 } from '../src/signal-extraction.js';
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -209,6 +211,21 @@ describe('extractNetFeatures', () => {
     expect(extractNetFeatures('7922')).toBe('asn=consumer');
   });
 
+  // ── Full DC_ASN_SET coverage ───────────────────────────────────────────
+  test.each([
+    [16509, 'AWS primary'], [14618, 'AWS secondary'],
+    [396982, 'GCP'], [36492, 'GCP secondary'], [15169, 'Google infra'],
+    [8075, 'Azure'], [8069, 'Azure secondary'], [8068, 'Azure tertiary'],
+    [31898, 'Oracle Cloud'], [36351, 'IBM/SoftLayer'],
+    [45102, 'Alibaba Cloud'], [132203, 'Tencent Cloud'],
+    [14061, 'DigitalOcean'], [24940, 'Hetzner'], [213230, 'Hetzner Cloud'],
+    [16276, 'OVH'], [63949, 'Linode/Akamai'], [20473, 'Vultr'],
+    [12876, 'Scaleway'], [51167, 'Contabo'],
+    [60781, 'Leaseweb NL'], [28753, 'Leaseweb global'],
+  ])('DC ASN %i (%s) → asn=cloud', (asn) => {
+    expect(extractNetFeatures(asn)).toBe('asn=cloud');
+  });
+
   // ── Input type handling ────────────────────────────────────────────────
   test('numeric input (number, not string) → works', () => {
     expect(extractNetFeatures(16509)).toBe('asn=cloud');
@@ -360,6 +377,135 @@ describe('extractUAFeatures', () => {
     expect(extractUAFeatures(GOOGLEBOT)).toMatch(/dpf=server\/other\/bot/);
   });
 
+  // ── new device types ───────────────────────────────────────────────────
+  test('Smart TV (Tizen) → smarttv device', () => {
+    const ua = 'Mozilla/5.0 (SMART-TV; Linux; Tizen 5.0) AppleWebKit/537.36 Chrome/69.0.3497.106 TV Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=smarttv\//);
+  });
+
+  test('Smart TV (WebOS) → smarttv device', () => {
+    const ua = 'Mozilla/5.0 (Web0S; Linux/SmartTV) AppleWebKit/537.36 WebOS TV/5.0';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=smarttv\//);
+  });
+
+  test('Smart TV (Fire TV) → smarttv device', () => {
+    const ua = 'Mozilla/5.0 (Linux; Android 9; AFTS Build/PS7233) AppleWebKit/537.36 (KHTML, like Gecko) Silk/120.4.1 like Chrome/120.0.0.0 Mobile Safari/537.36 Fire TV';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=smarttv\//);
+  });
+
+  test('PlayStation → console device', () => {
+    const ua = 'Mozilla/5.0 (PlayStation 5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=console\//);
+  });
+
+  test('Xbox → console device', () => {
+    const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edge/44.18363.8131';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=console\//);
+  });
+
+  test('Nintendo → console device', () => {
+    const ua = 'Mozilla/5.0 (Nintendo Switch; WifiWebAuthApplet) AppleWebKit/606.4 (KHTML, like Gecko) NF/6.0.1.16.10 NintendoBrowser/5.1.0.22474';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=console\//);
+  });
+
+  test('Meta Quest (Oculus) → vr device', () => {
+    const ua = 'Mozilla/5.0 (Linux; Android 12; Quest 3) AppleWebKit/537.36 (KHTML, like Gecko) OculusBrowser/33.0 Chrome/126.0.6478.122 Mobile VR Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=vr\//);
+  });
+
+  test('Apple Watch → wearable device', () => {
+    const ua = 'Mozilla/5.0 (Watch; CPU Watch OS 10_0 like Mac OS X) AppleWebKit/605.1.15';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=wearable\//);
+  });
+
+  test('Tesla → car device', () => {
+    const ua = 'Mozilla/5.0 (X11; GNU/Linux; Tesla) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/^dpf=car\//);
+  });
+
+  // ── new platforms ──────────────────────────────────────────────────────
+  test('ChromeOS → chromeos platform', () => {
+    const ua = 'Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/dpf=desktop\/chromeos\/chrome/);
+  });
+
+  test('FreeBSD → freebsd platform', () => {
+    const ua = 'Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/dpf=desktop\/freebsd\/chrome/);
+  });
+
+  // ── new browser families ───────────────────────────────────────────────
+  test('UC Browser → ucbrowser family', () => {
+    const ua = 'Mozilla/5.0 (Linux; Android 10; SM-A505F) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/16.0.1.3715 Mobile Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/dpf=mobile\/android\/ucbrowser/);
+  });
+
+  test('UCWEB variant → ucbrowser family', () => {
+    const ua = 'UCWEB/2.0 (Java; U; MIDP-2.0)';
+    expect(extractUAFeatures(ua)).toMatch(/\/ucbrowser/);
+  });
+
+  test('Opera (OPR/) → chrome family (Chromium-based)', () => {
+    const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 OPR/120.0.0.0';
+    expect(extractUAFeatures(ua)).toMatch(/\/chrome/);
+  });
+
+  test('Brave → chrome family (indistinguishable UA)', () => {
+    // Brave UA is intentionally identical to Chrome
+    const ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
+    expect(extractUAFeatures(ua)).toMatch(/\/chrome/);
+  });
+
+  // ── AI/SEO bot detection ───────────────────────────────────────────────
+  test('GPTBot → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)')).toMatch(/\/bot/);
+  });
+
+  test('ClaudeBot → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 (compatible; ClaudeBot/1.0; +https://anthropic.com)')).toMatch(/\/bot/);
+  });
+
+  test('CCBot → bot family', () => {
+    expect(extractUAFeatures('CCBot/2.0 (https://commoncrawl.org/faq/)')).toMatch(/\/bot/);
+  });
+
+  test('Bytespider → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)')).toMatch(/\/bot/);
+  });
+
+  test('Applebot → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 (compatible; Applebot/0.1; +http://www.apple.com/go/applebot)')).toMatch(/\/bot/);
+  });
+
+  test('SemrushBot → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)')).toMatch(/\/bot/);
+  });
+
+  test('AhrefsBot → bot family', () => {
+    expect(extractUAFeatures('Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)')).toMatch(/\/bot/);
+  });
+
+  // ── new automation markers ─────────────────────────────────────────────
+  test('Scrapy → automation', () => {
+    expect(extractUAFeatures('Scrapy/2.11.0 (+https://scrapy.org)')).toMatch(/\bautomation\b/);
+  });
+
+  test('Java HttpURLConnection → automation', () => {
+    expect(extractUAFeatures('Java/17.0.2')).toMatch(/\bautomation\b/);
+  });
+
+  test('PostmanRuntime → automation', () => {
+    expect(extractUAFeatures('PostmanRuntime/7.36.1')).toMatch(/\bautomation\b/);
+  });
+
+  test('Deno → automation', () => {
+    expect(extractUAFeatures('Deno/1.40.0')).toMatch(/\bautomation\b/);
+  });
+
+  test('httpx → automation', () => {
+    expect(extractUAFeatures('python-httpx/0.27.0')).toMatch(/\bautomation\b/);
+  });
+
   test('iPad → tablet/ios/safari', () => {
     expect(extractUAFeatures(IPAD)).toMatch(/^dpf=tablet\/ios\/safari/);
   });
@@ -386,6 +532,7 @@ describe('extractUAFeatures', () => {
   });
 
   // Bucket boundary tests: verify version numbers at edges of each range
+  // Math-based 20-version spans starting at 80, capped at 420+.
   test.each([
     ['Chrome/79.0.0.0',  '0-79'],
     ['Chrome/80.0.0.0',  '80-99'],
@@ -394,8 +541,14 @@ describe('extractUAFeatures', () => {
     ['Chrome/119.0.0.0', '100-119'],
     ['Chrome/120.0.0.0', '120-139'],
     ['Chrome/139.0.0.0', '120-139'],
-    ['Chrome/140.0.0.0', '140+'],
-    ['Chrome/999.0.0.0', '140+'],
+    ['Chrome/140.0.0.0', '140-159'],
+    ['Chrome/159.0.0.0', '140-159'],
+    ['Chrome/160.0.0.0', '160-179'],
+    ['Chrome/200.0.0.0', '200-219'],
+    ['Chrome/400.0.0.0', '400-419'],
+    ['Chrome/419.0.0.0', '400-419'],
+    ['Chrome/420.0.0.0', '420+'],
+    ['Chrome/999.0.0.0', '420+'],
   ])('version bucket boundary: %s → ver=%s', (chromeToken, expected) => {
     // Wrap in a minimal browser-like UA so detectDevice/detectPlatform work
     const ua = `Mozilla/5.0 (X11; Linux x86_64) ${chromeToken} Safari/537.36`;
@@ -622,3 +775,228 @@ describe('computeConfidenceToken', () => {
     expect(a).not.toBe(b);
   });
 });
+
+// ═══════════════════════════════════════════════════════════════════════════
+//  loadVAIMetadata — Dynamic metadata loading (paywalls-site-fc4)
+// ═══════════════════════════════════════════════════════════════════════════
+
+/** Simple tracking mock for fetch — records call count and args */
+function mockFetchWith(response) {
+  let calls = 0;
+  const fn = async (url, opts) => {
+    calls++;
+    fn._lastUrl = url;
+    fn._lastOpts = opts;
+    return response;
+  };
+  fn.callCount = () => calls;
+  fn._lastUrl = null;
+  fn._lastOpts = null;
+  return fn;
+}
+
+function mockFetchReject(error) {
+  return async () => { throw error; };
+}
+
+describe('loadVAIMetadata', () => {
+  const originalFetch = globalThis.fetch;
+  const originalConsoleError = console.error;
+  const cfg = { paywallsAPIHost: 'https://cloud-api.example.com' };
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    console.error = originalConsoleError;
+    _resetVAIMetadata();
+  });
+
+  test('updates DC_ASN_SET from fetched metadata', async () => {
+    // Before: hardcoded defaults include 16509 (AWS)
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+    expect(extractNetFeatures(99999)).toBe('asn=consumer');
+
+    // Mock metadata with a custom ASN set
+    const mock = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [99999],  // Custom: only 99999 is cloud
+        automation_patterns: ['Puppeteer'],
+        headless_patterns: ['HeadlessChrome'],
+        bot_patterns: ['Googlebot'],
+      }),
+    });
+    globalThis.fetch = mock;
+
+    await loadVAIMetadata(cfg);
+
+    // After: 99999 is now cloud, 16509 is not
+    expect(extractNetFeatures(99999)).toBe('asn=cloud');
+    expect(extractNetFeatures(16509)).toBe('asn=consumer');
+    expect(mock.callCount()).toBe(1);
+    expect(mock._lastUrl).toBe('https://cloud-api.example.com/pw/vai/metadata');
+    expect(mock._lastOpts.method).toBe('GET');
+  });
+
+  test('updates automation markers from fetched metadata', async () => {
+    // Before: hardcoded includes Puppeteer
+    const before = extractUAFeatures('Mozilla/5.0 Puppeteer/1.0');
+    expect(before).toContain('automation');
+
+    // Mock with a custom automation list that doesn't include Puppeteer
+    globalThis.fetch = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [16509],
+        automation_patterns: ['CustomBot'],
+        headless_patterns: ['HeadlessChrome'],
+        bot_patterns: ['Googlebot'],
+      }),
+    });
+
+    await loadVAIMetadata(cfg);
+
+    // Puppeteer no longer matches automation
+    const after = extractUAFeatures('Mozilla/5.0 Puppeteer/1.0');
+    expect(after).not.toContain('automation');
+
+    // CustomBot does
+    const custom = extractUAFeatures('Mozilla/5.0 CustomBot/2.0');
+    expect(custom).toContain('automation');
+  });
+
+  test('updates bot patterns from fetched metadata', async () => {
+    // Before: hardcoded includes Googlebot → family=bot
+    const before = extractUAFeatures('Googlebot/2.1');
+    expect(before).toContain('bot');
+
+    // Mock with a different bot list
+    globalThis.fetch = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [16509],
+        automation_patterns: ['Puppeteer'],
+        headless_patterns: ['HeadlessChrome'],
+        bot_patterns: ['NewAIBot'],
+      }),
+    });
+
+    await loadVAIMetadata(cfg);
+
+    // Googlebot no longer detected as bot family
+    const afterGoogle = extractUAFeatures('Googlebot/2.1');
+    expect(afterGoogle).not.toContain('dpf=server/other/bot');
+
+    // NewAIBot is now detected
+    const afterNew = extractUAFeatures('NewAIBot/1.0');
+    expect(afterNew).toContain('bot');
+  });
+
+  test('falls back to defaults when fetch fails (network error)', async () => {
+    globalThis.fetch = mockFetchReject(new Error('Network error'));
+    console.error = () => {};  // suppress expected error
+
+    await loadVAIMetadata(cfg);
+
+    // Defaults still active: 16509 is cloud
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+    // Automation still works
+    const ua = extractUAFeatures('Mozilla/5.0 Puppeteer/1.0');
+    expect(ua).toContain('automation');
+  });
+
+  test('falls back to defaults when fetch returns non-OK', async () => {
+    globalThis.fetch = mockFetchWith({
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+    });
+    console.error = () => {};  // suppress expected error
+
+    await loadVAIMetadata(cfg);
+
+    // Defaults still active
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+  });
+
+  test('falls back to defaults when response has invalid schema', async () => {
+    globalThis.fetch = mockFetchWith({
+      ok: true,
+      json: async () => ({ invalid: true }),  // missing version
+    });
+    console.error = () => {};  // suppress expected error
+
+    await loadVAIMetadata(cfg);
+
+    // Defaults still active
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+  });
+
+  test('caches metadata and does not re-fetch within TTL', async () => {
+    const mock = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [16509],
+        automation_patterns: ['Puppeteer'],
+        headless_patterns: ['HeadlessChrome'],
+        bot_patterns: ['Googlebot'],
+      }),
+    });
+    globalThis.fetch = mock;
+
+    await loadVAIMetadata(cfg);
+    expect(mock.callCount()).toBe(1);
+
+    // Second call within 1 hour — should not fetch again
+    await loadVAIMetadata(cfg);
+    expect(mock.callCount()).toBe(1);
+  });
+
+  test('_resetVAIMetadata restores hardcoded defaults', async () => {
+    // Load custom metadata
+    globalThis.fetch = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [99999],
+        automation_patterns: ['OnlyThisOne'],
+        headless_patterns: ['OnlyHeadless'],
+        bot_patterns: ['OnlyBot'],
+      }),
+    });
+
+    await loadVAIMetadata(cfg);
+    expect(extractNetFeatures(99999)).toBe('asn=cloud');
+    expect(extractNetFeatures(16509)).toBe('asn=consumer');
+
+    // Reset
+    _resetVAIMetadata();
+
+    // Back to defaults
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+    expect(extractNetFeatures(99999)).toBe('asn=consumer');
+  });
+
+  test('ignores empty arrays in metadata (keeps defaults)', async () => {
+    globalThis.fetch = mockFetchWith({
+      ok: true,
+      json: async () => ({
+        version: 1,
+        dc_asns: [],
+        automation_patterns: [],
+        headless_patterns: [],
+        bot_patterns: [],
+      }),
+    });
+
+    await loadVAIMetadata(cfg);
+
+    // Defaults still active (empty arrays are ignored)
+    expect(extractNetFeatures(16509)).toBe('asn=cloud');
+    const ua = extractUAFeatures('Mozilla/5.0 Puppeteer/1.0');
+    expect(ua).toContain('automation');
+  });
+});