@paywalls-net/filter 1.3.0 → 1.3.2

package/package.json

@@ -3,7 +3,7 @@
   "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
   "author": "paywalls.net",
   "license": "MIT",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "publishConfig": {
     "access": "public"
   },

package/src/index.js

@@ -51,6 +51,71 @@ function getAllHeaders(request) {
     return headers;
 }
 
+/**
+ * Check if the request is for a VAI endpoint (vai.json or vai.js)
+ * @param {Request} request - The incoming request
+ * @param {string} vaiPath - The path prefix for VAI endpoints (default: '/pw')
+ * @returns {boolean} - True if this is a VAI endpoint request
+ */
+function isVAIRequest(request, vaiPath = '/pw') {
+    try {
+        const url = new URL(request.url || `http://host${request.uri || ''}`);
+        const pathname = url.pathname;
+        return pathname === `${vaiPath}/vai.json` || pathname === `${vaiPath}/vai.js`;
+    } catch (err) {
+        return false;
+    }
+}
+
+/**
+ * Proxy VAI requests to the cloud-api service
+ * @param {Object} cfg - Configuration object with paywallsAPIHost and paywallsAPIKey
+ * @param {Request} request - The incoming request
+ * @returns {Promise<Response>} - The proxied response from cloud-api
+ */
+async function proxyVAIRequest(cfg, request) {
+    try {
+        const url = new URL(request.url || `http://host${request.uri || ''}`);
+        const isJson = url.pathname.endsWith('/vai.json');
+        const cloudApiPath = isJson ? '/pw/vai.json' : '/pw/vai.js';
+        
+        // Get all request headers
+        const headers = getAllHeaders(request);
+        
+        // Build forwarding headers
+        const forwardHeaders = {
+            'User-Agent': headers['user-agent'] || sdkUserAgent,
+            'Authorization': `Bearer ${cfg.paywallsAPIKey}`
+        };
+        
+        // Add forwarding headers if available
+        if (headers['x-forwarded-for']) {
+            forwardHeaders['X-Forwarded-For'] = headers['x-forwarded-for'];
+        } else if (headers['cf-connecting-ip']) {
+            forwardHeaders['X-Forwarded-For'] = headers['cf-connecting-ip'];
+        }
+        
+        if (headers['host']) {
+            forwardHeaders['X-Original-Host'] = headers['host'];
+        }
+        
+        // Forward request to cloud-api
+        const response = await fetch(`${cfg.paywallsAPIHost}${cloudApiPath}`, {
+            method: 'GET',
+            headers: forwardHeaders
+        });
+        
+        if (!response.ok) {
+            console.error(`VAI proxy error: ${response.status} ${response.statusText}`);
+        }
+        
+        return response;
+    } catch (err) {
+        console.error(`Error proxying VAI request: ${err.message}`);
+        return new Response('Internal Server Error', { status: 500 });
+    }
+}
+
 async function logAccess(cfg, request, access) {
     // Separate html from the status in the access object.
     const { response, ...status } = access;
@@ -263,8 +328,15 @@ async function cloudflare(config = null) {
         const paywallsConfig = {
             paywallsAPIHost: env.PAYWALLS_CLOUD_API_HOST || PAYWALLS_CLOUD_API_HOST,
             paywallsAPIKey: env.PAYWALLS_CLOUD_API_KEY,
-            paywallsPublisherId: env.PAYWALLS_PUBLISHER_ID
+            paywallsPublisherId: env.PAYWALLS_PUBLISHER_ID,
+            vaiPath: env.PAYWALLS_VAI_PATH || '/pw'
         };
+        
+        // Check if this is a VAI endpoint request and proxy it
+        if (isVAIRequest(request, paywallsConfig.vaiPath)) {
+            return await proxyVAIRequest(paywallsConfig, request);
+        }
+        
         await loadAgentPatterns(paywallsConfig);
 
         if (await isRecognizedBot(paywallsConfig, request)) {
@@ -288,8 +360,14 @@ async function fastly() {
         const paywallsConfig = {
             paywallsAPIHost: config.get('PAYWALLS_CLOUD_API_HOST') || PAYWALLS_CLOUD_API_HOST,
             paywallsAPIKey: config.get('PAYWALLS_API_KEY'),
-            paywallsPublisherId: config.get('PAYWALLS_PUBLISHER_ID')
+            paywallsPublisherId: config.get('PAYWALLS_PUBLISHER_ID'),
+            vaiPath: config.get('PAYWALLS_VAI_PATH') || '/pw'
         };
+        
+        // Check if this is a VAI endpoint request and proxy it
+        if (isVAIRequest(request, paywallsConfig.vaiPath)) {
+            return await proxyVAIRequest(paywallsConfig, request);
+        }
 
         await loadAgentPatterns(paywallsConfig);
 
@@ -304,6 +382,34 @@ async function fastly() {
         }
     };
 }
+/**
+ * Convert a standard Response to CloudFront format
+ * @param {Response} response - Standard fetch Response object
+ * @returns {Promise<Object>} - CloudFront-formatted response
+ */
+async function responseToCloudFront(response) {
+    const headers = {};
+    
+    // Convert response headers to CloudFront format
+    for (const [key, value] of response.headers.entries()) {
+        headers[key.toLowerCase()] = [
+            {
+                key: key,
+                value: value
+            }
+        ];
+    }
+    
+    const body = await response.text();
+    
+    return {
+        status: response.status,
+        statusDescription: response.statusText || 'OK',
+        headers: headers,
+        body: body
+    };
+}
+
 /**
  * Adapt to CloudFront format
  * Lambda@Edge events see https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/lambda-event-structure.html#lambda-event-structure-request
@@ -336,13 +442,21 @@ async function cloudfront(config) {
     const paywallsConfig = {
         paywallsAPIHost: config.PAYWALLS_CLOUD_API_HOST || PAYWALLS_CLOUD_API_HOST,
         paywallsAPIKey: config.PAYWALLS_API_KEY,
-        paywallsPublisherId: config.PAYWALLS_PUBLISHER_ID
+        paywallsPublisherId: config.PAYWALLS_PUBLISHER_ID,
+        vaiPath: config.PAYWALLS_VAI_PATH || '/pw'
     };
     await loadAgentPatterns(paywallsConfig);
 
     return async function handle(event, ctx) {
         let request = event.Records[0].cf.request;
         request = requestShim(request);
+        
+        // Check if this is a VAI endpoint request and proxy it
+        if (isVAIRequest(request, paywallsConfig.vaiPath)) {
+            const response = await proxyVAIRequest(paywallsConfig, request);
+            return await responseToCloudFront(response);
+        }
+        
         if (await isRecognizedBot(paywallsConfig, request)) {
             const authz = await checkAgentStatus(paywallsConfig, request);
 

package/src/user-agent-classification.js

@@ -5,6 +5,39 @@ let cachedUserAgentPatterns = null;
 let cacheTimestamp = null;
 const CACHE_DURATION = 60 * 60 * 1000; // 1 hour
 
+// Cache for user agent classifications
+// 
+// CACHE STRATEGY CONSIDERATIONS:
+// 
+// Current approach: Raw user-agent string as cache key
+// - Pro: No parsing overhead before cache lookup
+// - Pro: Exact matches are very fast
+// - Con: User-agents with minor version differences create separate cache entries
+// - Con: Cache could grow large with many unique UAs (especially browser traffic)
+//
+// Alternative approaches to consider:
+// 1. Normalized keys (e.g., browser name + major version + OS)
+//    - Would improve hit rate and reduce memory
+//    - But adds parsing cost before every cache check
+//    - Risk: Might miss pattern-specific matches if patterns are version-sensitive
+//
+// 2. LRU cache with size limit
+//    - Bounds memory usage
+//    - Evicts least-recently-used entries
+//    - Good if traffic patterns are consistent
+//
+// 3. Separate caches for bots vs browsers
+//    - Bot UAs are typically more stable (better cache hit rate)
+//    - Browser UAs change frequently with versions (lower hit rate)
+//    - Could optimize each differently
+//
+// Decision: Start with raw UA keys until we have production metrics showing:
+// - Actual cache size growth
+// - Cache hit rates
+// - Memory pressure
+// Then optimize based on data rather than speculation.
+let classificationCache = new Map();
+
 /**
  * Fetch user agent patterns from the API and cache them.
  * @returns {Promise<Array>} The user agent patterns.
@@ -39,6 +72,10 @@ export async function loadAgentPatterns(cfg) {
         }));
 
         cacheTimestamp = now;
+        
+        // Clear classification cache when patterns are refreshed
+        classificationCache.clear();
+        
         return cachedUserAgentPatterns;
     } catch (error) {
         console.error('Error loading agent patterns:', error);
@@ -53,6 +90,14 @@ export async function loadAgentPatterns(cfg) {
  * @returns {Promise<Object>} An object containing the browser, OS, operator, usage, and user_initiated status.
  */
 export async function classifyUserAgent(cfg, userAgent) {
+    // Check classification cache first (single lookup is more efficient than has + get)
+    const cached = classificationCache.get(userAgent);
+    if (cached !== undefined) {
+        console.log(`User agent classification cache hit for: ${userAgent}`);
+        return cached;
+    }
+    console.log(`User agent classification cache miss for: ${userAgent}`);
+
     const parsedUA = new UAParser(userAgent).getResult();
 
     const browser = parsedUA.browser.name || 'Unknown';
@@ -64,7 +109,7 @@ export async function classifyUserAgent(cfg, userAgent) {
         if (!config.patterns) continue;
         for (const pattern of config.patterns) {
             if (new RegExp(pattern).test(userAgent)) {         
-                return {
+                const result = {
                     operator: config.operator,
                     agent: config.agent || browser,
                     usage: config.usage,
@@ -72,12 +117,18 @@ export async function classifyUserAgent(cfg, userAgent) {
                     browser,
                     os,
                 };
+                // Cache the classification result
+                classificationCache.set(userAgent, result);
+                return result;
             }
         }
     }
 
-    return {
+    const result = {
         browser,
         os
     };
+    // Cache the default classification
+    classificationCache.set(userAgent, result);
+    return result;
 }