@paywalls-net/filter 1.0.6 → 1.1.1

package/package.json

@@ -3,7 +3,7 @@
   "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
   "author": "paywalls.net",
   "license": "MIT",
-  "version": "1.0.6",
+  "version": "1.1.1",
   "publishConfig": {
     "access": "public"
   },

package/src/index.js

@@ -3,7 +3,7 @@
  * filters bot-like requests by using paywalls.net authorization services.
  */
 
-import { classifyUserAgent } from './user-agent-classification.js';
+import { classifyUserAgent, loadAgentPatterns } from './user-agent-classification.js';
 
 async function logAccess(cfg, request, access) {
     // Separate html from the status in the access object.
@@ -69,7 +69,7 @@ async function checkAgentStatus(cfg, request) {
         };
     }
 
-    const agentInfo = classifyUserAgent(userAgent);
+    const agentInfo = await classifyUserAgent(cfg, userAgent);
 
     const body = JSON.stringify({
         account_id: cfg.paywallsPublisherId,
@@ -129,14 +129,14 @@ function isTestBot(request) {
     const uaParam = url.searchParams.get("user-agent");
     return uaParam && uaParam.includes("bot");
 }
-function isPaywallsKnownBot(request) {
+async function isPaywallsKnownBot(cfg, request) {
     const userAgent = request.headers.get("User-Agent");
-    const uaClassification = classifyUserAgent(userAgent);
+    const uaClassification = await classifyUserAgent(cfg, userAgent);
     return uaClassification.operator && uaClassification.agent;
 }
 
-function isRecognizedBot(request) {
-    return isFastlyKnownBot(request) || isCloudflareKnownBot(request) || isTestBot(request) || isPaywallsKnownBot(request);
+async function isRecognizedBot(cfg, request) {
+    return isFastlyKnownBot(request) || isCloudflareKnownBot(request) || isTestBot(request) || await isPaywallsKnownBot(cfg, request);
 }
 
 
@@ -172,8 +172,9 @@ async function cloudflare(config = null) {
             paywallsAPIKey: env.PAYWALLS_CLOUD_API_KEY,
             paywallsPublisherId: env.PAYWALLS_PUBLISHER_ID
         };
+        await loadAgentPatterns(paywallsConfig);
 
-        if (isRecognizedBot(request)) {
+        if (isRecognizedBot(paywallsConfig, request)) {
             const authz = await checkAgentStatus(paywallsConfig, request);
 
             ctx.waitUntil(logAccess(paywallsConfig, request, authz));
@@ -196,9 +197,10 @@ async function fastly(config) {
         paywallsAPIKey: config.get('PAYWALLS_API_KEY'),
         paywallsPublisherId: config.get('PAYWALLS_PUBLISHER_ID')
     };
+    await loadAgentPatterns(paywallsConfig);
 
     return async function handle(request) {
-        if (isRecognizedBot(request)) {
+        if (isRecognizedBot(paywallsConfig,request)) {
             const authz = await checkAgentStatus(paywallsConfig, request);
 
             await logAccess(paywallsConfig, request, authz);
@@ -220,6 +222,7 @@ async function fastly(config) {
  * @returns {Function} - The handler function for the specified CDN.
  */
 export async function init(cdn, config = {}) {
+
     switch (cdn.toLowerCase()) {
         case 'cloudflare':
             return await cloudflare(config);

package/src/user-agent-classification.js

@@ -1,200 +1,62 @@
 import { UAParser } from 'ua-parser-js';
 
-const userAgentPatterns = [
-    {
-        operator: 'Anthropic',
-        agent: 'ClaudeBot',
-        usage: ['ai_training'],
-        user_initiated: 'no',
-        patterns: [/ClaudeBot/, /anthropic-ai/]
-    },
-    {
-        operator: 'Anthropic',
-        agent: 'Claude-User',
-        usage: ['ai_chat'],
-        user_initiated: 'yes',
-        patterns: [/Claude-User/]
-    },
-    {
-        operator: 'Anthropic',
-        agent: 'Claude-SearchBot',
-        usage: ['ai_indexing'],
-        user_initiated: 'maybe',
-        patterns: [/Claude-SearchBot/]
-    },
+// Cache for user agent patterns
+let cachedUserAgentPatterns = null;
+let cacheTimestamp = null;
+const CACHE_DURATION = 60 * 60 * 1000; // 1 hour
 
-    {
-        operator: 'Lumar',
-        agent: 'DeepCrawl',
-        usage: ['webmaster tools'],
-        user_initiated: 'no',
-        patterns: [/deepcrawl.com/]
-    },
-
-    {
-        operator: 'Google',
-        agent: 'Googlebot',
-        usage: ['search_indexing','ai_training'],
-        user_initiated: 'maybe',
-        patterns: [/Googlebot/]
-    },
-    {
-        operator: 'Google',
-        agent: 'Gemini-Deep-Research',
-        usage: ['ai_agents'],
-        user_initiated: 'maybe',
-        patterns: [/Gemini-Deep-Research/]
-    }, 
-    {
-        operator: 'Google',
-        agent: 'Google-Extended',
-        usage: ['ai_training'],
-        usage_prefs_only: true,
-        user_initiated: 'no',
-    },
-    {
-        operator: 'Google',
-        agent: 'Googlebot-News',
-        usage: ['Google News'],
-        usage_prefs_only: true,
-        user_initiated: 'no'
-    },    
-    {
-        operator: 'Google',
-        agent: 'Googlebot-Image',
-        usage: ['image indexing'],
-        user_initiated: 'no',
-        patterns: [/Googlebot-Image/]
-    },
-    {
-        operator: 'Google',
-        agent: 'Google-Site-Verification',
-        usage: ['site verification'],
-        user_initiated: 'no',
-        patterns: [/Google-Site-Verification/]
-    },
-    {
-        operator: 'Google',
-        agent: 'Google Web Preview',
-        usage: ['web preview'],
-        user_initiated: 'no',
-        patterns: [/Google Web Preview/]
-    },
-    {
-        operator: 'Google',
-        agent: 'Googlebot-Video',
-        usage: ['video indexing'],
-        user_initiated: 'no',
-        patterns: [/Googlebot-Video/]
-    },
-    {
-        operator: 'Google',
-        agent: 'FeedFetcher-Google',
-        usage: ['Feed crawling'],
-        user_initiated: 'yes',
-        patterns: [/FeedFetcher-Google/]
-    },
-
-    {
-        operator: 'OpenAI',
-        agent: 'GPTBot',
-        usage: ['ai_training'],
-        user_initiated: 'no',
-        patterns: [/GPTBot/]
-    },
-    {
-        operator: 'OpenAI',
-        agent: 'OAI-SearchBot',
-        usage: ['ai_indexing'],
-        user_initiated: 'no',
-        patterns: [/OAI-SearchBot/]
-    },
-    {
-        operator: 'OpenAI',
-        agent: 'ChatGPT-User',
-        usage: ['ai_chat'],
-        user_initiated: 'yes',
-        patterns: [/ChatGPT-User/]
-    },
+/**
+ * Fetch user agent patterns from the API and cache them.
+ * @returns {Promise<Array>} The user agent patterns.
+ */
+export async function loadAgentPatterns(cfg) {
+    const now = Date.now();
 
-   
-    {
-        operator: 'Meta',
-        agent: 'facebookexternalhit',
-        usage: ['content sharing'],
-        user_initiated: 'no',
-        patterns: [/facebookexternalhit/]
-    },
-    {
-        operator: 'Meta',
-        agent: 'meta-externalagent',
-        usage: ['ai_training'],
-        user_initiated: 'no',
-        patterns: [/meta-externalagent/]
-    },
-    {
-        operator: 'Meta',
-        agent: 'meta-externalfetcher',
-        usage: ['web preview'],
-        user_initiated: 'no',
-        patterns: [/meta-externalfetcher/]
-    },
+    // Return cached patterns if still valid
+    if (cachedUserAgentPatterns && (now - cacheTimestamp < CACHE_DURATION)) {
+        return cachedUserAgentPatterns;
+    }
 
-    {
-        operator: 'Perplexity',
-        agent: 'Perplexity-User',
-        usage: ['ai_chat'],
-        user_initiated: 'yes',
-        patterns: [/Perplexity-User/]
-    },
-    {
-        operator: 'Perplexity',
-        agent: 'PerplexityBot',
-        usage: ['ai_indexing'],
-        user_initiated: 'maybe',
-        patterns: [/PerplexityBot/]
-    },
-    
-    {
-        operator: 'Cohere',
-        agent: 'cohere-ai',
-        usage: ['ai_training'],
-        user_initiated: 'no',
-        patterns: [/cohere-ai/i]
-    },
+    try {
+        const response = await fetch(`${cfg.paywallsAPIHost}/api/filter/agents/metadata`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${cfg.paywallsAPIKey}`
+            }
+        });
 
-    {
-        operator: 'Bing',
-        agent: 'BingBot',
-        usage: ['search_indexing','ai_indexing'],
-        user_initiated: 'maybe',
-        patterns: [/bingbot/i, /BingPreview/]
-    },
+        if (!response.ok) {
+            throw new Error(`Failed to fetch user agent patterns: ${response.status} ${response.statusText}`);
+        }
 
-    {
-        operator: 'Microsoft',
-        agent: 'BF-DirectLine',
-        usage: ['Bot Framework SDK'],
-        user_initiated: 'no',
-        patterns: [/BF-DirectLine/]
+        cachedUserAgentPatterns = await response.json();
+        cacheTimestamp = now;
+        return cachedUserAgentPatterns;
+    } catch (error) {
+        console.error('Failed to fetch user agent patterns:', error);
+        throw new Error('Could not load user agent patterns');
     }
-];
+}
 
 /**
- * Classifies the user agent string based on predefined patterns.
+ * Classifies the user agent string based on fetched patterns.
+ * @param {Object} cfg - Configuration object containing API host details.
  * @param {string} userAgent - The user agent string to classify.
- * @returns {Object} An object containing the browser, OS, operator, usage, and user_initiated status.
+ * @returns {Promise<Object>} An object containing the browser, OS, operator, usage, and user_initiated status.
  */
-export function classifyUserAgent(userAgent) {
+export async function classifyUserAgent(cfg, userAgent) {
     const parsedUA = new UAParser(userAgent).getResult();
 
     const browser = parsedUA.browser.name || 'Unknown';
     const os = parsedUA.os.name || 'Unknown';
 
+    const userAgentPatterns = await loadAgentPatterns(cfg);
+
     for (const config of userAgentPatterns) {
         if (!config.patterns) continue;
         for (const pattern of config.patterns) {
-            if (pattern.test(userAgent)) {
+            if (new RegExp(pattern).test(userAgent)) {
                 return {
                     operator: config.operator,
                     agent: config.agent || browser,