@paywalls-net/filter 1.0.5

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 paywalls-net
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,28 @@
+# paywalls-net/client
+
+SDK for integrating paywalls.net authorization services with CDN or edge environments.
+
+## Install
+
+```bash
+npm install @paywalls-net/client
+```
+
+## Environment Variables
+- `PAYWALLS_PUBLISHER_ID`: The unique identifier for the publisher using Paywalls.net services.
+- `PAYWALLS_CLOUD_API_HOST`: The host for the Paywalls.net API. eg `https://cloud-api.paywalls.net`.
+- `PAYWALLS_CLOUD_API_KEY`: The API key for accessing Paywalls.net services. NOTE: This key should be treated like a password and kept secret and stored in a secure secrets vault or environment variable.
+
+## Usage
+```javascript
+import { init } from '@paywalls-net/client';
+
+const handleRequest = await init('cloudflare');
+
+export default {
+  async fetch(request, env, ctx) {
+    return handleRequest(request, env, ctx);
+  }
+};
+```
+

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@paywalls-net/filter",
+  "description": "Client SDK for integrating paywalls.net bot filtering and authorization services into your server or CDN.",
+  "author": "paywalls.net",
+  "license": "MIT",
+  "version": "1.0.5",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/paywalls-net/filter.git"
+  },
+  "type": "module",
+  "main": "index.js",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "ua-parser-js": "^2.0.4"
+  }
+}

package/src/index.js

@@ -0,0 +1,231 @@
+/**
+ * Example publisher-hosted client code for a Cloudflare Worker that
+ * filters bot-like requests by using paywalls.net authorization services.
+ */
+
+import { classifyUserAgent } from './user-agent-classification.js';
+
+async function logAccess(cfg, request, access) {
+    // Separate html from the status in the access object.
+    const { response, ...status } = access;
+
+    // Get all headers as a plain object (name-value pairs)
+    let headers = {};
+    for (const [key, value] of request.headers.entries()) {
+        headers[key] = value;
+    }
+    const url = new URL(request.url);
+    let body = {
+        account_id: cfg.paywallsPublisherId,
+        status: status,
+        method: request.method,
+        hostname: url.hostname,
+        resource: url.pathname + url.search,
+        user_agent: headers['user-agent'],
+        headers: headers
+    };
+
+    const logResponse = await fetch(`${cfg.paywallsAPIHost}/api/filter/access/logs`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${cfg.paywallsAPIKey}`
+        },
+        body: JSON.stringify(body)
+    });
+    if (!logResponse.ok) {
+        console.error(`Error logging access: ${logResponse.status} ${logResponse.statusText}`);
+        // Optionally, you can handle the error here, e.g., retry or log to a different service
+    }
+}
+
+/**
+ * Typedef for AgentStatus
+ * @typedef {Object} AgentStatus
+ * @property {string} access - Whether access is granted (allow, deny)
+ * @property {string} reason - The reason for the status (e.g., missing_user_agent, unknown_error)
+ * @property {object} response - The response object
+ * @property {number} response.code - The HTTP status code to be sent to the client
+ * @property {object} response.headers - The headers to be sent to the client
+ * @property {string} response.html - The HTML response to be sent to the client
+ */
+
+/**
+ * 
+ * @param {*} cfg 
+ * @param {Request} request - The incoming request object
+ * @returns {Promise<AgentStatus>} - The authorization for this agent
+ */
+async function checkAgentStatus(cfg, request) {
+    const userAgent = request.headers.get("User-Agent");
+    const token = getAcessToken(request);
+
+    if (!userAgent) {
+        console.error("Missing user-agent");
+        return {
+            access: 'deny',
+            reason: 'missing_user_agent',
+            response: { code: 401, html: "Unauthorized access." }
+        };
+    }
+
+    const agentInfo = classifyUserAgent(userAgent);
+
+    const body = JSON.stringify({
+        account_id: cfg.paywallsPublisherId,
+        operator: agentInfo.operator,
+        agent: agentInfo.agent,
+        token: token
+    });
+
+    const response = await fetch(`${cfg.paywallsAPIHost}/api/filter/agents/auth`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${cfg.paywallsAPIKey}`
+        },
+        body: body
+    });
+
+    if (!response.ok) {
+        console.error(`Failed to fetch agent auth: ${response.status} ${response.statusText}`);
+        return {
+            access: 'deny',
+            reason: 'unknown_error',
+            response: { code: 502, html: "Bad Gateway." }
+        };
+    }
+
+    return response.json();
+}
+
+function getAcessToken(request) {
+    const authHeader = request.headers.get("Authorization");
+    if (!authHeader) {
+        return null;
+    }
+
+    const token = authHeader.split(" ")[1];
+    return token;
+}
+
+function isFastlyKnownBot(request) {
+    const botScore = request.headers.get("X-Fastly-Bot-Score");
+    const isKnownBot = request.headers.get("X-Fastly-Known-Bot");
+
+    return isKnownBot === "true" || (botScore && parseInt(botScore) < 30);
+}
+
+function isCloudflareKnownBot(request) {
+    const cf = request.cf || {};
+    const botScore = cf.botManagement?.score;
+    const isKnownBot = cf.botManagement?.verifiedBot;
+    return isKnownBot || (botScore !== undefined && botScore < 30);
+}
+
+function isTestBot(request) {
+    // check if the URL has a query parameter to always test as a bot
+    const url = new URL(request.url);
+    const uaParam = url.searchParams.get("user-agent");
+    return uaParam && uaParam.includes("bot");
+}
+function isPaywallsKnownBot(request) {
+    const userAgent = request.headers.get("User-Agent");
+    const uaClassification = classifyUserAgent(userAgent);
+    return uaClassification.operator && uaClassification.agent;
+}
+
+function isRecognizedBot(request) {
+    return isFastlyKnownBot(request) || isCloudflareKnownBot(request) || isTestBot(request) || isPaywallsKnownBot(request);
+}
+
+
+function sendResponse(authz) {
+    let headers = {
+        "Content-Type": "text/html",
+    };
+
+    if (authz.response?.headers) {
+        for (const [key, value] of Object.entries(authz.response.headers)) {
+            headers[key] = value;
+        }
+    }
+
+    return new Response(authz.response?.html || "Payment required.", {
+        status: authz.response?.code || 402,
+        headers: headers
+    });
+}
+
+/**
+ * Detect AI Bot and authorize it using paywalls.net.
+ * @param {Request} request 
+ * @param {*} env 
+ * @param {*} ctx 
+ * @returns 
+ */
+async function cloudflare(config = null) {
+
+    return async function handle(request, env, ctx) {
+        const paywallsConfig = {
+            paywallsAPIHost: env.PAYWALLS_CLOUD_API_HOST,
+            paywallsAPIKey: env.PAYWALLS_CLOUD_API_KEY,
+            paywallsPublisherId: env.PAYWALLS_PUBLISHER_ID
+        };
+
+        if (isRecognizedBot(request)) {
+            const authz = await checkAgentStatus(paywallsConfig, request);
+
+            ctx.waitUntil(logAccess(paywallsConfig, request, authz));
+
+            if (authz.access === 'deny') {
+                return sendResponse(authz);
+            } else {
+                console.log("Bot-like request allowed. Proceeding to origin/CDN.");
+            }
+        }
+
+        return fetch(request); // Proceed to origin/CDN
+    };
+}
+
+
+async function fastly(config) {
+    const paywallsConfig = {
+        paywallsAPIHost: config.get('PAYWALLS_CLOUD_API_HOST'),
+        paywallsAPIKey: config.get('PAYWALLS_API_KEY'),
+        paywallsPublisherId: config.get('PAYWALLS_PUBLISHER_ID')
+    };
+
+    return async function handle(request) {
+        if (isRecognizedBot(request)) {
+            const authz = await checkAgentStatus(paywallsConfig, request);
+
+            await logAccess(paywallsConfig, request, authz);
+
+            if (authz.access === 'deny') {
+                return sendResponse(authz);
+            }
+        }
+
+        return fetch(request, { backend: 'origin' });
+    };
+}
+
+
+
+/**
+ * Initializes the appropriate handler based on the CDN.
+ * @param {string} cdn - The name of the CDN (e.g., 'cloudflare', 'fastly', 'cloudfront').
+ * @returns {Function} - The handler function for the specified CDN.
+ */
+export async function init(cdn, config = {}) {
+    switch (cdn.toLowerCase()) {
+        case 'cloudflare':
+            return await cloudflare(config);
+        case 'fastly':
+            return await fastly(config);
+        default:
+            throw new Error(`Unsupported CDN: ${cdn}`);
+    }
+}

package/src/user-agent-classification.js

@@ -0,0 +1,214 @@
+import { UAParser } from 'ua-parser-js';
+
+const userAgentPatterns = [
+    {
+        operator: 'Anthropic',
+        agent: 'ClaudeBot',
+        usage: ['ai_training'],
+        user_initiated: 'no',
+        patterns: [/ClaudeBot/, /anthropic-ai/]
+    },
+    {
+        operator: 'Anthropic',
+        agent: 'Claude-User',
+        usage: ['ai_chat'],
+        user_initiated: 'yes',
+        patterns: [/Claude-User/]
+    },
+    {
+        operator: 'Anthropic',
+        agent: 'Claude-SearchBot',
+        usage: ['ai_indexing'],
+        user_initiated: 'maybe',
+        patterns: [/Claude-SearchBot/]
+    },
+
+    {
+        operator: 'Lumar',
+        agent: 'DeepCrawl',
+        usage: ['webmaster tools'],
+        user_initiated: 'no',
+        patterns: [/deepcrawl.com/]
+    },
+
+    {
+        operator: 'Google',
+        agent: 'Googlebot',
+        usage: ['search_indexing','ai_training'],
+        user_initiated: 'maybe',
+        patterns: [/Googlebot/]
+    },
+    {
+        operator: 'Google',
+        agent: 'Gemini-Deep-Research',
+        usage: ['ai_agents'],
+        user_initiated: 'maybe',
+        patterns: [/Gemini-Deep-Research/]
+    }, 
+    {
+        operator: 'Google',
+        agent: 'Google-Extended',
+        usage: ['ai_training'],
+        usage_prefs_only: true,
+        user_initiated: 'no',
+    },
+    {
+        operator: 'Google',
+        agent: 'Googlebot-News',
+        usage: ['Google News'],
+        usage_prefs_only: true,
+        user_initiated: 'no'
+    },    
+    {
+        operator: 'Google',
+        agent: 'Googlebot-Image',
+        usage: ['image indexing'],
+        user_initiated: 'no',
+        patterns: [/Googlebot-Image/]
+    },
+    {
+        operator: 'Google',
+        agent: 'Google-Site-Verification',
+        usage: ['site verification'],
+        user_initiated: 'no',
+        patterns: [/Google-Site-Verification/]
+    },
+    {
+        operator: 'Google',
+        agent: 'Google Web Preview',
+        usage: ['web preview'],
+        user_initiated: 'no',
+        patterns: [/Google Web Preview/]
+    },
+    {
+        operator: 'Google',
+        agent: 'Googlebot-Video',
+        usage: ['video indexing'],
+        user_initiated: 'no',
+        patterns: [/Googlebot-Video/]
+    },
+    {
+        operator: 'Google',
+        agent: 'FeedFetcher-Google',
+        usage: ['Feed crawling'],
+        user_initiated: 'yes',
+        patterns: [/FeedFetcher-Google/]
+    },
+
+    {
+        operator: 'OpenAI',
+        agent: 'GPTBot',
+        usage: ['ai_training'],
+        user_initiated: 'no',
+        patterns: [/GPTBot/]
+    },
+    {
+        operator: 'OpenAI',
+        agent: 'OAI-SearchBot',
+        usage: ['ai_indexing'],
+        user_initiated: 'no',
+        patterns: [/OAI-SearchBot/]
+    },
+    {
+        operator: 'OpenAI',
+        agent: 'ChatGPT-User',
+        usage: ['ai_chat'],
+        user_initiated: 'yes',
+        patterns: [/ChatGPT-User/]
+    },
+
+   
+    {
+        operator: 'Meta',
+        agent: 'facebookexternalhit',
+        usage: ['content sharing'],
+        user_initiated: 'no',
+        patterns: [/facebookexternalhit/]
+    },
+    {
+        operator: 'Meta',
+        agent: 'meta-externalagent',
+        usage: ['ai_training'],
+        user_initiated: 'no',
+        patterns: [/meta-externalagent/]
+    },
+    {
+        operator: 'Meta',
+        agent: 'meta-externalfetcher',
+        usage: ['web preview'],
+        user_initiated: 'no',
+        patterns: [/meta-externalfetcher/]
+    },
+
+    {
+        operator: 'Perplexity',
+        agent: 'Perplexity-User',
+        usage: ['ai_chat'],
+        user_initiated: 'yes',
+        patterns: [/Perplexity-User/]
+    },
+    {
+        operator: 'Perplexity',
+        agent: 'PerplexityBot',
+        usage: ['ai_indexing'],
+        user_initiated: 'maybe',
+        patterns: [/PerplexityBot/]
+    },
+    
+    {
+        operator: 'Cohere',
+        agent: 'cohere-ai',
+        usage: ['ai_training'],
+        user_initiated: 'no',
+        patterns: [/cohere-ai/i]
+    },
+
+    {
+        operator: 'Bing',
+        agent: 'BingBot',
+        usage: ['search_indexing','ai_indexing'],
+        user_initiated: 'maybe',
+        patterns: [/bingbot/i, /BingPreview/]
+    },
+
+    {
+        operator: 'Microsoft',
+        agent: 'BF-DirectLine',
+        usage: ['Bot Framework SDK'],
+        user_initiated: 'no',
+        patterns: [/BF-DirectLine/]
+    }
+];
+
+/**
+ * Classifies the user agent string based on predefined patterns.
+ * @param {string} userAgent - The user agent string to classify.
+ * @returns {Object} An object containing the browser, OS, operator, usage, and user_initiated status.
+ */
+export function classifyUserAgent(userAgent) {
+    const parsedUA = new UAParser(userAgent).getResult();
+
+    const browser = parsedUA.browser.name || 'Unknown';
+    const os = parsedUA.os.name || 'Unknown';
+
+    for (const config of userAgentPatterns) {
+        if (!config.patterns) continue;
+        for (const pattern of config.patterns) {
+            if (pattern.test(userAgent)) {
+                return {
+                    operator: config.operator,
+                    agent: config.agent || browser,
+                    usage: config.usage,
+                    user_initiated: config.user_initiated,
+                    browser,
+                    os,
+                };
+            }
+        }
+    }
+
+    return {
+        browser,
+        os
+    };
+}