@profullstack/x402-gateway 0.7.0 → 0.8.0

package/README.md

@@ -89,7 +89,11 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `header` | `x-crawl-pass` | where the pass goes; `Authorization: Bearer` works too |
 | `path` | `/crawl` | the sales page |
 | `openPaths` | `[]` | extra paths a refused crawler may read (`robots.txt`, the sales page, `security.txt` and `.well-known/` always are) |
-| `isPaidAgent` | training list | `(userAgent) => boolean` |
+| `isPaidAgent` | training + scraper + headless lists | `(userAgent) => boolean` |
+| `scrapers` | `SCRAPER_AGENTS` | SEO resellers (AhrefsBot, SemrushBot, MJ12bot, DataForSeoBot, ...): charged and refused in robots.txt |
+| `headless` | `HEADLESS_AGENTS` | headless browsers that name themselves (Lightpanda, HeadlessChrome, PhantomJS): charged |
+| `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
+| `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |
 | `secret` | the CoinPay key | pass signing secret |
 | `page` | built in | `(ctx) => html` |

package/index.d.ts

@@ -68,6 +68,14 @@ export interface GatewayOptions {
   denyCidrs?: string[];
   /** Charge a request that claims "Chrome/…" but lacks the Sec-Fetch-Mode header every Chromium sends. Default false. */
   chargeSpoofedBrowsers?: boolean;
+  /** Charge a desktop browser string ~2 years behind its auto-updates (a scraper rotation pool). Default true. */
+  chargeStaleBrowsers?: boolean;
+  /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
+  scrapers?: string[];
+  /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */
+  headless?: string[];
+  /** Search crawlers never metered when their address matches their name. Default WELCOME_CRAWLERS; false disables. */
+  welcome?: WelcomeCrawler[] | false;
   /** Requests never charged, e.g. ones carrying a signed-in cookie. */
   exempt?: (request: Request) => boolean;
   /** Pass signing secret. Defaults to the CoinPay key. */
@@ -191,6 +199,10 @@ export interface Gateway {
   passFrom: (request: Request) => string | null;
   /** Whether that token opens the site: a live day pass this gateway minted, or a valid, unrevoked Power Key. */
   verifyPass: (token: string | null) => Promise<boolean>;
+  /** Whether the gateway's own rules charge this request (lists, spoofed or stale browser). */
+  isPaid: (request: Request) => boolean;
+  /** Whether this is a welcome search crawler whose address matches its name. */
+  isWelcome: (request: Request) => boolean;
   /** The x402 offer this gateway would make, without answering a request. */
   offer: (days?: number) => Offer;
   /** The full 402 body around that offer: the offer plus the pass terms. */
@@ -201,6 +213,13 @@ export function createGateway(options: GatewayOptions): Gateway;
 export function wantsHtml(accept?: string | null): boolean;
 
 export const TRAINING_AGENTS: string[];
+export const SCRAPER_AGENTS: string[];
+export const HEADLESS_AGENTS: string[];
+export interface WelcomeCrawler {
+  token: string;
+  cidrs: string[];
+}
+export const WELCOME_CRAWLERS: WelcomeCrawler[];
 export const RETRIEVAL_AGENTS: string[];
 export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean;
 
@@ -213,6 +232,9 @@ export function compileCidrs(list?: string[]): Cidr[];
 export function inCidrs(ip: string, compiled: Cidr[]): boolean;
 export function clientIp(request: Request): string;
 export function isSpoofedBrowser(request: Request): boolean;
+export function isStaleBrowser(userAgent: string, now?: number, maxMajorsBehind?: number): boolean;
+export function currentMajor(browser: 'chrome' | 'firefox', now?: number): number;
+export function welcomeCrawler(request: Request, list: WelcomeCrawler[] | null | undefined): string | null;
 export function renderPage(ctx: PageContext): string;
 
 export function mintPass(args: { secret: string; ref: string | null; expiresAt: number; now?: number }): Promise<{ token: string; expiresAt: number; ref: string | null }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/agents.js

@@ -24,8 +24,35 @@ export const TRAINING_AGENTS = [
   'FacebookBot',
   'Bytespider',
   'Applebot-Extended',
+  // Amazon documents Amazonbot's crawl as one that "may be used to train
+  // Amazon AI models", and it sends no reader back from any of it.
+  'Amazonbot',
 ];
 
+/**
+ * Crawlers that resell what they read and send nobody back: SEO backlink and
+ * keyword databases. They do obey robots.txt, so they are refused there as well
+ * as charged, which is the cheapest outcome of all: they stop asking.
+ */
+export const SCRAPER_AGENTS = [
+  'AhrefsBot',
+  'SemrushBot',
+  'MJ12bot',
+  'DataForSeoBot',
+  'DotBot',
+  'BLEXBot',
+  'serpstatbot',
+  'Barkrowler',
+];
+
+/**
+ * Headless browsers that say so. Nobody reads a page through one of these; a
+ * scrape does, and it runs the page's scripts while it is at it. Lightpanda was
+ * 151k requests from 63k addresses on one site in a day (2026-10-19). They do
+ * not read robots.txt, so they are charged, never named there.
+ */
+export const HEADLESS_AGENTS = ['Lightpanda', 'HeadlessChrome', 'PhantomJS'];
+
 /**
  * Retrieval crawlers, named in robots.txt so their operators can see they are
  * welcome. Google-Extended is Google's training token but Google documents it
@@ -57,3 +84,23 @@ export function isTrainingAgent(userAgent = '', agents = TRAINING_AGENTS) {
   if (!ua) return false;
   return lower(agents).some((t) => ua.includes(t));
 }
+
+/**
+ * Search crawlers that are welcome everywhere, with the address ranges their
+ * operators publish. A user agent alone is not enough to spare a caller from
+ * the free allowance -- "Googlebot" is the first string a scraper copies -- so
+ * a crawler is only spared when its address agrees with its name.
+ *
+ * Ranges are the stable blocks of each operator's published list
+ * (developers.google.com/static/search/apis/ipranges/googlebot.json,
+ * bing.com/toolbox/bingbot.json, Apple's 17.0.0.0/8); a crawl from outside them
+ * is simply metered like everyone else, which fails safe.
+ */
+export const WELCOME_CRAWLERS = [
+  { token: 'Googlebot', cidrs: ['66.249.64.0/19', '192.178.4.0/22', '34.100.182.96/28'] },
+  {
+    token: 'bingbot',
+    cidrs: ['40.77.167.0/24', '157.55.39.0/24', '207.46.13.0/24', '52.167.144.0/24', '40.77.188.0/22', '20.15.133.160/27'],
+  },
+  { token: 'Applebot', cidrs: ['17.0.0.0/8'] },
+];

package/src/edge.js

@@ -111,3 +111,97 @@ export function isSpoofedBrowser(request) {
   if (DECLARES_ITSELF.test(ua)) return false;
   return !request.headers.has('sec-fetch-mode');
 }
+
+/* ---------------------------------------------------------- stale strings -- */
+
+/*
+ * Where each browser's version stood on a known day, and how fast it moves:
+ * both ship a major every four weeks. Computing "current" from the date keeps
+ * the check from going stale itself, which a hard-coded number would within a
+ * year.
+ */
+const DAY = 86_400_000;
+const CADENCE = 28 * DAY;
+const CHROME_ANCHOR = { major: 120, at: Date.UTC(2023, 11, 5) };
+const FIREFOX_ANCHOR = { major: 121, at: Date.UTC(2023, 11, 19) };
+
+/** The major a browser that auto-updates would be on at `now`. */
+export function currentMajor(browser, now = Date.now()) {
+  const a = browser === 'firefox' ? FIREFOX_ANCHOR : CHROME_ANCHOR;
+  return a.major + Math.max(0, Math.floor((now - a.at) / CADENCE));
+}
+
+/*
+ * The last version an old operating system can run, and Firefox's extended
+ * support releases. A person stuck on one of these is still a person, so these
+ * are never called stale, however old.
+ */
+const TERMINAL_CHROME = [
+  { major: 109, os: /Windows NT 6\./ }, // Windows 7, 8, 8.1
+  { major: 116, os: /Macintosh/ }, // macOS 10.13, 10.14
+  { major: 128, os: /Macintosh/ }, // macOS 10.15
+  { major: 138, os: /Macintosh/ }, // macOS 11
+];
+const FIREFOX_ESR = [115, 128, 140, 153];
+
+/**
+ * A desktop browser string far older than anything that auto-updates.
+ *
+ * Chrome, Edge and Firefox update themselves every four weeks, so a desktop
+ * "Chrome/120" two years after Chrome 120 shipped is a string copied into a
+ * scraper's rotation pool, not a browser. That is exactly what the swarms do:
+ * on 2026-10-19 one pool of ten strings (Chrome 118-120, Edge 119-120, Firefox
+ * 120-121) walked watchnews.now and r4ck.dev from thousands of residential
+ * addresses, running the page scripts as it went.
+ *
+ * Only desktop strings are judged. Phones and embedded webviews legitimately
+ * sit years behind, and a declared crawler is judged by the lists instead.
+ *
+ * @param {string} ua
+ * @param {number} [now]
+ * @param {number} [maxMajorsBehind=24]  about 22 months of releases
+ */
+export function isStaleBrowser(ua, now = Date.now(), maxMajorsBehind = 24) {
+  const s = String(ua ?? '');
+  if (!/Windows NT|Macintosh|X11; (?:Linux|Ubuntu|Fedora)/.test(s)) return false;
+  if (/Mobile|Android|iPhone|iPad|CrOS/.test(s)) return false;
+  if (DECLARES_ITSELF.test(s)) return false;
+
+  const ff = /\bFirefox\/(\d+)/.exec(s);
+  if (ff) {
+    const major = Number(ff[1]);
+    if (FIREFOX_ESR.includes(major)) return false;
+    return major < currentMajor('firefox', now) - maxMajorsBehind;
+  }
+  const chrome = /\bChrome\/(\d+)/.exec(s);
+  if (chrome) {
+    const major = Number(chrome[1]);
+    if (TERMINAL_CHROME.some((t) => t.major === major && t.os.test(s))) return false;
+    return major < currentMajor('chrome', now) - maxMajorsBehind;
+  }
+  return false;
+}
+
+/* -------------------------------------------------------- welcome crawlers -- */
+
+const COMPILED = new WeakMap();
+
+/**
+ * The welcome crawler a request is, judged by name AND address, or null.
+ *
+ * @param {Request} request
+ * @param {{token: string, cidrs: string[]}[]} list  e.g. WELCOME_CRAWLERS
+ */
+export function welcomeCrawler(request, list) {
+  if (!list?.length) return null;
+  const ua = (request.headers.get('user-agent') ?? '').toLowerCase();
+  if (!ua) return null;
+  const ip = clientIp(request);
+  for (const c of list) {
+    if (!ua.includes(c.token.toLowerCase())) continue;
+    let compiled = COMPILED.get(c);
+    if (!compiled) COMPILED.set(c, (compiled = compileCidrs(c.cidrs)));
+    if (inCidrs(ip, compiled)) return c.token;
+  }
+  return null;
+}

package/src/index.js

@@ -1,6 +1,6 @@
 import { HEADER as POWER_KEY_HEADER, isPowerKey, keyFrom, PROFULLSTACK_KEYS, SHOP_URL, verifyKey } from '@profullstack/keys';
-import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
-import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser } from './edge.js';
+import { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
+import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, isStaleBrowser, welcomeCrawler } from './edge.js';
 import { memoryQuotaStore, meters, normaliseQuota, quotaHeaders, spend } from './quota.js';
 import { renderPage } from './page.js';
 import { mintPass, readPass } from './pass.js';
@@ -16,8 +16,8 @@ import {
   verifyAndSettle,
 } from './x402.js';
 
-export { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
-export { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from './edge.js';
+export { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
+export { clientIp, compileCidrs, currentMajor, inCidrs, isSpoofedBrowser, isStaleBrowser, parseCidr, welcomeCrawler } from './edge.js';
 export { renderPage } from './page.js';
 export { mintPass, readPass } from './pass.js';
 export { robotsTxt } from './robots.js';
@@ -56,6 +56,10 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {(ua: string) => boolean} [options.isPaidAgent]
  * @param {string[]} [options.denyCidrs]           IPv4 ranges answered 403 before anything else, e.g. a VPS fleet's provider
  * @param {boolean} [options.chargeSpoofedBrowsers=false]  charge a "Chrome/…" request that lacks the Sec-Fetch-Mode header every Chromium sends
+ * @param {boolean} [options.chargeStaleBrowsers=true]  charge a desktop browser string two years behind its auto-updates (a rotation pool)
+ * @param {string[]} [options.scrapers]            SEO resellers charged and refused in robots.txt; default SCRAPER_AGENTS
+ * @param {string[]} [options.headless]            headless browsers charged; default HEADLESS_AGENTS
+ * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
  * @param {Record<string,string>|false} [options.powerKeys]  kid -> public key of the Power Keys honoured as a pass; defaults to Profullstack's, `false` switches it off
@@ -376,12 +380,22 @@ export function createGateway(options = {}) {
     const token = passFrom(request);
     if (await honours(token)) return null;
 
+    const ua = request.headers.get('user-agent') ?? '';
     const pays =
-      o.isPaidAgent(request.headers.get('user-agent') ?? '') ||
-      (o.chargeSpoofedBrowsers && isSpoofedBrowser(request));
+      o.isPaidAgent(ua) ||
+      (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)) ||
+      (o.chargeStaleBrowsers && isStaleBrowser(ua));
     if (pays && !isOpen(path)) return sell(request);
     if (isOpen(path)) return null;
 
+    /*
+     * A search crawler that is who it says it is reads free and unmetered.
+     * Googlebot walking a large site from a handful of addresses outruns any
+     * per-address allowance, and answering it 402 is how a site leaves the
+     * index. Name and address must agree, so copying the string buys nothing.
+     */
+    if (welcomeCrawler(request, o.welcome)) return null;
+
     /*
      * Everyone else gets the free allowance. Running out is answered with a
      * price rather than a 429: the caller has just shown it wants more than
@@ -419,6 +433,21 @@ export function createGateway(options = {}) {
      */
     passFrom,
     verifyPass: honours,
+    /**
+     * Whether a request is a charged crawler by this gateway's rules (the
+     * lists, a spoofed or a stale browser string), and whether it is a welcome
+     * search crawler whose address matches its name. Exposed so a throttle
+     * that meters every route charges and spares exactly the same callers.
+     */
+    isPaid: (request) => {
+      const ua = request.headers.get('user-agent') ?? '';
+      return (
+        o.isPaidAgent(ua) ||
+        (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)) ||
+        (o.chargeStaleBrowsers && isStaleBrowser(ua))
+      );
+    },
+    isWelcome: (request) => Boolean(welcomeCrawler(request, o.welcome)),
     /**
      * The x402 offer this gateway would make, and the full receipt body around
      * it, without answering a request.
@@ -435,7 +464,13 @@ export function createGateway(options = {}) {
     receipt,
     /** robots.txt with this gateway's lists and sales path. */
     robotsTxt: (extra = {}) =>
-      robotsTxt({ siteUrl: o.siteUrl, path: o.path, training: o.training, retrieval: o.retrieval, ...extra }),
+      robotsTxt({
+        siteUrl: o.siteUrl,
+        path: o.path,
+        training: [...o.training, ...o.scrapers],
+        retrieval: o.retrieval,
+        ...extra,
+      }),
     /** The sales page as HTML, for a site that mounts it on a route of its own. */
     page: () => o.page(pageCtx()),
   };
@@ -484,6 +519,9 @@ function normalise(options) {
   const siteUrl = String(options.siteUrl ?? '').replace(/\/+$/, '');
   if (!siteUrl) throw new Error('createGateway needs siteUrl');
   const training = options.training ?? TRAINING_AGENTS;
+  const scrapers = options.scrapers ?? SCRAPER_AGENTS;
+  const headless = options.headless ?? HEADLESS_AGENTS;
+  const paid = [...training, ...scrapers, ...headless];
   return {
     siteUrl,
     siteName: options.siteName || new URL(siteUrl).hostname,
@@ -501,10 +539,14 @@ function normalise(options) {
     openPaths: options.openPaths ?? [],
     denyCidrs: options.denyCidrs ?? [],
     chargeSpoofedBrowsers: Boolean(options.chargeSpoofedBrowsers),
+    chargeStaleBrowsers: options.chargeStaleBrowsers !== false,
     exempt: options.exempt ?? null,
     training,
+    scrapers,
+    headless,
+    welcome: options.welcome === false ? null : (options.welcome ?? WELCOME_CRAWLERS),
     retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
-    isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, training)),
+    isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, paid)),
     secret: options.secret ?? '',
     page: options.page ?? renderPage,
     contact: options.contact ?? '',