@profullstack/x402-gateway 0.1.0 → 0.2.1

package/README.md

@@ -93,8 +93,16 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `contact` | | mailto: or URL for bulk deals |
 | `onSale` | | `({ payer, ref, token, expiresAt, userAgent, priceCents, currency }) => …`, for accounting |
 
+| `denyCidrs` | `[]` | IPv4 ranges answered with a tiny `403` before anything else. For a VPS fleet that spoofs a browser: hosting ranges serve no readers. |
+| `chargeSpoofedBrowsers` | `false` | charge a request that claims `Chrome/…` but sends no `Sec-Fetch-Mode`. Every Chromium since 76, headless included, sends it on every request and no script or extension can remove it, so its absence means an HTTP client with a copied string. Firefox and Safari are not judged. |
+| `exempt` | | `(request) => boolean`, never charged: e.g. a request carrying your signed-in cookie |
+
 Without `coinpay.apiKey` and `payTo` the gateway still answers training crawlers with 402 and the page says payments are off. Nothing is sold, but nothing is given away either.
 
+## Crawlers that do not say who they are
+
+The lists catch crawlers that name themselves. Two do not: a VPS fleet wearing a browser string, and a residential-proxy rotation cycling a few Chrome strings across hundreds of addresses. `denyCidrs` handles the first (`['51.38.0.0/16', '54.38.0.0/16', …]` for one provider's ranges); `chargeSpoofedBrowsers` handles both by asking a question only a browser can answer. A request that answers it is left alone. One that cannot gets the same 402 as GPTBot, which costs the site a hash instead of a render.
+
 ## How the money moves
 
 The offer is x402 v2 in CoinPay's dialect: USDC under the `exact` scheme on Base, Polygon or Ethereum, EIP-3009 `transferWithAuthorization`. The buyer signs, the gateway sends the proof to CoinPay's `/api/x402/verify` and `/api/x402/settle`, and CoinPay's relayer broadcasts the transfer, paying the gas. The USDC goes straight to `payTo`.

package/index.d.ts

@@ -49,6 +49,12 @@ export interface GatewayOptions {
   retrieval?: string[];
   /** Who is charged. Default: the training list, substring-matched on the user agent. */
   isPaidAgent?: (userAgent: string) => boolean;
+  /** IPv4 CIDRs answered 403 before anything else (a VPS fleet's provider ranges). */
+  denyCidrs?: string[];
+  /** Charge a request that claims "Chrome/…" but lacks the Sec-Fetch-Mode header every Chromium sends. Default false. */
+  chargeSpoofedBrowsers?: boolean;
+  /** Requests never charged, e.g. ones carrying a signed-in cookie. */
+  exempt?: (request: Request) => boolean;
   /** Pass signing secret. Defaults to the CoinPay key. */
   secret?: string;
   page?: (ctx: PageContext) => string;
@@ -109,6 +115,14 @@ export const RETRIEVAL_AGENTS: string[];
 export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean;
 
 export function robotsTxt(options: RobotsOptions & { siteUrl: string }): string;
+
+/** ./edge (also re-exported from the root) */
+export interface Cidr { base: number; mask: number; text: string }
+export function parseCidr(cidr: string): Cidr | null;
+export function compileCidrs(list?: string[]): Cidr[];
+export function inCidrs(ip: string, compiled: Cidr[]): boolean;
+export function clientIp(request: Request): string;
+export function isSpoofedBrowser(request: Request): boolean;
 export function renderPage(ctx: PageContext): string;
 
 export function mintPass(args: { secret: string; ref: string | null; expiresAt: number; now?: number }): Promise<{ token: string; expiresAt: number; ref: string | null }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [
@@ -30,7 +30,8 @@
     "./hono": { "types": "./index.d.ts", "import": "./src/hono.js" },
     "./next": { "types": "./index.d.ts", "import": "./src/next.js" },
     "./robots": { "types": "./index.d.ts", "import": "./src/robots.js" },
-    "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" }
+    "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" },
+    "./edge": { "types": "./index.d.ts", "import": "./src/edge.js" }
   },
   "types": "./index.d.ts",
   "files": ["src", "index.d.ts", "README.md", "LICENSE"],

package/src/edge.js

@@ -0,0 +1,101 @@
+/**
+ * The two checks that do not need a user agent to be honest.
+ *
+ * A crawler that names itself is charged by the lists in ./agents. The ones
+ * that do not -- a VPS fleet wearing "Chrome/148", a residential-proxy
+ * rotation cycling three Chrome strings across five hundred addresses -- need
+ * something the request cannot help giving away. Two things qualify:
+ *
+ *   1. Where it came from. A hosting provider's address range serves no
+ *      readers, only machines. A CIDR denylist answers those with a tiny 403
+ *      before anything else runs.
+ *
+ *   2. Whether it is the browser it claims to be. Every Chromium since 76,
+ *      headless included, sends `Sec-Fetch-Mode` on every request; it is a
+ *      forbidden header, so no page script and no extension can remove it.
+ *      A request that says "Chrome/145" and does not send it is an HTTP client
+ *      with a copied string. That is not a person, and it is charged like any
+ *      other crawler.
+ */
+
+/* ------------------------------------------------------------------ CIDRs -- */
+
+function ipv4ToInt(ip) {
+  const parts = ip.split('.');
+  if (parts.length !== 4) return null;
+  let n = 0;
+  for (const p of parts) {
+    if (!/^\d{1,3}$/.test(p)) return null;
+    const v = Number(p);
+    if (v > 255) return null;
+    n = n * 256 + v;
+  }
+  return n;
+}
+
+/** Parse "a.b.c.d/len" (or a bare address) into a matcher. Null if unreadable. */
+export function parseCidr(cidr) {
+  const [ip, lenRaw] = String(cidr).trim().split('/');
+  const base = ipv4ToInt(ip);
+  if (base === null) return null;
+  const len = lenRaw === undefined ? 32 : Number(lenRaw);
+  if (!Number.isInteger(len) || len < 0 || len > 32) return null;
+  const mask = len === 0 ? 0 : (0xffffffff << (32 - len)) >>> 0;
+  return { base: (base & mask) >>> 0, mask, text: `${ip}/${len}` };
+}
+
+/** Compile a denylist once. Unreadable entries are dropped, not guessed at. */
+export function compileCidrs(list = []) {
+  return list.map(parseCidr).filter(Boolean);
+}
+
+/** Whether an IPv4 address falls inside any compiled range. */
+export function inCidrs(ip, compiled) {
+  const n = ipv4ToInt(String(ip ?? '').trim());
+  if (n === null) return false;
+  return compiled.some((c) => ((n & c.mask) >>> 0) === c.base);
+}
+
+/**
+ * The caller's address, as the edge reported it.
+ *
+ * `x-forwarded-for` is a list the client can seed, and a denylist read from
+ * its first entry is a denylist any client can step around by sending one.
+ * The entry our own edge appends is the LAST, on every platform in front of
+ * these sites (Railway's proxy, nginx with `$proxy_add_x_forwarded_for`), so
+ * last is what is used. `x-real-ip` is nginx's spelling of the same hop and
+ * is preferred when present, because nginx sets it from the socket and
+ * nothing a client sends survives into it.
+ *
+ * A CDN in front of the edge would make the last hop the CDN's; put its
+ * ranges nowhere near `denyCidrs` and this still fails safe: nothing is
+ * refused, nothing is charged, by this check.
+ */
+export function clientIp(request) {
+  const real = request.headers.get('x-real-ip')?.trim();
+  if (real) return real;
+  const xff = request.headers.get('x-forwarded-for');
+  if (!xff) return '';
+  const hops = xff.split(',').map((h) => h.trim()).filter(Boolean);
+  return hops[hops.length - 1] ?? '';
+}
+
+/* -------------------------------------------------------------- spoofing -- */
+
+const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
+
+/**
+ * A request that claims a Chromium user agent but carries none of the
+ * fetch-metadata headers Chromium cannot omit.
+ *
+ * Only Chromium is judged: Firefox and Safari added Sec-Fetch later and
+ * older builds of both are still out there, so their absence proves nothing.
+ * `sec-fetch-mode` is the one checked because it is present on every request
+ * kind -- navigation, subresource, fetch -- unlike `sec-ch-ua`, which a
+ * privacy proxy may strip.
+ */
+export function isSpoofedBrowser(request) {
+  const ua = request.headers.get('user-agent') ?? '';
+  if (!CLAIMS_CHROMIUM.test(ua)) return false;
+  return !request.headers.has('sec-fetch-mode');
+}

package/src/index.js

@@ -1,4 +1,5 @@
 import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
+import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser } from './edge.js';
 import { renderPage } from './page.js';
 import { mintPass, readPass } from './pass.js';
 import { robotsTxt } from './robots.js';
@@ -14,6 +15,7 @@ import {
 } from './x402.js';
 
 export { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
+export { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, parseCidr } from './edge.js';
 export { renderPage } from './page.js';
 export { mintPass, readPass } from './pass.js';
 export { robotsTxt } from './robots.js';
@@ -45,6 +47,9 @@ export { buildOffer, decodePayment, expectedFor, METHODS, verifyAndSettle } from
  * @param {string} [options.path='/crawl']         the sales page
  * @param {string[]} [options.openPaths]           extra paths a refused crawler may read
  * @param {(ua: string) => boolean} [options.isPaidAgent]
+ * @param {string[]} [options.denyCidrs]           IPv4 ranges answered 403 before anything else, e.g. a VPS fleet's provider
+ * @param {boolean} [options.chargeSpoofedBrowsers=false]  charge a "Chrome/…" request that lacks the Sec-Fetch-Mode header every Chromium sends
+ * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
  * @param {(ctx: object) => string} [options.page] custom sales page renderer
  * @param {string} [options.contact]               mailto: or URL for bulk deals
@@ -57,6 +62,7 @@ export function createGateway(options = {}) {
   const secret = o.secret || o.coinpay.apiKey || null;
 
   const openPaths = ['/robots.txt', o.path, '/security.txt', '/.well-known/', ...o.openPaths];
+  const denied = compileCidrs(o.denyCidrs);
   const isOpen = (path) => openPaths.some((p) => (p.endsWith('/') ? path.startsWith(p) : path === p));
 
   const price = `${(o.priceCents / 100).toFixed(2)} ${o.currency}`;
@@ -204,9 +210,26 @@ export function createGateway(options = {}) {
    * starts from the user agent.
    */
   async function handle(request) {
+    /*
+     * Addresses that serve no readers are refused before anything else, with
+     * a body small enough that refusing costs nothing. Not 402: there is no
+     * pass on sale to a hosting range that spoofs a browser, because whoever
+     * runs it has already declined to say who they are.
+     */
+    if (denied.length && inCidrs(clientIp(request), denied)) {
+      return new Response('Not available from this network.\n', {
+        status: 403,
+        headers: { 'content-type': 'text/plain; charset=utf-8', ...noStore },
+      });
+    }
+
     const path = new URL(request.url).pathname;
     if (path === o.path) return sell(request);
-    if (!o.isPaidAgent(request.headers.get('user-agent') ?? '')) return null;
+    if (o.exempt && o.exempt(request)) return null;
+    const pays =
+      o.isPaidAgent(request.headers.get('user-agent') ?? '') ||
+      (o.chargeSpoofedBrowsers && isSpoofedBrowser(request));
+    if (!pays) return null;
     if (isOpen(path)) return null;
 
     const token = passFrom(request);
@@ -248,6 +271,9 @@ function normalise(options) {
     header: String(options.header ?? 'x-crawl-pass').toLowerCase(),
     path: options.path ?? '/crawl',
     openPaths: options.openPaths ?? [],
+    denyCidrs: options.denyCidrs ?? [],
+    chargeSpoofedBrowsers: Boolean(options.chargeSpoofedBrowsers),
+    exempt: options.exempt ?? null,
     training,
     retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
     isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, training)),