@profullstack/x402-gateway 0.8.1 → 0.9.0

package/README.md

@@ -92,6 +92,8 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `isPaidAgent` | training + scraper + headless lists | `(userAgent) => boolean` |
 | `scrapers` | `SCRAPER_AGENTS` | SEO resellers (AhrefsBot, SemrushBot, MJ12bot, DataForSeoBot, ...): charged and refused in robots.txt |
 | `headless` | `HEADLESS_AGENTS` | headless browsers that name themselves (Lightpanda, HeadlessChrome, PhantomJS): charged |
+| `aiIndex` | `AI_INDEX_AGENTS` | AI answer-engine indexers (meta-webindexer, YouBot, ExaSearchBot, AionBot): charged and named in robots.txt |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js' }`: every GET to these paths must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Named retrieval crawlers and address-verified search crawlers skip it; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |

package/index.d.ts

@@ -70,6 +70,10 @@ export interface GatewayOptions {
   chargeSpoofedBrowsers?: boolean;
   /** Charge a desktop browser string ~2 years behind its auto-updates (a scraper rotation pool). Default true. */
   chargeStaleBrowsers?: boolean;
+  /** AI answer-engine indexers: charged, named in robots.txt. Default AI_INDEX_AGENTS. */
+  aiIndex?: string[];
+  /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
+  challenge?: { paths: string[]; cookie?: string } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
   scrapers?: string[];
   /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */
@@ -215,6 +219,11 @@ export function wantsHtml(accept?: string | null): boolean;
 export const TRAINING_AGENTS: string[];
 export const SCRAPER_AGENTS: string[];
 export const HEADLESS_AGENTS: string[];
+export const AI_INDEX_AGENTS: string[];
+export function challengeToken(secret: string, ip: string, day?: number): Promise<string>;
+export function passedChallenge(request: Request, options: { secret: string; cookie: string; ip: string; now?: number }): Promise<boolean>;
+export function challengePage(options: { token: string; cookie: string; buyUrl: string }): string;
+export function cookieValue(request: Request, name: string): string | null;
 export interface WelcomeCrawler {
   token: string;
   cidrs: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/agents.js

@@ -51,6 +51,15 @@ export const SCRAPER_AGENTS = [
  * 151k requests from 63k addresses on one site in a day (2026-10-19). They do
  * not read robots.txt, so they are charged, never named there.
  */
+/**
+ * Crawlers that feed an AI answer engine's own index and send next to nobody
+ * back: measured on our fleet 2026-10-19, meta-webindexer alone was ~5k
+ * requests an hour with no referrals to show for it. Anthony's call the same
+ * day: charge them like training crawlers. They read robots.txt, so they are
+ * named there too.
+ */
+export const AI_INDEX_AGENTS = ['meta-webindexer', 'YouBot', 'ExaSearchBot', 'AionBot'];
+
 export const HEADLESS_AGENTS = ['Lightpanda', 'HeadlessChrome', 'PhantomJS'];
 
 /**

package/src/challenge.js

@@ -0,0 +1,60 @@
+/**
+ * A question only a browser answers: run this script, then ask again.
+ *
+ * The swarm that walks a site's item pages from a million residential
+ * addresses wears a current Chrome string, sends one request per address and
+ * never runs the page: on nichedb.dev (2026-10-19) 4.6M browser-looking
+ * requests a day came from 1.58M addresses, and the analytics beacon fired for
+ * about 2% of them. No user agent rule and no per-address allowance can see
+ * that. A person's browser can: it runs a one-line script that sets a cookie
+ * and reloads, once a day, and from then on is never asked again.
+ *
+ * The cookie is an HMAC of the caller's address and the day, so it cannot be
+ * minted without asking, and an address that rotates has to ask from each new
+ * address. The page that asks is a 402 carrying a link to the sales page, so a
+ * scraper that cannot run scripts learns what a pass costs, which is the same
+ * answer every other refused crawler gets.
+ */
+
+const enc = new TextEncoder();
+const DAY = 86_400_000;
+
+async function hmac(secret, data) {
+  const key = await crypto.subtle.importKey('raw', enc.encode(String(secret)), { name: 'HMAC', hash: 'SHA-256' }, false, [
+    'sign',
+  ]);
+  const sig = new Uint8Array(await crypto.subtle.sign('HMAC', key, enc.encode(data)));
+  let hex = '';
+  for (const b of sig.subarray(0, 16)) hex += b.toString(16).padStart(2, '0');
+  return hex;
+}
+
+/** The token a browser at `ip` gets on day `day` (days since the epoch). */
+export const challengeToken = (secret, ip, day = Math.floor(Date.now() / DAY)) => hmac(secret, `js|${ip}|${day}`);
+
+/** Read one cookie by name. */
+export function cookieValue(request, name) {
+  const raw = request.headers.get('cookie') ?? '';
+  for (const part of raw.split(';')) {
+    const i = part.indexOf('=');
+    if (i > 0 && part.slice(0, i).trim() === name) return part.slice(i + 1).trim();
+  }
+  return null;
+}
+
+/** Whether the request carries today's or yesterday's token for its address. */
+export async function passedChallenge(request, { secret, cookie, ip, now = Date.now() }) {
+  const got = cookieValue(request, cookie);
+  if (!got) return false;
+  const day = Math.floor(now / DAY);
+  for (const d of [day, day - 1]) if (got === (await challengeToken(secret, ip, d))) return true;
+  return false;
+}
+
+/** The page that asks. Tiny on purpose: answering it must cost us nothing. */
+export function challengePage({ token, cookie, buyUrl }) {
+  return `<!doctype html><html><head><meta charset="utf-8"><meta name="robots" content="noindex"><title>One moment</title></head><body>
+<script>document.cookie=${JSON.stringify(`${cookie}=${token}; Max-Age=172800; Path=/; SameSite=Lax; Secure`)};location.reload();</script>
+<noscript><p>This page needs JavaScript. Automated access is sold by the day: <a href="${buyUrl}">${buyUrl}</a></p></noscript>
+</body></html>`;
+}

package/src/index.js

@@ -1,7 +1,8 @@
 import { HEADER as POWER_KEY_HEADER, isPowerKey, keyFrom, PROFULLSTACK_KEYS, SHOP_URL, verifyKey } from '@profullstack/keys';
-import { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
+import { AI_INDEX_AGENTS, HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
 import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, isStaleBrowser, welcomeCrawler } from './edge.js';
 import { memoryQuotaStore, meters, normaliseQuota, quotaHeaders, spend } from './quota.js';
+import { challengePage, challengeToken, passedChallenge } from './challenge.js';
 import { renderPage } from './page.js';
 import { mintPass, readPass } from './pass.js';
 import { robotsTxt } from './robots.js';
@@ -16,8 +17,9 @@ import {
   verifyAndSettle,
 } from './x402.js';
 
-export { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
+export { AI_INDEX_AGENTS, HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
 export { clientIp, compileCidrs, currentMajor, inCidrs, isSpoofedBrowser, isStaleBrowser, parseCidr, welcomeCrawler } from './edge.js';
+export { challengePage, challengeToken, cookieValue, passedChallenge } from './challenge.js';
 export { renderPage } from './page.js';
 export { mintPass, readPass } from './pass.js';
 export { robotsTxt } from './robots.js';
@@ -59,6 +61,8 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {boolean} [options.chargeStaleBrowsers=true]  charge a desktop browser string two years behind its auto-updates (a rotation pool)
  * @param {string[]} [options.scrapers]            SEO resellers charged and refused in robots.txt; default SCRAPER_AGENTS
  * @param {string[]} [options.headless]            headless browsers charged; default HEADLESS_AGENTS
+ * @param {string[]} [options.aiIndex]             AI answer-engine indexers charged and named in robots.txt; default AI_INDEX_AGENTS
+ * @param {{paths: string[], cookie?: string}|false} [options.challenge]  browser-looking requests to these paths must run a one-line script (cookie + reload) first; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
@@ -396,6 +400,23 @@ export function createGateway(options = {}) {
      */
     if (welcomeCrawler(request, o.welcome)) return null;
 
+    /*
+     * Every caller on a challenged path proves it runs scripts, except the
+     * retrieval crawlers named as welcome (they cite back) and the search
+     * crawlers verified by address just above. Exempting anything that merely
+     * calls itself a bot would let the swarm walk in wearing "compatible;".
+     * Only GET and HEAD, and only with a secret to sign the answer.
+     */
+    if (o.challenge && secret && meters(o.challenge, path) && (request.method === 'GET' || request.method === 'HEAD')) {
+      if (!isTrainingAgent(request.headers.get('user-agent') ?? '', o.retrieval)) {
+        const ip = clientIp(request);
+        if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
+          const token = await challengeToken(secret, ip);
+          return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl }), 402);
+        }
+      }
+    }
+
     /*
      * Everyone else gets the free allowance. Running out is answered with a
      * price rather than a 429: the caller has just shown it wants more than
@@ -467,7 +488,7 @@ export function createGateway(options = {}) {
       robotsTxt({
         siteUrl: o.siteUrl,
         path: o.path,
-        training: [...o.training, ...o.scrapers],
+        training: [...o.training, ...o.scrapers, ...o.aiIndex],
         retrieval: o.retrieval,
         ...extra,
       }),
@@ -521,7 +542,8 @@ function normalise(options) {
   const training = options.training ?? TRAINING_AGENTS;
   const scrapers = options.scrapers ?? SCRAPER_AGENTS;
   const headless = options.headless ?? HEADLESS_AGENTS;
-  const paid = [...training, ...scrapers, ...headless];
+  const aiIndex = options.aiIndex ?? AI_INDEX_AGENTS;
+  const paid = [...training, ...scrapers, ...headless, ...aiIndex];
   return {
     siteUrl,
     siteName: options.siteName || new URL(siteUrl).hostname,
@@ -544,6 +566,11 @@ function normalise(options) {
     training,
     scrapers,
     headless,
+    aiIndex,
+    challenge:
+      options.challenge && Array.isArray(options.challenge.paths) && options.challenge.paths.length
+        ? { paths: options.challenge.paths, cookie: options.challenge.cookie ?? 'pf_js' }
+        : null,
     welcome: options.welcome === false ? null : (options.welcome ?? WELCOME_CRAWLERS),
     retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
     isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, paid)),