@profullstack/x402-gateway 0.9.0 → 0.10.0

package/README.md

@@ -92,8 +92,10 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `isPaidAgent` | training + scraper + headless lists | `(userAgent) => boolean` |
 | `scrapers` | `SCRAPER_AGENTS` | SEO resellers (AhrefsBot, SemrushBot, MJ12bot, DataForSeoBot, ...): charged and refused in robots.txt |
 | `headless` | `HEADLESS_AGENTS` | headless browsers that name themselves (Lightpanda, HeadlessChrome, PhantomJS): charged |
-| `aiIndex` | `AI_INDEX_AGENTS` | AI answer-engine indexers (meta-webindexer, YouBot, ExaSearchBot, AionBot): charged and named in robots.txt |
-| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js' }`: every GET to these paths must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Named retrieval crawlers and address-verified search crawlers skip it; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
+| `aiIndex` | none | extra agents to charge and refuse in robots.txt. Since 0.10 the AI answer-engine indexers (`AI_INDEX_AGENTS`: meta-webindexer, YouBot, ExaSearchBot, AionBot) are welcome retrieval crawlers; pass `aiIndex: AI_INDEX_AGENTS` to charge them anyway |
+| `verifiedRetrieval` | `VERIFIED_RETRIEVAL` | retrieval crawlers believed only from their operator's published addresses (OAI-SearchBot, ChatGPT-User, PerplexityBot, Perplexity-User); from elsewhere they are metered. Other retrieval and AI chat fetchers (Claude-User, Claude-SearchBot, Meta-ExternalFetcher, DuckAssistBot, MistralAI-User, the AI indexers) publish no addresses and are believed by name |
+| `chargeImpostors` | `true` | a request wearing a `welcome` crawler's name (Googlebot, bingbot, Applebot, DuckDuckBot, Cloudflare-*) from an address outside that operator's published list is charged. Never judged without a readable address |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js' }`: every GET to these paths must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |
@@ -190,3 +192,13 @@ A pass is `cp_<payload>.<hmac>`: its own expiry and the payment's nonce, signed
 ## Licence
 
 MIT
+
+## Crawler address lists
+
+`src/ranges.js` is generated from each operator's published list (Google's googlebot.json, Bing, DuckDuckGo, OpenAI's searchbot/chatgpt-user, Perplexity, Cloudflare's ips-v4/ips-v6). Refresh it before every release:
+
+```sh
+npm run ranges
+```
+
+Behind Cloudflare's proxy the caller is read from `CF-Connecting-IP`, but only when the edge hop is itself a Cloudflare address.

package/index.d.ts

@@ -70,8 +70,12 @@ export interface GatewayOptions {
   chargeSpoofedBrowsers?: boolean;
   /** Charge a desktop browser string ~2 years behind its auto-updates (a scraper rotation pool). Default true. */
   chargeStaleBrowsers?: boolean;
-  /** AI answer-engine indexers: charged, named in robots.txt. Default AI_INDEX_AGENTS. */
+  /** Extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS. Default none: since 0.10 the AI indexers are welcome retrieval crawlers. */
   aiIndex?: string[];
+  /** Retrieval crawlers believed only from their published addresses (OpenAI, Perplexity). Default VERIFIED_RETRIEVAL; false disables. */
+  verifiedRetrieval?: WelcomeCrawler[] | false;
+  /** Charge a request wearing a welcome search crawler's name from outside its published ranges. Default true. */
+  chargeImpostors?: boolean;
   /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
   challenge?: { paths: string[]; cookie?: string } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
@@ -229,6 +233,9 @@ export interface WelcomeCrawler {
   cidrs: string[];
 }
 export const WELCOME_CRAWLERS: WelcomeCrawler[];
+export const VERIFIED_RETRIEVAL: WelcomeCrawler[];
+/** The welcome crawler a request claims to be from outside its published ranges, or null. */
+export function impostorCrawler(request: Request, list: WelcomeCrawler[]): string | null;
 export const RETRIEVAL_AGENTS: string[];
 export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [
@@ -69,7 +69,8 @@
   },
   "sideEffects": false,
   "scripts": {
-    "test": "node --test"
+    "test": "node --test",
+    "ranges": "node scripts/ranges.mjs"
   },
   "dependencies": {
     "@profullstack/keys": "^0.1.0"

package/src/agents.js

@@ -1,3 +1,5 @@
+import * as R from './ranges.js';
+
 /**
  * Which crawlers pay, and which read free.
  *
@@ -52,20 +54,27 @@ export const SCRAPER_AGENTS = [
  * not read robots.txt, so they are charged, never named there.
  */
 /**
- * Crawlers that feed an AI answer engine's own index and send next to nobody
- * back: measured on our fleet 2026-10-19, meta-webindexer alone was ~5k
- * requests an hour with no referrals to show for it. Anthony's call the same
- * day: charge them like training crawlers. They read robots.txt, so they are
- * named there too.
+ * Crawlers that feed an AI answer engine's own index: meta-webindexer, YouBot,
+ * ExaSearchBot, AionBot. 0.9.0 charged them; Anthony's call on 2026-10-19 was
+ * the other way ("allow ... ai chat bots"), so since 0.10.0 they are retrieval
+ * crawlers: named welcome in robots.txt and never metered. The list stays
+ * exported for a site that wants to charge them anyway (`aiIndex`).
  */
 export const AI_INDEX_AGENTS = ['meta-webindexer', 'YouBot', 'ExaSearchBot', 'AionBot'];
 
 export const HEADLESS_AGENTS = ['Lightpanda', 'HeadlessChrome', 'PhantomJS'];
 
 /**
- * Retrieval crawlers, named in robots.txt so their operators can see they are
- * welcome. Google-Extended is Google's training token but Google documents it
- * as also gating grounding in the Gemini app, so it stays on this side.
+ * Retrieval crawlers and AI chat fetchers, named in robots.txt so their
+ * operators can see they are welcome, and never metered: a person asked an
+ * assistant about this page and the fetch is that person reading it.
+ * Google-Extended is Google's training token but Google documents it as also
+ * gating grounding in the Gemini app, so it stays on this side.
+ *
+ * Where the operator publishes its addresses (OpenAI, Perplexity) the name is
+ * only believed from them: see VERIFIED_RETRIEVAL. The rest are believed by
+ * name, which a swarm can copy; ThreatCrush's repeat-402 ladder is the
+ * backstop for that, not this list.
  */
 export const RETRIEVAL_AGENTS = [
   'OAI-SearchBot',
@@ -76,6 +85,10 @@ export const RETRIEVAL_AGENTS = [
   'Perplexity-User',
   'Google-Extended',
   'Bingbot',
+  'Meta-ExternalFetcher',
+  'DuckAssistBot',
+  'MistralAI-User',
+  ...AI_INDEX_AGENTS,
 ];
 
 const lower = (list) => list.map((t) => t.toLowerCase());
@@ -96,20 +109,35 @@ export function isTrainingAgent(userAgent = '', agents = TRAINING_AGENTS) {
 
 /**
  * Search crawlers that are welcome everywhere, with the address ranges their
- * operators publish. A user agent alone is not enough to spare a caller from
- * the free allowance -- "Googlebot" is the first string a scraper copies -- so
- * a crawler is only spared when its address agrees with its name.
+ * operators publish. A user agent alone is not enough to spare a caller --
+ * "Googlebot" is the first string a scraper copies -- so a crawler is only
+ * spared when its address agrees with its name, and a request that wears one
+ * of these names from an address outside the list is an impostor and is
+ * charged (`chargeImpostors`).
  *
- * Ranges are the stable blocks of each operator's published list
- * (developers.google.com/static/search/apis/ipranges/googlebot.json,
- * bing.com/toolbox/bingbot.json, Apple's 17.0.0.0/8); a crawl from outside them
- * is simply metered like everyone else, which fails safe.
+ * The lists are the operators' own (src/ranges.js, regenerated by
+ * scripts/ranges.mjs before each release): googlebot.json, bingbot.json,
+ * duckduckbot.json, Cloudflare's ips-v4/ips-v6 for its own crawlers
+ * (Always Online, health checks, Radar), and Apple's 17.0.0.0/8 plus its
+ * AS714 IPv6 blocks for Applebot.
  */
 export const WELCOME_CRAWLERS = [
-  { token: 'Googlebot', cidrs: ['66.249.64.0/19', '192.178.4.0/22', '34.100.182.96/28'] },
-  {
-    token: 'bingbot',
-    cidrs: ['40.77.167.0/24', '157.55.39.0/24', '207.46.13.0/24', '52.167.144.0/24', '40.77.188.0/22', '20.15.133.160/27'],
-  },
-  { token: 'Applebot', cidrs: ['17.0.0.0/8'] },
+  { token: 'Googlebot', cidrs: R.googlebot },
+  { token: 'bingbot', cidrs: R.bingbot },
+  { token: 'Applebot', cidrs: ['17.0.0.0/8', '2620:149::/32', '2a01:b740::/32'] },
+  { token: 'DuckDuckBot', cidrs: R.duckduckbot },
+  { token: 'Cloudflare-', cidrs: R.cloudflare },
+];
+
+/**
+ * Retrieval crawlers whose operators publish their addresses. These are
+ * unmetered only from those addresses; from anywhere else they are metered
+ * like any caller (not charged outright: the lists move faster than releases,
+ * and a real ChatGPT-User from a new block should still read the page).
+ */
+export const VERIFIED_RETRIEVAL = [
+  { token: 'OAI-SearchBot', cidrs: R.oaiSearchbot },
+  { token: 'ChatGPT-User', cidrs: R.chatgptUser },
+  { token: 'PerplexityBot', cidrs: R.perplexitybot },
+  { token: 'Perplexity-User', cidrs: R.perplexityUser },
 ];

package/src/edge.js

@@ -1,3 +1,5 @@
+import * as RANGES from './ranges.js';
+
 /**
  * The two checks that do not need a user agent to be honest.
  *
@@ -33,9 +35,51 @@ function ipv4ToInt(ip) {
   return n;
 }
 
-/** Parse "a.b.c.d/len" (or a bare address) into a matcher. Null if unreadable. */
+function ipv6ToBig(ip) {
+  let s = String(ip).toLowerCase();
+  if (s.startsWith('[') && s.endsWith(']')) s = s.slice(1, -1);
+  const zone = s.indexOf('%');
+  if (zone >= 0) s = s.slice(0, zone);
+  if (!s.includes(':')) return null;
+  // An IPv4 tail ("::ffff:1.2.3.4") becomes the last two groups.
+  const tail = /(\d+\.\d+\.\d+\.\d+)$/.exec(s);
+  if (tail) {
+    const v4 = ipv4ToInt(tail[1]);
+    if (v4 === null) return null;
+    s = `${s.slice(0, -tail[1].length)}${(v4 >>> 16).toString(16)}:${(v4 & 0xffff).toString(16)}`;
+  }
+  const halves = s.split('::');
+  if (halves.length > 2) return null;
+  const head = halves[0] ? halves[0].split(':') : [];
+  const rest = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+  const fill = halves.length === 2 ? 8 - head.length - rest.length : 0;
+  if (fill < 0 || (halves.length === 1 && head.length !== 8)) return null;
+  const groups = [...head, ...Array(fill).fill('0'), ...rest];
+  if (groups.length !== 8) return null;
+  let n = 0n;
+  for (const g of groups) {
+    if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
+    n = (n << 16n) | BigInt(Number.parseInt(g, 16));
+  }
+  return n;
+}
+
+const ALL6 = (1n << 128n) - 1n;
+
+/**
+ * Parse "a.b.c.d/len" or "2001:db8::/32" (or a bare address) into a matcher.
+ * Null if unreadable.
+ */
 export function parseCidr(cidr) {
   const [ip, lenRaw] = String(cidr).trim().split('/');
+  if (ip.includes(':')) {
+    const base = ipv6ToBig(ip);
+    if (base === null) return null;
+    const len = lenRaw === undefined ? 128 : Number(lenRaw);
+    if (!Number.isInteger(len) || len < 0 || len > 128) return null;
+    const mask = len === 0 ? 0n : (ALL6 << BigInt(128 - len)) & ALL6;
+    return { v6: true, base: base & mask, mask, text: `${ip}/${len}` };
+  }
   const base = ipv4ToInt(ip);
   if (base === null) return null;
   const len = lenRaw === undefined ? 32 : Number(lenRaw);
@@ -49,13 +93,23 @@ export function compileCidrs(list = []) {
   return list.map(parseCidr).filter(Boolean);
 }
 
-/** Whether an IPv4 address falls inside any compiled range. */
+/** Whether an address (IPv4 or IPv6) falls inside any compiled range. */
 export function inCidrs(ip, compiled) {
-  const n = ipv4ToInt(String(ip ?? '').trim());
+  const s = String(ip ?? '').trim();
+  if (s.includes(':')) {
+    const n = ipv6ToBig(s);
+    if (n === null) return false;
+    // An IPv4-mapped address is the IPv4 address it maps.
+    if (n >> 32n === 0xffffn) return inCidrs(int4(Number(n & 0xffffffffn)), compiled);
+    return compiled.some((c) => c.v6 && (n & c.mask) === c.base);
+  }
+  const n = ipv4ToInt(s);
   if (n === null) return false;
-  return compiled.some((c) => ((n & c.mask) >>> 0) === c.base);
+  return compiled.some((c) => !c.v6 && ((n & c.mask) >>> 0) === c.base);
 }
 
+const int4 = (n) => [n >>> 24, (n >>> 16) & 255, (n >>> 8) & 255, n & 255].join('.');
+
 /**
  * The caller's address, as the edge reported it.
  *
@@ -72,6 +126,20 @@ export function inCidrs(ip, compiled) {
  * refused, nothing is charged, by this check.
  */
 export function clientIp(request) {
+  const edge = edgeIp(request);
+  /*
+   * Behind Cloudflare's proxy the edge hop is a Cloudflare address and the
+   * caller is in CF-Connecting-IP. That header is only believed when the hop
+   * really is Cloudflare's, so a client cannot set it to choose its address.
+   */
+  if (edge && inCidrs(edge, cloudflare())) {
+    const cf = request.headers.get('cf-connecting-ip')?.trim();
+    if (cf) return cf;
+  }
+  return edge;
+}
+
+function edgeIp(request) {
   const real = request.headers.get('x-real-ip')?.trim();
   if (real) return real;
   const xff = request.headers.get('x-forwarded-for');
@@ -80,6 +148,9 @@ export function clientIp(request) {
   return hops[hops.length - 1] ?? '';
 }
 
+let CF = null;
+const cloudflare = () => (CF ??= compileCidrs(RANGES.cloudflare));
+
 /* -------------------------------------------------------------- spoofing -- */
 
 const CLAIMS_CHROMIUM = /\bChrome\/\d+/;
@@ -210,3 +281,29 @@ export function welcomeCrawler(request, list) {
   }
   return null;
 }
+
+/**
+ * The welcome crawler a request claims to be but is not: its user agent names
+ * one of `list` and its address is outside that crawler's published ranges.
+ * Null when it names none, when it is the real thing, or when there is no
+ * readable address to judge (fail safe: never charge on a guess).
+ *
+ * @param {Request} request
+ * @param {{token: string, cidrs: string[]}[]} list
+ */
+export function impostorCrawler(request, list) {
+  if (!list?.length) return null;
+  const ua = (request.headers.get('user-agent') ?? '').toLowerCase();
+  if (!ua) return null;
+  const ip = clientIp(request);
+  if (!ip || (!ip.includes(':') && !/^\d+\.\d+\.\d+\.\d+$/.test(ip))) return null;
+  let claimed = null;
+  for (const c of list) {
+    if (!ua.includes(c.token.toLowerCase())) continue;
+    let compiled = COMPILED.get(c);
+    if (!compiled) COMPILED.set(c, (compiled = compileCidrs(c.cidrs)));
+    if (inCidrs(ip, compiled)) return null;
+    claimed ??= c.token;
+  }
+  return claimed;
+}

package/src/index.js

@@ -1,6 +1,6 @@
 import { HEADER as POWER_KEY_HEADER, isPowerKey, keyFrom, PROFULLSTACK_KEYS, SHOP_URL, verifyKey } from '@profullstack/keys';
-import { AI_INDEX_AGENTS, HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
-import { clientIp, compileCidrs, inCidrs, isSpoofedBrowser, isStaleBrowser, welcomeCrawler } from './edge.js';
+import { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, VERIFIED_RETRIEVAL, WELCOME_CRAWLERS } from './agents.js';
+import { clientIp, compileCidrs, impostorCrawler, inCidrs, isSpoofedBrowser, isStaleBrowser, welcomeCrawler } from './edge.js';
 import { memoryQuotaStore, meters, normaliseQuota, quotaHeaders, spend } from './quota.js';
 import { challengePage, challengeToken, passedChallenge } from './challenge.js';
 import { renderPage } from './page.js';
@@ -17,8 +17,27 @@ import {
   verifyAndSettle,
 } from './x402.js';
 
-export { AI_INDEX_AGENTS, HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, WELCOME_CRAWLERS } from './agents.js';
-export { clientIp, compileCidrs, currentMajor, inCidrs, isSpoofedBrowser, isStaleBrowser, parseCidr, welcomeCrawler } from './edge.js';
+export {
+  AI_INDEX_AGENTS,
+  HEADLESS_AGENTS,
+  isTrainingAgent,
+  RETRIEVAL_AGENTS,
+  SCRAPER_AGENTS,
+  TRAINING_AGENTS,
+  VERIFIED_RETRIEVAL,
+  WELCOME_CRAWLERS,
+} from './agents.js';
+export {
+  clientIp,
+  compileCidrs,
+  currentMajor,
+  impostorCrawler,
+  inCidrs,
+  isSpoofedBrowser,
+  isStaleBrowser,
+  parseCidr,
+  welcomeCrawler,
+} from './edge.js';
 export { challengePage, challengeToken, cookieValue, passedChallenge } from './challenge.js';
 export { renderPage } from './page.js';
 export { mintPass, readPass } from './pass.js';
@@ -61,7 +80,9 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {boolean} [options.chargeStaleBrowsers=true]  charge a desktop browser string two years behind its auto-updates (a rotation pool)
  * @param {string[]} [options.scrapers]            SEO resellers charged and refused in robots.txt; default SCRAPER_AGENTS
  * @param {string[]} [options.headless]            headless browsers charged; default HEADLESS_AGENTS
- * @param {string[]} [options.aiIndex]             AI answer-engine indexers charged and named in robots.txt; default AI_INDEX_AGENTS
+ * @param {string[]} [options.aiIndex]             extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS; default none (they are welcome retrieval crawlers since 0.10)
+ * @param {{token: string, cidrs: string[]}[]|false} [options.verifiedRetrieval]  retrieval crawlers believed only from their published addresses; default VERIFIED_RETRIEVAL
+ * @param {boolean} [options.chargeImpostors=true]  charge a request wearing a welcome search crawler's name from outside its published ranges
  * @param {{paths: string[], cookie?: string}|false} [options.challenge]  browser-looking requests to these paths must run a one-line script (cookie + reload) first; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
@@ -202,6 +223,40 @@ export function createGateway(options = {}) {
     return Boolean(await readPass(token, { secret }));
   };
 
+  /**
+   * Whether a request is charged by the lists, as a spoofed or stale browser
+   * string, or as an impostor wearing a welcome search crawler's name from an
+   * address that crawler's operator does not publish.
+   */
+  const isPaid = (request) => {
+    const ua = request.headers.get('user-agent') ?? '';
+    return Boolean(
+      o.isPaidAgent(ua) ||
+        (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)) ||
+        (o.chargeStaleBrowsers && isStaleBrowser(ua)) ||
+        (o.chargeImpostors && impostorCrawler(request, o.welcome)),
+    );
+  };
+
+  /**
+   * Whether a request reads free and unmetered: a search crawler whose address
+   * agrees with its name, a retrieval crawler from its operator's published
+   * addresses, or a named retrieval crawler / AI chat fetcher whose operator
+   * publishes none. A name that belongs to an address-checked list is never
+   * believed on its name alone.
+   */
+  const isWelcome = (request) => {
+    if (welcomeCrawler(request, o.welcome)) return true;
+    if (welcomeCrawler(request, o.verifiedRetrieval)) return true;
+    const ua = (request.headers.get('user-agent') ?? '').toLowerCase();
+    if (!ua) return false;
+    // The default lists count even when switched off: switching off address
+    // checks must not turn "bingbot" into a name anyone can wear.
+    const checked = [...WELCOME_CRAWLERS, ...VERIFIED_RETRIEVAL, ...(o.welcome ?? []), ...(o.verifiedRetrieval ?? [])];
+    if (checked.some((c) => ua.includes(c.token.toLowerCase()))) return false;
+    return isTrainingAgent(ua, o.retrieval);
+  };
+
   /**
    * Answer one request with the sale.
    *
@@ -385,11 +440,7 @@ export function createGateway(options = {}) {
     if (await honours(token)) return null;
 
     const ua = request.headers.get('user-agent') ?? '';
-    const pays =
-      o.isPaidAgent(ua) ||
-      (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)) ||
-      (o.chargeStaleBrowsers && isStaleBrowser(ua));
-    if (pays && !isOpen(path)) return sell(request);
+    if (isPaid(request) && !isOpen(path)) return sell(request);
     if (isOpen(path)) return null;
 
     /*
@@ -398,22 +449,30 @@ export function createGateway(options = {}) {
      * per-address allowance, and answering it 402 is how a site leaves the
      * index. Name and address must agree, so copying the string buys nothing.
      */
-    if (welcomeCrawler(request, o.welcome)) return null;
+    if (isWelcome(request)) return null;
 
     /*
      * Every caller on a challenged path proves it runs scripts, except the
-     * retrieval crawlers named as welcome (they cite back) and the search
-     * crawlers verified by address just above. Exempting anything that merely
+     * welcome crawlers just above (search crawlers verified by address,
+     * retrieval crawlers and AI chat fetchers). Exempting anything that merely
      * calls itself a bot would let the swarm walk in wearing "compatible;".
-     * Only GET and HEAD, and only with a secret to sign the answer.
+     * Only GET and HEAD, and only with a secret to sign the answer. A caller
+     * that asked for HTML gets the one-line script; anything else gets the
+     * JSON offer, since it was never going to run a script.
      */
     if (o.challenge && secret && meters(o.challenge, path) && (request.method === 'GET' || request.method === 'HEAD')) {
-      if (!isTrainingAgent(request.headers.get('user-agent') ?? '', o.retrieval)) {
-        const ip = clientIp(request);
-        if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
-          const token = await challengeToken(secret, ip);
-          return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl }), 402);
+      const ip = clientIp(request);
+      if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
+        if (!wantsHtml(request.headers.get('accept'))) {
+          return json(
+            receipt(1, {
+              error: `${path} is read in a browser (it sets a cookie with one line of script) or with a pass. Read ${buyUrl} for how.`,
+            }),
+            402,
+          );
         }
+        const token = await challengeToken(secret, ip);
+        return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl }), 402);
       }
     }
 
@@ -460,15 +519,8 @@ export function createGateway(options = {}) {
      * search crawler whose address matches its name. Exposed so a throttle
      * that meters every route charges and spares exactly the same callers.
      */
-    isPaid: (request) => {
-      const ua = request.headers.get('user-agent') ?? '';
-      return (
-        o.isPaidAgent(ua) ||
-        (o.chargeSpoofedBrowsers && isSpoofedBrowser(request)) ||
-        (o.chargeStaleBrowsers && isStaleBrowser(ua))
-      );
-    },
-    isWelcome: (request) => Boolean(welcomeCrawler(request, o.welcome)),
+    isPaid,
+    isWelcome,
     /**
      * The x402 offer this gateway would make, and the full receipt body around
      * it, without answering a request.
@@ -542,7 +594,7 @@ function normalise(options) {
   const training = options.training ?? TRAINING_AGENTS;
   const scrapers = options.scrapers ?? SCRAPER_AGENTS;
   const headless = options.headless ?? HEADLESS_AGENTS;
-  const aiIndex = options.aiIndex ?? AI_INDEX_AGENTS;
+  const aiIndex = options.aiIndex ?? [];
   const paid = [...training, ...scrapers, ...headless, ...aiIndex];
   return {
     siteUrl,
@@ -562,6 +614,7 @@ function normalise(options) {
     denyCidrs: options.denyCidrs ?? [],
     chargeSpoofedBrowsers: Boolean(options.chargeSpoofedBrowsers),
     chargeStaleBrowsers: options.chargeStaleBrowsers !== false,
+    chargeImpostors: options.chargeImpostors !== false,
     exempt: options.exempt ?? null,
     training,
     scrapers,
@@ -572,6 +625,7 @@ function normalise(options) {
         ? { paths: options.challenge.paths, cookie: options.challenge.cookie ?? 'pf_js' }
         : null,
     welcome: options.welcome === false ? null : (options.welcome ?? WELCOME_CRAWLERS),
+    verifiedRetrieval: options.verifiedRetrieval === false ? null : (options.verifiedRetrieval ?? VERIFIED_RETRIEVAL),
     retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
     isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, paid)),
     secret: options.secret ?? '',