@profullstack/x402-gateway 0.10.0 → 0.10.2

package/README.md

@@ -18,7 +18,7 @@ npm install @profullstack/x402-gateway
 | --- | --- |
 | A person, Googlebot, Bingbot, Applebot, OAI-SearchBot, Claude-SearchBot, PerplexityBot… | the site, untouched |
 | GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended… | `402` with an x402 offer, or the HTML sales page if it asked for HTML |
-| Anyone at `/crawl` | the sales page (HTML) or the offer (JSON), and the place to pay |
+| Anyone at `/crawl` | the sales page (HTML, or always with `?format=html`) or the offer (JSON), and the place to pay. The 402 prose points at `/crawl?format=html`, so a client that sent no Accept header still reaches the explanation |
 | A request with a valid pass | the site |
 
 The sales page explains the price, how to pay with an x402 client, and how to pay with the CoinPay CLI:
@@ -95,7 +95,7 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `aiIndex` | none | extra agents to charge and refuse in robots.txt. Since 0.10 the AI answer-engine indexers (`AI_INDEX_AGENTS`: meta-webindexer, YouBot, ExaSearchBot, AionBot) are welcome retrieval crawlers; pass `aiIndex: AI_INDEX_AGENTS` to charge them anyway |
 | `verifiedRetrieval` | `VERIFIED_RETRIEVAL` | retrieval crawlers believed only from their operator's published addresses (OAI-SearchBot, ChatGPT-User, PerplexityBot, Perplexity-User); from elsewhere they are metered. Other retrieval and AI chat fetchers (Claude-User, Claude-SearchBot, Meta-ExternalFetcher, DuckAssistBot, MistralAI-User, the AI indexers) publish no addresses and are believed by name |
 | `chargeImpostors` | `true` | a request wearing a `welcome` crawler's name (Googlebot, bingbot, Applebot, DuckDuckBot, Cloudflare-*) from an address outside that operator's published list is charged. Never judged without a readable address |
-| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js' }`: every GET to these paths must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |

package/index.d.ts

@@ -77,7 +77,8 @@ export interface GatewayOptions {
   /** Charge a request wearing a welcome search crawler's name from outside its published ranges. Default true. */
   chargeImpostors?: boolean;
   /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
-  challenge?: { paths: string[]; cookie?: string } | false;
+  /** Only GETs to `paths` that `when` accepts (default: all of them), e.g. `(req, url) => url.search.length > 1` for faceted URLs. */
+  challenge?: { paths: string[]; cookie?: string; when?: (request: Request, url: URL) => boolean } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
   scrapers?: string[];
   /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/index.js

@@ -83,7 +83,7 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {string[]} [options.aiIndex]             extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS; default none (they are welcome retrieval crawlers since 0.10)
  * @param {{token: string, cidrs: string[]}[]|false} [options.verifiedRetrieval]  retrieval crawlers believed only from their published addresses; default VERIFIED_RETRIEVAL
  * @param {boolean} [options.chargeImpostors=true]  charge a request wearing a welcome search crawler's name from outside its published ranges
- * @param {{paths: string[], cookie?: string}|false} [options.challenge]  browser-looking requests to these paths must run a one-line script (cookie + reload) first; off by default
+ * @param {{paths: string[], cookie?: string, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
@@ -106,6 +106,13 @@ export function createGateway(options = {}) {
   const money = (cents) => `${(cents / 100).toFixed(2)} ${o.currency}`;
   const price = money(o.priceCents);
   const buyUrl = `${o.siteUrl}${o.path}`;
+  /*
+   * Where the prose points. A bare GET of the sales page answers JSON, because
+   * the body is the only place an x402 client reads the offer from, so a
+   * pointer to it would loop for the reader who needs the explanation most.
+   * `?format=html` is the explainer whatever the Accept header says.
+   */
+  const howUrl = `${buyUrl}?format=html`;
 
   /**
    * How many terms a request is asking to buy: `?days=N`, clamped to
@@ -374,7 +381,7 @@ export function createGateway(options = {}) {
       );
     }
 
-    if (wantsHtml(request.headers.get('accept'))) {
+    if (wantsHtml(request.headers.get('accept')) || new URL(request.url).searchParams.get('format') === 'html') {
       return html(o.page(pageCtx(asked, usage, quota)), 402, rateHeaders);
     }
 
@@ -401,7 +408,7 @@ export function createGateway(options = {}) {
       );
     }
 
-    return json(receipt(asked, { error: `Payment required for training crawlers. Read ${buyUrl} for how.` }), 402);
+    return json(receipt(asked, { error: `Payment required for training crawlers. Read ${howUrl} for how.` }), 402);
   }
 
   /**
@@ -460,13 +467,19 @@ export function createGateway(options = {}) {
      * that asked for HTML gets the one-line script; anything else gets the
      * JSON offer, since it was never going to run a script.
      */
-    if (o.challenge && secret && meters(o.challenge, path) && (request.method === 'GET' || request.method === 'HEAD')) {
+    if (
+      o.challenge &&
+      secret &&
+      meters(o.challenge, path) &&
+      (!o.challenge.when || o.challenge.when(request, new URL(request.url))) &&
+      (request.method === 'GET' || request.method === 'HEAD')
+    ) {
       const ip = clientIp(request);
       if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
         if (!wantsHtml(request.headers.get('accept'))) {
           return json(
             receipt(1, {
-              error: `${path} is read in a browser (it sets a cookie with one line of script) or with a pass. Read ${buyUrl} for how.`,
+              error: `${path} is read in a browser (it sets a cookie with one line of script) or with a pass. Read ${howUrl} for how.`,
             }),
             402,
           );
@@ -622,7 +635,11 @@ function normalise(options) {
     aiIndex,
     challenge:
       options.challenge && Array.isArray(options.challenge.paths) && options.challenge.paths.length
-        ? { paths: options.challenge.paths, cookie: options.challenge.cookie ?? 'pf_js' }
+        ? {
+            paths: options.challenge.paths,
+            cookie: options.challenge.cookie ?? 'pf_js',
+            when: typeof options.challenge.when === 'function' ? options.challenge.when : null,
+          }
         : null,
     welcome: options.welcome === false ? null : (options.welcome ?? WELCOME_CRAWLERS),
     verifiedRetrieval: options.verifiedRetrieval === false ? null : (options.verifiedRetrieval ?? VERIFIED_RETRIEVAL),