@profullstack/x402-gateway 0.10.0 → 0.10.1

package/README.md

@@ -95,7 +95,7 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `aiIndex` | none | extra agents to charge and refuse in robots.txt. Since 0.10 the AI answer-engine indexers (`AI_INDEX_AGENTS`: meta-webindexer, YouBot, ExaSearchBot, AionBot) are welcome retrieval crawlers; pass `aiIndex: AI_INDEX_AGENTS` to charge them anyway |
 | `verifiedRetrieval` | `VERIFIED_RETRIEVAL` | retrieval crawlers believed only from their operator's published addresses (OAI-SearchBot, ChatGPT-User, PerplexityBot, Perplexity-User); from elsewhere they are metered. Other retrieval and AI chat fetchers (Claude-User, Claude-SearchBot, Meta-ExternalFetcher, DuckAssistBot, MistralAI-User, the AI indexers) publish no addresses and are believed by name |
 | `chargeImpostors` | `true` | a request wearing a `welcome` crawler's name (Googlebot, bingbot, Applebot, DuckDuckBot, Cloudflare-*) from an address outside that operator's published list is charged. Never judged without a readable address |
-| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js' }`: every GET to these paths must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |

package/index.d.ts

@@ -77,7 +77,8 @@ export interface GatewayOptions {
   /** Charge a request wearing a welcome search crawler's name from outside its published ranges. Default true. */
   chargeImpostors?: boolean;
   /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
-  challenge?: { paths: string[]; cookie?: string } | false;
+  /** Only GETs to `paths` that `when` accepts (default: all of them), e.g. `(req, url) => url.search.length > 1` for faceted URLs. */
+  challenge?: { paths: string[]; cookie?: string; when?: (request: Request, url: URL) => boolean } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
   scrapers?: string[];
   /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/index.js

@@ -83,7 +83,7 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {string[]} [options.aiIndex]             extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS; default none (they are welcome retrieval crawlers since 0.10)
  * @param {{token: string, cidrs: string[]}[]|false} [options.verifiedRetrieval]  retrieval crawlers believed only from their published addresses; default VERIFIED_RETRIEVAL
  * @param {boolean} [options.chargeImpostors=true]  charge a request wearing a welcome search crawler's name from outside its published ranges
- * @param {{paths: string[], cookie?: string}|false} [options.challenge]  browser-looking requests to these paths must run a one-line script (cookie + reload) first; off by default
+ * @param {{paths: string[], cookie?: string, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
@@ -460,7 +460,13 @@ export function createGateway(options = {}) {
      * that asked for HTML gets the one-line script; anything else gets the
      * JSON offer, since it was never going to run a script.
      */
-    if (o.challenge && secret && meters(o.challenge, path) && (request.method === 'GET' || request.method === 'HEAD')) {
+    if (
+      o.challenge &&
+      secret &&
+      meters(o.challenge, path) &&
+      (!o.challenge.when || o.challenge.when(request, new URL(request.url))) &&
+      (request.method === 'GET' || request.method === 'HEAD')
+    ) {
       const ip = clientIp(request);
       if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
         if (!wantsHtml(request.headers.get('accept'))) {
@@ -622,7 +628,11 @@ function normalise(options) {
     aiIndex,
     challenge:
       options.challenge && Array.isArray(options.challenge.paths) && options.challenge.paths.length
-        ? { paths: options.challenge.paths, cookie: options.challenge.cookie ?? 'pf_js' }
+        ? {
+            paths: options.challenge.paths,
+            cookie: options.challenge.cookie ?? 'pf_js',
+            when: typeof options.challenge.when === 'function' ? options.challenge.when : null,
+          }
         : null,
     welcome: options.welcome === false ? null : (options.welcome ?? WELCOME_CRAWLERS),
     verifiedRetrieval: options.verifiedRetrieval === false ? null : (options.verifiedRetrieval ?? VERIFIED_RETRIEVAL),