@profullstack/x402-gateway 0.11.0 → 0.12.0

package/README.md

@@ -95,7 +95,7 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `aiIndex` | none | extra agents to charge and refuse in robots.txt. Since 0.10 the AI answer-engine indexers (`AI_INDEX_AGENTS`: meta-webindexer, YouBot, ExaSearchBot, AionBot) are welcome retrieval crawlers; pass `aiIndex: AI_INDEX_AGENTS` to charge them anyway |
 | `verifiedRetrieval` | `VERIFIED_RETRIEVAL` | retrieval crawlers believed only from their operator's published addresses (OAI-SearchBot, ChatGPT-User, PerplexityBot, Perplexity-User); from elsewhere they are metered. Other retrieval and AI chat fetchers (Claude-User, Claude-SearchBot, Meta-ExternalFetcher, DuckAssistBot, MistralAI-User, the AI indexers) publish no addresses and are believed by name |
 | `chargeImpostors` | `true` | a request wearing a `welcome` crawler's name (Googlebot, bingbot, Applebot, DuckDuckBot, Cloudflare-*) from an address outside that operator's published list is charged. Never judged without a readable address |
-| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', work?: 0, when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page. Set `work` (bits; `true` = 16, max 24) to make the script also solve a SHA-256 proof of work before it sets the cookie (`<token>.<nonce>`): ~250 ms at 16 bits on a desktop, once per address per day, but per page for a swarm that drives real headless Chrome and rotates its address every page. `navigator.webdriver` browsers are not given the cookie. |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |

package/index.d.ts

@@ -105,7 +105,7 @@ export interface GatewayOptions {
   chargeImpostors?: boolean;
   /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
   /** Only GETs to `paths` that `when` accepts (default: all of them), e.g. `(req, url) => url.search.length > 1` for faceted URLs. */
-  challenge?: { paths: string[]; cookie?: string; when?: (request: Request, url: URL) => boolean } | false;
+  challenge?: { paths: string[]; cookie?: string; /** Proof-of-work bits the script must solve; true = 16. Default 0 (cookie + reload only). */ work?: number | boolean; when?: (request: Request, url: URL) => boolean } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
   scrapers?: string[];
   /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */
@@ -276,8 +276,10 @@ export const SCRAPER_AGENTS: string[];
 export const HEADLESS_AGENTS: string[];
 export const AI_INDEX_AGENTS: string[];
 export function challengeToken(secret: string, ip: string, day?: number): Promise<string>;
-export function passedChallenge(request: Request, options: { secret: string; cookie: string; ip: string; now?: number }): Promise<boolean>;
-export function challengePage(options: { token: string; cookie: string; buyUrl: string }): string;
+export function passedChallenge(request: Request, options: { secret: string; cookie: string; ip: string; work?: number; now?: number }): Promise<boolean>;
+export function challengePage(options: { token: string; cookie: string; buyUrl: string; work?: number }): string;
+export function solves(seed: string, nonce: string | undefined, work: number): Promise<boolean>;
+export function leadingZeros(bytes: Uint8Array, bits: number): boolean;
 export function cookieValue(request: Request, name: string): string | null;
 export interface WelcomeCrawler {
   token: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/challenge.js

@@ -14,6 +14,15 @@
  * address. The page that asks is a 402 carrying a link to the sales page, so a
  * scraper that cannot run scripts learns what a pass costs, which is the same
  * answer every other refused crawler gets.
+ *
+ * That was not enough. On 2026-10-21 the slice of the swarm that drives a real
+ * Chrome ran the script and walked in: 402, then the same address in the same
+ * second, then a 200, then never again. It rotates its address on every page,
+ * so "once per address per day" is once per page and the script cost it one
+ * request. With `work` set the script must also find a nonce whose SHA-256
+ * starts with that many zero bits. A person pays about a second once a day; a
+ * swarm that changes address every page pays it on every page. The server
+ * checks the answer with one hash and keeps nothing.
  */
 
 const enc = new TextEncoder();
@@ -42,19 +51,48 @@ export function cookieValue(request, name) {
   return null;
 }
 
-/** Whether the request carries today's or yesterday's token for its address. */
-export async function passedChallenge(request, { secret, cookie, ip, now = Date.now() }) {
+/** Whether a SHA-256 digest starts with `bits` zero bits. */
+export function leadingZeros(bytes, bits) {
+  let i = 0;
+  for (; bits >= 8; bits -= 8) if (bytes[i++]) return false;
+  return !bits || !(bytes[i] >> (8 - bits));
+}
+
+/** Whether `nonce` answers the proof of work for `seed`. */
+export async function solves(seed, nonce, work) {
+  if (!work) return true;
+  if (!/^\d{1,12}$/.test(nonce ?? '')) return false;
+  const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', enc.encode(`${seed}.${nonce}`)));
+  return leadingZeros(digest, work);
+}
+
+/**
+ * Whether the request carries today's or yesterday's token for its address,
+ * and, with `work`, a nonce that answers it (cookie `<token>.<nonce>`).
+ */
+export async function passedChallenge(request, { secret, cookie, ip, work = 0, now = Date.now() }) {
   const got = cookieValue(request, cookie);
   if (!got) return false;
+  const [seed, nonce] = got.split('.');
   const day = Math.floor(now / DAY);
-  for (const d of [day, day - 1]) if (got === (await challengeToken(secret, ip, d))) return true;
+  for (const d of [day, day - 1]) if (seed === (await challengeToken(secret, ip, d))) return solves(seed, nonce, work);
   return false;
 }
 
+/** The script that finds the nonce, 256 digests at a time, then sets the cookie and reloads. */
+const solver = (token, cookie, work) => `(async()=>{const s=${JSON.stringify(token)},w=${work},e=new TextEncoder();
+if(navigator.webdriver){document.body.insertAdjacentHTML('beforeend','<p>Automated browsers need a pass.</p>');return}
+const ok=b=>{let n=w,i=0;for(;n>=8;n-=8)if(b[i++])return 0;return!n||!(b[i]>>(8-n))};
+for(let n=0;;n+=256){const h=await Promise.all(Array.from({length:256},(_,k)=>crypto.subtle.digest('SHA-256',e.encode(s+'.'+(n+k)))));
+for(let k=0;k<256;k++)if(ok(new Uint8Array(h[k]))){document.cookie=${JSON.stringify(`${cookie}=`)}+s+'.'+(n+k)+${JSON.stringify('; Max-Age=172800; Path=/; SameSite=Lax; Secure')};location.reload();return}}})();`;
+
 /** The page that asks. Tiny on purpose: answering it must cost us nothing. */
-export function challengePage({ token, cookie, buyUrl }) {
+export function challengePage({ token, cookie, buyUrl, work = 0 }) {
+  const script = work
+    ? solver(token, cookie, work)
+    : `document.cookie=${JSON.stringify(`${cookie}=${token}; Max-Age=172800; Path=/; SameSite=Lax; Secure`)};location.reload();`;
   return `<!doctype html><html><head><meta charset="utf-8"><meta name="robots" content="noindex"><title>One moment</title></head><body>
-<script>document.cookie=${JSON.stringify(`${cookie}=${token}; Max-Age=172800; Path=/; SameSite=Lax; Secure`)};location.reload();</script>
+${work ? '<p>Checking your browser. This takes about a second, once a day.</p>\n' : ''}<script>${script}</script>
 <noscript><p>This page needs JavaScript. Automated access is sold by the day: <a href="${buyUrl}">${buyUrl}</a></p></noscript>
 </body></html>`;
 }

package/src/index.js

@@ -41,7 +41,7 @@ export {
   parseCidr,
   welcomeCrawler,
 } from './edge.js';
-export { challengePage, challengeToken, cookieValue, passedChallenge } from './challenge.js';
+export { challengePage, challengeToken, cookieValue, leadingZeros, passedChallenge, solves } from './challenge.js';
 export { renderNotice, renderPage, renderPassPage } from './page.js';
 export { checkoutClient, sessionIdFromWebhook, sessionState } from './checkout.js';
 export { mintPass, readPass } from './pass.js';
@@ -97,7 +97,7 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {string[]} [options.aiIndex]             extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS; default none (they are welcome retrieval crawlers since 0.10)
  * @param {{token: string, cidrs: string[]}[]|false} [options.verifiedRetrieval]  retrieval crawlers believed only from their published addresses; default VERIFIED_RETRIEVAL
  * @param {boolean} [options.chargeImpostors=true]  charge a request wearing a welcome search crawler's name from outside its published ranges
- * @param {{paths: string[], cookie?: string, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; off by default
+ * @param {{paths: string[], cookie?: string, work?: number|boolean, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; with `work` (bits, `true` = 16) the script must also solve a SHA-256 proof of work, so a swarm that rotates its address every page pays it on every page; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
@@ -830,7 +830,7 @@ export function createGateway(options = {}) {
       (request.method === 'GET' || request.method === 'HEAD')
     ) {
       const ip = clientIp(request);
-      if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
+      if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip, work: o.challenge.work }))) {
         if (!wantsHtml(request.headers.get('accept'))) {
           return json(
             receipt(1, {
@@ -841,7 +841,7 @@ export function createGateway(options = {}) {
           );
         }
         const token = await challengeToken(secret, ip);
-        return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl }), 402, required(1));
+        return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl, work: o.challenge.work }), 402, required(1));
       }
     }
 
@@ -1015,6 +1015,7 @@ function normalise(options) {
         ? {
             paths: options.challenge.paths,
             cookie: options.challenge.cookie ?? 'pf_js',
+            work: challengeWork(options.challenge.work),
             when: typeof options.challenge.when === 'function' ? options.challenge.when : null,
           }
         : null,
@@ -1040,4 +1041,10 @@ function normalise(options) {
   };
 }
 
+/** `true` means the default 16 bits; a number is clamped to 0-24 (24 is ~16M digests, too slow for a phone). */
+function challengeWork(work) {
+  if (work === true) return 16;
+  return Number.isInteger(work) ? Math.max(0, Math.min(24, work)) : 0;
+}
+
 export { METHODS as X402_METHODS };