@profullstack/x402-gateway 0.11.0 → 0.13.0

package/README.md

@@ -95,7 +95,7 @@ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
 | `aiIndex` | none | extra agents to charge and refuse in robots.txt. Since 0.10 the AI answer-engine indexers (`AI_INDEX_AGENTS`: meta-webindexer, YouBot, ExaSearchBot, AionBot) are welcome retrieval crawlers; pass `aiIndex: AI_INDEX_AGENTS` to charge them anyway |
 | `verifiedRetrieval` | `VERIFIED_RETRIEVAL` | retrieval crawlers believed only from their operator's published addresses (OAI-SearchBot, ChatGPT-User, PerplexityBot, Perplexity-User); from elsewhere they are metered. Other retrieval and AI chat fetchers (Claude-User, Claude-SearchBot, Meta-ExternalFetcher, DuckAssistBot, MistralAI-User, the AI indexers) publish no addresses and are believed by name |
 | `chargeImpostors` | `true` | a request wearing a `welcome` crawler's name (Googlebot, bingbot, Applebot, DuckDuckBot, Cloudflare-*) from an address outside that operator's published list is charged. Never judged without a readable address |
-| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page |
+| `challenge` | off | `{ paths: ['/i/'], cookie?: 'pf_js', work?: 0, when?: (request, url) => boolean }`: every GET to these paths (that `when` accepts, if given) must run a one-line script (sets a per-address, per-day HMAC cookie and reloads) or gets a 402. Welcome crawlers (address-verified search crawlers, retrieval crawlers, AI chat fetchers) skip it; a client that did not ask for HTML gets the JSON offer instead of the script; sessions and passes never see it. For residential swarms that wear a current browser string and never run the page. Set `work` (bits; `true` = 16, max 24) to make the script also solve a SHA-256 proof of work before it sets the cookie (`<token>.<nonce>`): ~250 ms at 16 bits on a desktop, once per address per day, but per page for a swarm that drives real headless Chrome and rotates its address every page. `navigator.webdriver` browsers are not given the cookie. Set `css: true` (path `cssPath`, default `/__pf/c/`) to also require a second cookie that only fetching a per-address stylesheet linked from the page sets: scrapers that block stylesheets to save bandwidth never pass (nichedb, 2026-10-21: 11 of 5,394 swarm addresses ever loaded CSS). |
 | `chargeStaleBrowsers` | `true` | charge a desktop Chrome/Edge/Firefox string about two years behind its auto-updates. Scraper rotation pools copy old strings; people's browsers update themselves. Phones, Firefox ESRs and the last version an old OS can run are never judged |
 | `welcome` | `WELCOME_CRAWLERS` | Googlebot, bingbot and Applebot read unmetered when their address is in the operator's published range. A copied string from anywhere else is metered like everyone; `false` turns this off |
 | `training`, `retrieval` | the lists in `./agents` | |

package/index.d.ts

@@ -105,7 +105,7 @@ export interface GatewayOptions {
   chargeImpostors?: boolean;
   /** Require a one-line script (cookie + reload) on these paths before serving them. Off by default. */
   /** Only GETs to `paths` that `when` accepts (default: all of them), e.g. `(req, url) => url.search.length > 1` for faceted URLs. */
-  challenge?: { paths: string[]; cookie?: string; when?: (request: Request, url: URL) => boolean } | false;
+  challenge?: { paths: string[]; cookie?: string; /** Proof-of-work bits the script must solve; true = 16. Default 0 (cookie + reload only). */ work?: number | boolean; /** Link a per-address stylesheet whose fetch sets `<cookie>_css`; both cookies are then required. */ css?: boolean; /** Where that stylesheet is served. Default '/__pf/c/'. */ cssPath?: string; when?: (request: Request, url: URL) => boolean } | false;
   /** SEO resellers: charged, and refused in robots.txt. Default SCRAPER_AGENTS. */
   scrapers?: string[];
   /** Headless browsers that name themselves: charged. Default HEADLESS_AGENTS. */
@@ -276,8 +276,12 @@ export const SCRAPER_AGENTS: string[];
 export const HEADLESS_AGENTS: string[];
 export const AI_INDEX_AGENTS: string[];
 export function challengeToken(secret: string, ip: string, day?: number): Promise<string>;
-export function passedChallenge(request: Request, options: { secret: string; cookie: string; ip: string; now?: number }): Promise<boolean>;
-export function challengePage(options: { token: string; cookie: string; buyUrl: string }): string;
+export function passedChallenge(request: Request, options: { secret: string; cookie: string; ip: string; work?: number; css?: boolean; now?: number }): Promise<boolean>;
+export function challengePage(options: { token: string; cookie: string; buyUrl: string; work?: number; cssPath?: string | null }): string;
+export function stylesheetToken(secret: string, ip: string, day?: number): Promise<string>;
+export function stylesheetResponse(request: Request, options: { secret: string; cookie: string; ip: string | null; cssPath: string; now?: number }): Promise<Response>;
+export function solves(seed: string, nonce: string | undefined, work: number): Promise<boolean>;
+export function leadingZeros(bytes: Uint8Array, bits: number): boolean;
 export function cookieValue(request: Request, name: string): string | null;
 export interface WelcomeCrawler {
   token: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/x402-gateway",
-  "version": "0.11.0",
+  "version": "0.13.0",
   "type": "module",
   "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, Profullstack Power Keys honoured as a pass, and a robots.txt that keeps search crawlers welcome.",
   "keywords": [

package/src/challenge.js

@@ -14,6 +14,24 @@
  * address. The page that asks is a 402 carrying a link to the sales page, so a
  * scraper that cannot run scripts learns what a pass costs, which is the same
  * answer every other refused crawler gets.
+ *
+ * That was not enough. On 2026-10-21 the slice of the swarm that drives a real
+ * Chrome ran the script and walked in: 402, then the same address in the same
+ * second, then a 200, then never again. It rotates its address on every page,
+ * so "once per address per day" is once per page and the script cost it one
+ * request. With `work` set the script must also find a nonce whose SHA-256
+ * starts with that many zero bits. A person pays about a second once a day; a
+ * swarm that changes address every page pays it on every page. The server
+ * checks the answer with one hash and keeps nothing.
+ *
+ * The proof of work did not slow it either (~46 item pages a minute before,
+ * ~40 after): its Chrome is real and solves 17 bits easily. But it never
+ * loads the page's stylesheet. Of 5,394 addresses that read an item page as
+ * a browser, 11 fetched any CSS or JS; a real Chrome fetches the stylesheet
+ * on every first visit. With `css` set the page links a stylesheet unique to
+ * the address and day, and fetching it is what sets the second cookie. A
+ * scraper that blocks stylesheets to save bandwidth never gets it, however
+ * much script it runs.
  */
 
 const enc = new TextEncoder();
@@ -32,6 +50,17 @@ async function hmac(secret, data) {
 /** The token a browser at `ip` gets on day `day` (days since the epoch). */
 export const challengeToken = (secret, ip, day = Math.floor(Date.now() / DAY)) => hmac(secret, `js|${ip}|${day}`);
 
+/** The second cookie, set only by fetching the challenge stylesheet. */
+export const stylesheetToken = (secret, ip, day = Math.floor(Date.now() / DAY)) => hmac(secret, `css|${ip}|${day}`);
+
+/** Whether `got` is today's or yesterday's value of `mint(day)`. */
+async function current(got, mint, now) {
+  if (!got) return false;
+  const day = Math.floor(now / DAY);
+  for (const d of [day, day - 1]) if (got === (await mint(d))) return true;
+  return false;
+}
+
 /** Read one cookie by name. */
 export function cookieValue(request, name) {
   const raw = request.headers.get('cookie') ?? '';
@@ -42,19 +71,71 @@ export function cookieValue(request, name) {
   return null;
 }
 
-/** Whether the request carries today's or yesterday's token for its address. */
-export async function passedChallenge(request, { secret, cookie, ip, now = Date.now() }) {
+/** Whether a SHA-256 digest starts with `bits` zero bits. */
+export function leadingZeros(bytes, bits) {
+  let i = 0;
+  for (; bits >= 8; bits -= 8) if (bytes[i++]) return false;
+  return !bits || !(bytes[i] >> (8 - bits));
+}
+
+/** Whether `nonce` answers the proof of work for `seed`. */
+export async function solves(seed, nonce, work) {
+  if (!work) return true;
+  if (!/^\d{1,12}$/.test(nonce ?? '')) return false;
+  const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', enc.encode(`${seed}.${nonce}`)));
+  return leadingZeros(digest, work);
+}
+
+/**
+ * Whether the request carries today's or yesterday's token for its address,
+ * and, with `work`, a nonce that answers it (cookie `<token>.<nonce>`), and,
+ * with `css`, the cookie the challenge stylesheet sets (`<cookie>_css`).
+ */
+export async function passedChallenge(request, { secret, cookie, ip, work = 0, css = false, now = Date.now() }) {
   const got = cookieValue(request, cookie);
   if (!got) return false;
-  const day = Math.floor(now / DAY);
-  for (const d of [day, day - 1]) if (got === (await challengeToken(secret, ip, d))) return true;
-  return false;
+  const [seed, nonce] = got.split('.');
+  if (!(await current(seed, (d) => challengeToken(secret, ip, d), now))) return false;
+  if (!(await solves(seed, nonce, work))) return false;
+  if (!css) return true;
+  return current(cookieValue(request, `${cookie}_css`), (d) => stylesheetToken(secret, ip, d), now);
 }
 
+/**
+ * The challenge stylesheet: `<cssPath><token>`. Sets `<cookie>_css` when the
+ * token in the URL is today's or yesterday's for the caller's address, so the
+ * URL cannot be fetched on behalf of another address. Always answers CSS, so a
+ * browser never logs a broken stylesheet.
+ */
+export async function stylesheetResponse(request, { secret, cookie, ip, cssPath, now = Date.now() }) {
+  const token = new URL(request.url).pathname.slice(cssPath.length);
+  const headers = new Headers({ 'content-type': 'text/css; charset=utf-8', 'cache-control': 'no-store' });
+  if (ip && /^[0-9a-f]{32}$/.test(token) && (await current(token, (d) => challengeToken(secret, ip, d), now))) {
+    const value = await stylesheetToken(secret, ip);
+    headers.append('set-cookie', `${cookie}_css=${value}; Max-Age=172800; Path=/; SameSite=Lax; Secure; HttpOnly`);
+  }
+  return new Response('/* ok */\n', { status: 200, headers });
+}
+
+/** Wait for the challenge stylesheet; if it cannot load, say so rather than reload forever. */
+const WAIT_CSS = `const l=document.getElementById('pfc');if(!(l.sheet||await new Promise(r=>{l.onload=()=>r(1);l.onerror=()=>r(0)}))){document.body.insertAdjacentHTML('beforeend','<p>This check needs the page stylesheet. Reload, or buy a pass.</p>');return}
+`;
+
+/** The script that finds the nonce, 256 digests at a time, then sets the cookie and reloads. */
+const solver = (token, cookie, work, css) => `(async()=>{const s=${JSON.stringify(token)},w=${work},e=new TextEncoder();${css ? WAIT_CSS : ''}
+if(navigator.webdriver){document.body.insertAdjacentHTML('beforeend','<p>Automated browsers need a pass.</p>');return}
+const ok=b=>{let n=w,i=0;for(;n>=8;n-=8)if(b[i++])return 0;return!n||!(b[i]>>(8-n))};
+for(let n=0;;n+=256){const h=await Promise.all(Array.from({length:256},(_,k)=>crypto.subtle.digest('SHA-256',e.encode(s+'.'+(n+k)))));
+for(let k=0;k<256;k++)if(ok(new Uint8Array(h[k]))){document.cookie=${JSON.stringify(`${cookie}=`)}+s+'.'+(n+k)+${JSON.stringify('; Max-Age=172800; Path=/; SameSite=Lax; Secure')};location.reload();return}}})();`;
+
 /** The page that asks. Tiny on purpose: answering it must cost us nothing. */
-export function challengePage({ token, cookie, buyUrl }) {
-  return `<!doctype html><html><head><meta charset="utf-8"><meta name="robots" content="noindex"><title>One moment</title></head><body>
-<script>document.cookie=${JSON.stringify(`${cookie}=${token}; Max-Age=172800; Path=/; SameSite=Lax; Secure`)};location.reload();</script>
+export function challengePage({ token, cookie, buyUrl, work = 0, cssPath = null }) {
+  const css = Boolean(cssPath);
+  const script = work || css
+    ? solver(token, cookie, work, css)
+    : `document.cookie=${JSON.stringify(`${cookie}=${token}; Max-Age=172800; Path=/; SameSite=Lax; Secure`)};location.reload();`;
+  return `<!doctype html><html><head><meta charset="utf-8"><meta name="robots" content="noindex"><title>One moment</title>${css ? `<link id="pfc" rel="stylesheet" href="${cssPath}${token}">` : ''}</head><body>
+${work || css ? '<p>Checking your browser. This takes about a second, once a day.</p>\n' : ''}<script>${script}</script>
 <noscript><p>This page needs JavaScript. Automated access is sold by the day: <a href="${buyUrl}">${buyUrl}</a></p></noscript>
 </body></html>`;
 }

package/src/index.js

@@ -2,7 +2,7 @@ import { HEADER as POWER_KEY_HEADER, isPowerKey, keyFrom, PROFULLSTACK_KEYS, SHO
 import { HEADLESS_AGENTS, isTrainingAgent, RETRIEVAL_AGENTS, SCRAPER_AGENTS, TRAINING_AGENTS, VERIFIED_RETRIEVAL, WELCOME_CRAWLERS } from './agents.js';
 import { clientIp, compileCidrs, impostorCrawler, inCidrs, isSpoofedBrowser, isStaleBrowser, welcomeCrawler } from './edge.js';
 import { memoryQuotaStore, meters, normaliseQuota, quotaHeaders, spend } from './quota.js';
-import { challengePage, challengeToken, cookieValue, passedChallenge } from './challenge.js';
+import { challengePage, challengeToken, cookieValue, passedChallenge, stylesheetResponse } from './challenge.js';
 import { checkoutClient, isEmail, isSessionId, sessionIdFromWebhook, sessionState } from './checkout.js';
 import { renderNotice, renderPage, renderPassPage } from './page.js';
 import { mintPass, readPass } from './pass.js';
@@ -41,7 +41,16 @@ export {
   parseCidr,
   welcomeCrawler,
 } from './edge.js';
-export { challengePage, challengeToken, cookieValue, passedChallenge } from './challenge.js';
+export {
+  challengePage,
+  challengeToken,
+  cookieValue,
+  leadingZeros,
+  passedChallenge,
+  solves,
+  stylesheetResponse,
+  stylesheetToken,
+} from './challenge.js';
 export { renderNotice, renderPage, renderPassPage } from './page.js';
 export { checkoutClient, sessionIdFromWebhook, sessionState } from './checkout.js';
 export { mintPass, readPass } from './pass.js';
@@ -97,7 +106,7 @@ export { isPowerKey, keyFrom, PROFULLSTACK_KEYS, verifyKey } from '@profullstack
  * @param {string[]} [options.aiIndex]             extra agents to charge and refuse in robots.txt, e.g. AI_INDEX_AGENTS; default none (they are welcome retrieval crawlers since 0.10)
  * @param {{token: string, cidrs: string[]}[]|false} [options.verifiedRetrieval]  retrieval crawlers believed only from their published addresses; default VERIFIED_RETRIEVAL
  * @param {boolean} [options.chargeImpostors=true]  charge a request wearing a welcome search crawler's name from outside its published ranges
- * @param {{paths: string[], cookie?: string, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; off by default
+ * @param {{paths: string[], cookie?: string, work?: number|boolean, css?: boolean, cssPath?: string, when?: (request: Request, url: URL) => boolean}|false} [options.challenge]  requests to these paths (and, with `when`, only those it accepts, e.g. faceted URLs) must run a one-line script (cookie + reload) first; with `work` (bits, `true` = 16) the script must also solve a SHA-256 proof of work, so a swarm that rotates its address every page pays it on every page; with `css` the page links a per-address stylesheet (at `cssPath`, default `/__pf/c/`) whose fetch sets a second cookie, so a scraper that blocks stylesheets never passes; off by default
  * @param {{token: string, cidrs: string[]}[]|false} [options.welcome]  search crawlers never metered when name and address agree; default WELCOME_CRAWLERS
  * @param {(request: Request) => boolean} [options.exempt]  requests never charged, e.g. ones carrying a signed-in cookie
  * @param {string} [options.secret]                pass signing secret; defaults to the CoinPay key
@@ -789,6 +798,10 @@ export function createGateway(options = {}) {
       const answer = await checkoutRoute(request, path);
       if (answer) return answer;
     }
+    // The challenge stylesheet answers everyone: fetching it is the test.
+    if (o.challenge?.cssPath && secret && path.startsWith(o.challenge.cssPath)) {
+      return stylesheetResponse(request, { secret, cookie: o.challenge.cookie, ip: clientIp(request), cssPath: o.challenge.cssPath });
+    }
     if (o.exempt && o.exempt(request)) return null;
 
     /*
@@ -830,7 +843,7 @@ export function createGateway(options = {}) {
       (request.method === 'GET' || request.method === 'HEAD')
     ) {
       const ip = clientIp(request);
-      if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip }))) {
+      if (ip && !(await passedChallenge(request, { secret, cookie: o.challenge.cookie, ip, work: o.challenge.work, css: Boolean(o.challenge.cssPath) }))) {
         if (!wantsHtml(request.headers.get('accept'))) {
           return json(
             receipt(1, {
@@ -841,7 +854,7 @@ export function createGateway(options = {}) {
           );
         }
         const token = await challengeToken(secret, ip);
-        return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl }), 402, required(1));
+        return html(challengePage({ token, cookie: o.challenge.cookie, buyUrl, work: o.challenge.work, cssPath: o.challenge.cssPath }), 402, required(1));
       }
     }
 
@@ -1015,6 +1028,8 @@ function normalise(options) {
         ? {
             paths: options.challenge.paths,
             cookie: options.challenge.cookie ?? 'pf_js',
+            work: challengeWork(options.challenge.work),
+            cssPath: options.challenge.css ? (options.challenge.cssPath ?? '/__pf/c/') : null,
             when: typeof options.challenge.when === 'function' ? options.challenge.when : null,
           }
         : null,
@@ -1040,4 +1055,10 @@ function normalise(options) {
   };
 }
 
+/** `true` means the default 16 bits; a number is clamped to 0-24 (24 is ~16M digests, too slow for a phone). */
+function challengeWork(work) {
+  if (work === true) return 16;
+  return Number.isInteger(work) ? Math.max(0, Math.min(24, work)) : 0;
+}
+
 export { METHODS as X402_METHODS };