agent402-tollbooth 0.1.3 → 0.2.0

package/README.md

@@ -92,27 +92,41 @@ The same gate is also built on the Web Crypto + Fetch APIs (`edge.js`), so it ru
 anywhere — no Node required. The gate returns a `402 Response` when the client
 must pay, or `null` to let it through.
 
-**Cloudflare Workers** (poetic: the open pay-per-crawl, on the incumbent's own platform). `wrangler.toml`:
+**Ready-to-deploy templates** (copy a folder, don't assemble from docs):
+
+- **Cloudflare Workers** → [`deploy/cloudflare/`](deploy/cloudflare/) — a ready
+  `wrangler.toml` + a 3-step deploy guide (the open pay-per-crawl, on the
+  incumbent's own platform).
+- **Next.js / Vercel** → [`deploy/nextjs/`](deploy/nextjs/) — a drop-in
+  `middleware.js` + a 3-step deploy guide.
+- **Docker** → [`deploy/docker/`](deploy/docker/) — a `Dockerfile` +
+  `docker-compose.yml` to run the reverse proxy in front of any site with
+  `docker compose up -d` (includes the live `/__tollbooth` dashboard).
+
+The short version of each:
 
 ```toml
+# wrangler.toml  (full template: deploy/cloudflare/wrangler.toml)
 name = "tollbooth"
 main = "node_modules/agent402-tollbooth/worker.js"
+compatibility_date = "2026-01-01"
 [vars]
 TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
 TOLLBOOTH_PAYTO    = "0xYourWallet"   # optional: advertise a USDC x402 quote
-# wrangler secret put TOLLBOOTH_SECRET
+# npx wrangler secret put TOLLBOOTH_SECRET
 # optional single-use replay store:  [[kv_namespaces]] binding = "TOLLBOOTH_KV"
 ```
 
-**Next.js middleware** (`middleware.ts`):
-
-```ts
+```js
+// middleware.js  (full template: deploy/nextjs/middleware.js)
+import { NextResponse } from "next/server";
 import { createEdgeTollbooth } from "agent402-tollbooth/edge";
-const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET! });
+const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET });
 
-export async function middleware(req: Request) {
+export async function middleware(req) {
   return (await gate(req)) ?? NextResponse.next();
 }
+export const config = { matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"] };
 ```
 
 **Any Fetch-API runtime** (Deno, Bun, custom): `const gate = createEdgeTollbooth({ secret }); const blocked = await gate(request); return blocked ?? fetch(request);`
@@ -152,23 +166,46 @@ app.use(createTollbooth({
 | `network` | `"base"` | x402 network |
 | `pow` | `true` | Enable the free proof-of-work rail |
 | `powDifficulty` | `18` | PoW difficulty in leading zero bits (~0.1–0.5s of CPU) |
-| `botUserAgents` | `AI_BOTS` | User-agents to charge (AI/LLM crawlers; search indexers excluded) |
-| `charge(req)` | UA match | Custom predicate for "should this client pay?" |
-| `free(req)` | – | Custom force-allow predicate (wins over `charge`) |
+| `mode` | `"bots"` | Who pays: `"bots"` (AI-crawler UAs) · `"all"` (everyone but `free()`) · `"strict"` (anything that isn't a real-browser request) |
+| `adaptive` | `false` | Raise PoW difficulty as charged-request load climbs (anti-abuse under traffic spikes) |
+| `maxDifficulty` | `base+6` | Ceiling for adaptive difficulty |
+| `adaptivePerBit` | `300` | +1 difficulty bit per N charged requests/min |
+| `botUserAgents` | `AI_BOTS` | User-agents to charge in `"bots"` mode |
+| `charge(req)` | mode | Custom "should this client pay?" predicate (wins over `mode`) |
+| `free(req)` | – | Custom force-allow predicate (wins over everything) |
 | `verifyX402(req, reqs)` | – | Async USDC settlement check (return `true` to allow) |
 | `resourceBaseUrl` | `""` | Absolute base used for the `resource` field / PoW binding |
 
 Environment variables: `TOLLBOOTH_UPSTREAM`, `TOLLBOOTH_PAYTO`, `TOLLBOOTH_PRICE`,
-`TOLLBOOTH_NETWORK`, `TOLLBOOTH_POW_BITS`, `TOLLBOOTH_SECRET`, `PORT`.
+`TOLLBOOTH_NETWORK`, `TOLLBOOTH_POW_BITS`, `TOLLBOOTH_MODE`, `TOLLBOOTH_ADAPTIVE`,
+`TOLLBOOTH_MAX_POW_BITS`, `TOLLBOOTH_ADAPTIVE_PER_BIT`, `TOLLBOOTH_SECRET`, `PORT`.
 
 ## How it decides who pays
 
-By default it charges requests whose `User-Agent` matches a known **AI/LLM
-crawler** (GPTBot, ClaudeBot, CCBot, PerplexityBot, Bytespider, Google-Extended,
-Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are intentionally
-**not** charged so your SEO indexing stays free. Override with `botUserAgents`,
-or take full control with a `charge(req)` predicate (e.g. charge anything
-without a browser-like `Accept` header, or charge everyone on `/api/*`).
+By default (`mode: "bots"`) it charges requests whose `User-Agent` matches a known
+**AI/LLM crawler** (GPTBot, ClaudeBot, CCBot, PerplexityBot, Bytespider,
+Google-Extended, Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are
+intentionally **not** charged so your SEO indexing stays free.
+
+**Don't want to play whack-a-mole with bot detection?** That's the point of the
+other modes — you stop trying to *identify* bots and instead make access *cost
+something*:
+- `mode: "all"` charges every client (except a `free()` match). A "more
+  sophisticated" bot gains nothing by disguising itself — everyone pays or solves
+  a proof-of-work.
+- `mode: "strict"` charges anything that isn't a real-browser request (browser-like
+  UA **and** an HTML `Accept`), letting genuine human page-loads through free.
+- `adaptive: true` makes proof-of-work **harder as load climbs**, so a high-volume
+  scraper pays escalating CPU per request regardless of how it looks — detection is
+  cat-and-mouse, economics isn't.
+
+## Analytics
+
+The middleware keeps aggregate counters (no per-request data): `gate.stats()`
+returns `{ requests, freeAllowed, charged, powSolved, x402Paid, difficultyNow }`.
+The reverse-proxy CLI exposes them as JSON at **`/__tollbooth/stats`** and as a
+live **dashboard at `/__tollbooth`** — requests, how many were charged,
+proof-of-work solves, USDC collected, and what share of your traffic is bots.
 
 ## Production checklist (read this)
 
@@ -183,8 +220,9 @@ without a browser-like `Accept` header, or charge everyone on `/api/*`).
   redirect it elsewhere) and **strips client-forged trust/forwarding headers**
   (`X-Tollbooth-Paid`, `X-Forwarded-Host`, etc.) before forwarding.
 - **UA matching is the default, not a security boundary** — a bot can forge a
-  human UA to get the *same free access a human gets* (it gains nothing more). If
-  you need to charge everyone, pass `charge: () => true`.
+  human UA to get the *same free access a human gets* (it gains nothing more). To
+  stop relying on detection entirely, use `mode: "all"` / `mode: "strict"`, and
+  turn on `adaptive` so high-volume abuse pays escalating proof-of-work.
 
 ## Notes
 

package/dashboard.js

@@ -0,0 +1,48 @@
+// A tiny self-contained operator dashboard for the tollbooth, served at
+// /__tollbooth by the reverse-proxy CLI. It polls /__tollbooth/stats and shows
+// how much of your traffic is bots and what the gate is collecting — the visual
+// answer to "I had to upgrade my plan to serve bots I don't monetize."
+// Pure function (no deps) so it's trivially testable.
+export function dashboardHtml() {
+  return `<!doctype html>
+<html lang="en"><head>
+<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<title>agent402-tollbooth — live stats</title>
+<style>
+  :root{--bg:#0b0e14;--card:#131826;--text:#e6e9f0;--muted:#8b93a7;--accent:#4ade80;--line:#1e2638;--mono:ui-monospace,SFMono-Regular,Menlo,monospace}
+  *{box-sizing:border-box;margin:0}body{background:var(--bg);color:var(--text);font:16px/1.5 system-ui,-apple-system,sans-serif}
+  .wrap{max-width:880px;margin:0 auto;padding:40px 20px}
+  h1{font-size:1.4rem;display:flex;align-items:center;gap:10px}
+  .badge{font:700 14px/1 var(--mono);color:var(--accent);background:#000;border:1px solid #1f4a1d;border-radius:8px;padding:8px 10px}
+  .sub{color:var(--muted);margin:6px 0 24px;font-size:.9rem}
+  .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:14px}
+  .card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:18px}
+  .k{color:var(--muted);font-size:.8rem;text-transform:uppercase;letter-spacing:.04em}
+  .v{font:800 1.8rem/1.1 var(--mono);margin-top:8px}
+  .v.accent{color:var(--accent)}
+  footer{color:var(--muted);font-size:.8rem;margin-top:28px}
+  a{color:var(--accent)}
+</style></head>
+<body><div class="wrap">
+  <h1><span class="badge">402</span> agent402-tollbooth</h1>
+  <div class="sub">Live pay-per-crawl stats · refreshes every 5s · <span id="since"></span></div>
+  <div class="grid" id="grid"></div>
+  <footer>Bots are <b id="botpct">—</b> of requests. Aggregate counts only (no per-request data). Raw JSON: <a href="/__tollbooth/stats">/__tollbooth/stats</a>.</footer>
+</div>
+<script>
+const cards=[["requests","Requests",false],["freeAllowed","Humans (free)",false],["charged","Charged (402)","accent"],["powSolved","Proof-of-work paid","accent"],["x402Paid","USDC paid (x402)","accent"],["difficultyNow","PoW difficulty",false]];
+async function tick(){
+  try{
+    const s=await (await fetch("/__tollbooth/stats",{cache:"no-store"})).json();
+    document.getElementById("grid").innerHTML=cards.map(([k,label,acc])=>
+      '<div class="card"><div class="k">'+label+'</div><div class="v '+(acc||"")+'">'+(s[k]??0)+'</div></div>').join("");
+    const pct=s.requests?Math.round((s.charged/s.requests)*100):0;
+    document.getElementById("botpct").textContent=pct+"%";
+    if(s.since)document.getElementById("since").textContent="since "+new Date(s.since).toLocaleString();
+  }catch(e){/* keep last values */}
+}
+tick();setInterval(tick,5000);
+</script>
+</body></html>`;
+}

package/edge.js

@@ -37,22 +37,28 @@ function constEq(a, b) {
 /**
  * Web Crypto proof-of-work. `secret` is required (edge runtimes can't keep a
  * random per-process secret across invocations). `store` is an optional
- * single-use replay backend ({ has(token), add(token, expMs) }, both async) —
- * e.g. a Cloudflare KV wrapper; defaults to in-isolate memory.
+ * single-use replay backend: { claim(token, expMs) => bool|Promise<bool> } that
+ * atomically returns true only the first time a token is seen — e.g. a Cloudflare
+ * KV wrapper; defaults to in-isolate memory.
  */
 export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000, store } = {}) {
   if (!secret) throw new Error("createEdgePow requires a stable `secret` string");
   const mem = new Map();
   const MAX = 50_000; // bound memory: edge runtimes have no timer to sweep, so prune on write
   const used = store || {
-    has: async (k) => { const e = mem.get(k); if (e && e < Date.now()) { mem.delete(k); return false; } return mem.has(k); },
-    add: async (k, exp) => {
+    // Atomic single-use claim. The body is synchronous (no await), so two
+    // concurrent claims of the same token cannot both win within an isolate —
+    // closing the TOCTOU that a separate has()+add() would open.
+    claim: (k, exp) => {
+      const e = mem.get(k);
+      if (e && e >= Date.now()) return false; // a still-valid entry => already used
       if (mem.size >= MAX) {
         const now = Date.now();
         for (const [kk, ee] of mem) if (ee < now) mem.delete(kk);          // drop expired first
         if (mem.size >= MAX) mem.delete(mem.keys().next().value);          // hard cap: evict oldest
       }
       mem.set(k, exp);
+      return true;
     },
   };
 
@@ -88,10 +94,11 @@ export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000,
     if (!constEq(sig, expected)) return { ok: false, reason: "bad signature" };
     if (res !== resource) return { ok: false, reason: "wrong resource" };
     if (Date.now() > Number(expStr)) return { ok: false, reason: "expired" };
-    if (await used.has(token)) return { ok: false, reason: "already used" };
     const h = await sha256(`${chal}:${nonce}`);
     if (leadingZeroBits(h) < Number(diffStr)) return { ok: false, reason: "insufficient work" };
-    await used.add(token, Number(expStr));
+    // Atomically claim single-use only AFTER the work is validated, so an invalid
+    // attempt never consumes the token and concurrent dupes can't both pass.
+    if (!(await used.claim(token, Number(expStr)))) return { ok: false, reason: "already used" };
     return { ok: true };
   }
 
@@ -114,6 +121,10 @@ export function createEdgeTollbooth(config = {}) {
     secret,
     store,
     botUserAgents = AI_BOTS,
+    // "bots" (default, charge AI crawler UAs) | "all" (charge all but free()) |
+    // "strict" (charge anything that isn't a real-browser request). An explicit
+    // charge()/free() still wins. Default preserves the original behavior.
+    mode = "bots",
     charge,
     free,
     resourceBaseUrl = "",
@@ -123,12 +134,19 @@ export function createEdgeTollbooth(config = {}) {
   const isBot = makeBotMatcher(botUserAgents);
   const powEngine = pow ? createEdgePow({ secret, difficulty: powDifficulty, store }) : null;
 
+  const looksHuman = (request) => {
+    const ua = request.headers.get("user-agent") || "";
+    const accept = request.headers.get("accept") || "";
+    return /mozilla\/5\.0/i.test(ua) && /text\/html/i.test(accept);
+  };
   const shouldCharge = (request) => {
     try {
       if (typeof free === "function" && free(request)) return false;
       if (typeof charge === "function") return Boolean(charge(request));
     } catch { return true; /* fail closed: charge on predicate error */ }
-    return isBot(request.headers.get("user-agent") || "");
+    if (mode === "all") return true;
+    if (mode === "strict") return !looksHuman(request);
+    return isBot(request.headers.get("user-agent") || ""); // "bots" (default)
   };
 
   return async function gate(request) {

package/index.js

@@ -55,22 +55,58 @@ export function createTollbooth(config = {}) {
     powDifficulty,
     powSecret,
     botUserAgents = AI_BOTS,
+    // Who pays. Default "bots" = the original behavior (charge AI crawler UAs).
+    //  "all"    — charge every client except a `free()` match (UA detection is
+    //             not a security boundary; this stops relying on it).
+    //  "strict" — charge anything that isn't a real-browser request (browser-like
+    //             UA + an HTML Accept). Raises the bar on naive scrapers.
+    // An explicit charge()/free() still wins over the mode.
+    mode = process.env.TOLLBOOTH_MODE || "bots",
     charge,
     free,
     verifyX402,
     resourceBaseUrl = process.env.TOLLBOOTH_RESOURCE_BASE || "",
+    // Adaptive proof-of-work: raise difficulty as charged-request load climbs, so
+    // high-volume abuse pays escalating CPU regardless of how it disguises itself.
+    // Off by default — behavior is unchanged unless explicitly enabled.
+    adaptive = config.adaptive ?? (process.env.TOLLBOOTH_ADAPTIVE === "true"),
+    adaptivePerBit = Number(process.env.TOLLBOOTH_ADAPTIVE_PER_BIT) || 300, // +1 bit per N charged req/min
+    maxDifficulty,
     message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
   } = config;
 
   const isBot = makeBotMatcher(botUserAgents);
   const powEngine = pow ? createPow({ difficulty: powDifficulty, secret: powSecret }) : null;
 
+  // Passive analytics — never affects request handling, just counts what happens.
+  const stats = { since: new Date().toISOString(), requests: 0, freeAllowed: 0, charged: 0, powSolved: 0, x402Paid: 0 };
+
+  const looksHuman = (req) => {
+    const ua = req.headers["user-agent"] || "";
+    const accept = req.headers["accept"] || "";
+    return /mozilla\/5\.0/i.test(ua) && /text\/html/i.test(accept);
+  };
   const shouldCharge = (req) => {
     try {
       if (typeof free === "function" && free(req)) return false;
       if (typeof charge === "function") return Boolean(charge(req));
     } catch { return true; /* fail closed: charge on predicate error */ }
-    return isBot(req.headers["user-agent"] || "");
+    if (mode === "all") return true;
+    if (mode === "strict") return !looksHuman(req);
+    return isBot(req.headers["user-agent"] || ""); // "bots" (default)
+  };
+
+  // Sliding-window of recent charged requests → adaptive PoW difficulty.
+  const baseDifficulty = powEngine?.difficulty ?? (Number(process.env.TOLLBOOTH_POW_BITS) || 18);
+  const ceilDifficulty = Math.min(Number(maxDifficulty) || baseDifficulty + 6, 32);
+  const ADAPT_WINDOW_MS = 60_000;
+  let chargedWindow = [];
+  const difficultyNow = () => {
+    if (!adaptive) return baseDifficulty;
+    const cut = Date.now() - ADAPT_WINDOW_MS;
+    if (chargedWindow.length > 100_000) chargedWindow = chargedWindow.filter((t) => t > cut); // hard bound
+    else while (chargedWindow.length && chargedWindow[0] < cut) chargedWindow.shift();
+    return Math.min(baseDifficulty + Math.floor(chargedWindow.length / Math.max(1, adaptivePerBit)), ceilDifficulty);
   };
   const resourceOf = (req) => {
     // Canonicalize to path+search (matches the edge impl) so the PoW binding is
@@ -82,10 +118,13 @@ export function createTollbooth(config = {}) {
   };
 
   function tollbooth(req, res, next) {
-    if (!shouldCharge(req)) return next();
+    stats.requests++;
+    if (!shouldCharge(req)) { stats.freeAllowed++; return next(); }
     const resource = resourceOf(req);
 
     const send402 = (extra = {}) => {
+      stats.charged++;
+      if (adaptive) chargedWindow.push(Date.now());
       const body = {
         error: "Payment Required",
         message,
@@ -94,7 +133,7 @@ export function createTollbooth(config = {}) {
           : [],
         ...extra,
       };
-      if (powEngine) body.proofOfWork = powEngine.challenge(resource);
+      if (powEngine) body.proofOfWork = powEngine.challenge(resource, difficultyNow());
       res.status(402).json(body);
     };
 
@@ -102,7 +141,7 @@ export function createTollbooth(config = {}) {
     const powHeader = req.headers["x-pow-solution"];
     if (powEngine && powHeader) {
       const r = powEngine.verify(powHeader, resource);
-      if (r.ok) { res.setHeader("X-Tollbooth-Paid", "pow"); return next(); }
+      if (r.ok) { stats.powSolved++; res.setHeader("X-Tollbooth-Paid", "pow"); return next(); }
       res.setHeader("X-Pow-Error", r.reason);
     }
 
@@ -114,7 +153,7 @@ export function createTollbooth(config = {}) {
       const timeout = new Promise((resolve) => setTimeout(() => resolve(false), VERIFY_TIMEOUT_MS));
       return Promise.race([Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource })), timeout])
         .then((ok) => {
-          if (ok) { res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
+          if (ok) { stats.x402Paid++; res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
           send402();
         })
         .catch(() => { res.setHeader("X-Tollbooth-Error", "x402-verify-failed"); send402(); });
@@ -125,6 +164,9 @@ export function createTollbooth(config = {}) {
 
   tollbooth.shouldCharge = shouldCharge;
   tollbooth.pow = powEngine;
+  // Live counters for operators: how much traffic, how much was charged, and how
+  // it was settled. A point-in-time snapshot (never mutated by the caller).
+  tollbooth.stats = () => ({ ...stats, difficultyNow: difficultyNow() });
   return tollbooth;
 }
 
@@ -187,6 +229,11 @@ async function startCli() {
   const { default: express } = await import("express");
   const app = express();
   const gate = createTollbooth({ resourceBaseUrl: process.env.TOLLBOOTH_RESOURCE_BASE || upstream || "" });
+  // Operator analytics — aggregate counts only (no per-request data), mounted
+  // before the gate so they're always reachable and never themselves charged.
+  const { dashboardHtml } = await import("./dashboard.js");
+  app.get("/__tollbooth", (_req, res) => res.type("html").send(dashboardHtml()));
+  app.get("/__tollbooth/stats", (_req, res) => res.json(gate.stats()));
   app.use(gate);
   if (upstream) {
     app.use(createProxy(upstream));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent402-tollbooth",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
   "type": "module",
   "main": "index.js",
@@ -16,6 +16,7 @@
     "worker.js",
     "bots.js",
     "pow.js",
+    "dashboard.js",
     "example.js",
     "demo.js",
     "README.md"
@@ -24,7 +25,7 @@
     "node": ">=20"
   },
   "scripts": {
-    "test": "node test.js && node edge.test.js",
+    "test": "node test.js && node edge.test.js && node features.test.js",
     "demo": "node demo.js",
     "example": "node example.js"
   },

package/pow.js

@@ -35,19 +35,22 @@ export function createPow({
 
   const sign = (payload) => createHmac("sha256", secret).update(payload).digest("base64url");
 
-  /** Mint a signed, resource-scoped challenge. */
-  function challenge(resource) {
+  /** Mint a signed, resource-scoped challenge. `diff` lets the caller raise the
+   *  difficulty for this one challenge (e.g. adaptive-under-load); verify reads
+   *  the difficulty from the token, so any value is self-describing and safe. */
+  function challenge(resource, diff = difficulty) {
+    const d = Math.max(1, Math.min(Math.floor(Number(diff) || difficulty), 32));
     const chal = randomBytes(16).toString("hex");
     const exp = Date.now() + ttlMs;
-    const payload = `${chal}.${exp}.${difficulty}.${resource}`;
+    const payload = `${chal}.${exp}.${d}.${resource}`;
     const token = `${payload}.${sign(payload)}`;
     return {
       algorithm: "sha256",
       challenge: chal,
-      difficulty,
+      difficulty: d,
       expires: exp,
       token,
-      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${difficulty} leading zero bits, then resend the request with header  X-Pow-Solution: ${token}:<nonce>`,
+      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${d} leading zero bits, then resend the request with header  X-Pow-Solution: ${token}:<nonce>`,
     };
   }
 

package/worker.js

@@ -13,8 +13,14 @@
 import { createEdgeTollbooth } from "./edge.js";
 
 const kvStore = (kv) => ({
-  has: async (k) => (await kv.get(k)) != null,
-  add: async (k, expMs) => kv.put(k, "1", { expiration: Math.ceil(expMs / 1000) }),
+  // Best-effort atomic claim. KV has no native compare-and-set, so this is
+  // get-then-put (eventually consistent); for strict single-use, back it with a
+  // Durable Object. Returns true only the first time a token is seen.
+  claim: async (k, expMs) => {
+    if ((await kv.get(k)) != null) return false;
+    await kv.put(k, "1", { expiration: Math.ceil(expMs / 1000) });
+    return true;
+  },
 });
 
 export default {