agent402-tollbooth 0.1.4 → 0.3.0

package/README.md

@@ -92,27 +92,41 @@ The same gate is also built on the Web Crypto + Fetch APIs (`edge.js`), so it ru
 anywhere — no Node required. The gate returns a `402 Response` when the client
 must pay, or `null` to let it through.
 
-**Cloudflare Workers** (poetic: the open pay-per-crawl, on the incumbent's own platform). `wrangler.toml`:
+**Ready-to-deploy templates** (copy a folder, don't assemble from docs):
+
+- **Cloudflare Workers** → [`deploy/cloudflare/`](deploy/cloudflare/) — a ready
+  `wrangler.toml` + a 3-step deploy guide (the open pay-per-crawl, on the
+  incumbent's own platform).
+- **Next.js / Vercel** → [`deploy/nextjs/`](deploy/nextjs/) — a drop-in
+  `middleware.js` + a 3-step deploy guide.
+- **Docker** → [`deploy/docker/`](deploy/docker/) — a `Dockerfile` +
+  `docker-compose.yml` to run the reverse proxy in front of any site with
+  `docker compose up -d` (includes the live `/__tollbooth` dashboard).
+
+The short version of each:
 
 ```toml
+# wrangler.toml  (full template: deploy/cloudflare/wrangler.toml)
 name = "tollbooth"
 main = "node_modules/agent402-tollbooth/worker.js"
+compatibility_date = "2026-01-01"
 [vars]
 TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
 TOLLBOOTH_PAYTO    = "0xYourWallet"   # optional: advertise a USDC x402 quote
-# wrangler secret put TOLLBOOTH_SECRET
+# npx wrangler secret put TOLLBOOTH_SECRET
 # optional single-use replay store:  [[kv_namespaces]] binding = "TOLLBOOTH_KV"
 ```
 
-**Next.js middleware** (`middleware.ts`):
-
-```ts
+```js
+// middleware.js  (full template: deploy/nextjs/middleware.js)
+import { NextResponse } from "next/server";
 import { createEdgeTollbooth } from "agent402-tollbooth/edge";
-const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET! });
+const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET });
 
-export async function middleware(req: Request) {
+export async function middleware(req) {
   return (await gate(req)) ?? NextResponse.next();
 }
+export const config = { matcher: ["/((?!_next/static|_next/image|favicon.ico).*)"] };
 ```
 
 **Any Fetch-API runtime** (Deno, Bun, custom): `const gate = createEdgeTollbooth({ secret }); const blocked = await gate(request); return blocked ?? fetch(request);`
@@ -152,23 +166,127 @@ app.use(createTollbooth({
 | `network` | `"base"` | x402 network |
 | `pow` | `true` | Enable the free proof-of-work rail |
 | `powDifficulty` | `18` | PoW difficulty in leading zero bits (~0.1–0.5s of CPU) |
-| `botUserAgents` | `AI_BOTS` | User-agents to charge (AI/LLM crawlers; search indexers excluded) |
-| `charge(req)` | UA match | Custom predicate for "should this client pay?" |
-| `free(req)` | – | Custom force-allow predicate (wins over `charge`) |
+| `mode` | `"bots"` | Who pays: `"bots"` (AI-crawler UAs) · `"all"` (everyone but `free()`) · `"strict"` (anything that isn't a real-browser request) |
+| `adaptive` | `false` | Raise PoW difficulty as charged-request load climbs (anti-abuse under traffic spikes) |
+| `maxDifficulty` | `base+6` | Ceiling for adaptive difficulty |
+| `adaptivePerBit` | `300` | +1 difficulty bit per N charged requests/min |
+| `botUserAgents` | `AI_BOTS` | User-agents to charge in `"bots"` mode |
+| `charge(req)` | mode | Custom "should this client pay?" predicate (wins over `mode`) |
+| `free(req)` | – | Custom force-allow predicate (wins over everything) |
 | `verifyX402(req, reqs)` | – | Async USDC settlement check (return `true` to allow) |
 | `resourceBaseUrl` | `""` | Absolute base used for the `resource` field / PoW binding |
+| `observe` | `false` | Observe-only: classify and count, but never 402. For pre-launch traffic measurement. |
+| `statsSink` | in-memory | Durable stats backend. Built-ins: `memorySink`, `kvStatsSink(kv)`, `httpStatsSink(url)`. |
 
 Environment variables: `TOLLBOOTH_UPSTREAM`, `TOLLBOOTH_PAYTO`, `TOLLBOOTH_PRICE`,
-`TOLLBOOTH_NETWORK`, `TOLLBOOTH_POW_BITS`, `TOLLBOOTH_SECRET`, `PORT`.
+`TOLLBOOTH_NETWORK`, `TOLLBOOTH_POW_BITS`, `TOLLBOOTH_MODE`, `TOLLBOOTH_ADAPTIVE`,
+`TOLLBOOTH_MAX_POW_BITS`, `TOLLBOOTH_ADAPTIVE_PER_BIT`, `TOLLBOOTH_SECRET`,
+`TOLLBOOTH_OBSERVE`, `TOLLBOOTH_STATS_TOKEN`, `TOLLBOOTH_STATS_BUCKET`, `PORT`.
 
 ## How it decides who pays
 
-By default it charges requests whose `User-Agent` matches a known **AI/LLM
-crawler** (GPTBot, ClaudeBot, CCBot, PerplexityBot, Bytespider, Google-Extended,
-Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are intentionally
-**not** charged so your SEO indexing stays free. Override with `botUserAgents`,
-or take full control with a `charge(req)` predicate (e.g. charge anything
-without a browser-like `Accept` header, or charge everyone on `/api/*`).
+By default (`mode: "bots"`) it charges requests whose `User-Agent` matches a known
+**AI/LLM crawler** (GPTBot, ClaudeBot, CCBot, PerplexityBot, Bytespider,
+Google-Extended, Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are
+intentionally **not** charged so your SEO indexing stays free.
+
+**Don't want to play whack-a-mole with bot detection?** That's the point of the
+other modes — you stop trying to *identify* bots and instead make access *cost
+something*:
+- `mode: "all"` charges every client (except a `free()` match). A "more
+  sophisticated" bot gains nothing by disguising itself — everyone pays or solves
+  a proof-of-work.
+- `mode: "strict"` charges anything that isn't a real-browser request (browser-like
+  UA **and** an HTML `Accept`), letting genuine human page-loads through free.
+- `adaptive: true` makes proof-of-work **harder as load climbs**, so a high-volume
+  scraper pays escalating CPU per request regardless of how it looks — detection is
+  cat-and-mouse, economics isn't.
+
+## Observe before charging
+
+Don't want to flip a meter on cold? **Run the gate in observe-only mode for a
+week first** — every request is still classified (bot vs. human) and counted,
+but nothing ever gets a 402:
+
+```js
+app.use(createTollbooth({ observe: true })); // or: TOLLBOOTH_OBSERVE=true
+```
+
+On the edge / Cloudflare Worker / Next.js: set `TOLLBOOTH_OBSERVE=true` in env.
+
+The dashboard grows a **"Would charge"** counter so you can show your team —
+or your client — exactly how much of their traffic is AI bots **before** you
+start returning 402s. Removing the flag flips on enforcement with no other
+changes. Bots see a `X-Tollbooth-Observed: would-charge` header in observe mode
+(handy for log filtering); humans see nothing.
+
+## Analytics
+
+The middleware keeps aggregate counters (no per-request data):
+- `gate.stats()` → sync, in-process mirror: `{ requests, freeAllowed, wouldCharge, charged, powSolved, x402Paid, difficultyNow, observe }`.
+- `gate.snapshot()` → async, reads from the configured durable sink (defaults to memory).
+- `gate.flush()` → flush any buffered deltas to the durable sink (call inside `ctx.waitUntil` on edge runtimes).
+
+The reverse-proxy CLI exposes them as JSON at **`/__tollbooth/stats`** and as a
+live **dashboard at `/__tollbooth`** — requests, how many were charged,
+proof-of-work solves, USDC collected, and what share of your traffic is bots.
+
+## Durable stats (survive restart, aggregate across instances)
+
+By default, stats live in process memory — fine for single-instance Node,
+useless across multiple replicas or on the edge. Pass a `statsSink` to make
+them survive:
+
+```js
+// Cloudflare Workers: aggregate across all isolates using the same KV namespace
+// that holds the PoW single-use store.
+import { createEdgeTollbooth, kvStatsSink } from "agent402-tollbooth/edge";
+const gate = createEdgeTollbooth({
+  secret: env.TOLLBOOTH_SECRET,
+  statsSink: kvStatsSink(env.TOLLBOOTH_KV, { bucket: "default" }),
+});
+// inside fetch():
+ctx.waitUntil(gate.flush()); // make sure deltas land in KV after the response
+```
+
+```js
+// Any Node deploy: POST batched deltas to a tiny collector (Vercel KV /
+// Upstash / your own API).
+import { createTollbooth, httpStatsSink } from "agent402-tollbooth";
+app.use(createTollbooth({
+  statsSink: httpStatsSink(process.env.TOLLBOOTH_STATS_URL, {
+    token: process.env.TOLLBOOTH_STATS_TOKEN,
+  }),
+}));
+```
+
+Sink interface (build your own — e.g. a Cloudflare Durable Object for strict
+consistency):
+
+```ts
+type StatsSink = {
+  incr(field: string, n?: number): void;        // fire-and-forget
+  flush?(): Promise<void>;                       // optional explicit flush
+  snapshot(): Promise<Record<string, number>>;   // aggregated view
+};
+```
+
+## Edge analytics (Cloudflare Worker / Next.js)
+
+The Cloudflare Worker entry (`worker.js`) auto-mounts both the dashboard and
+JSON endpoint, BEFORE the gate so they're never paywalled:
+
+- **`/__tollbooth`** → live dashboard
+- **`/__tollbooth/stats`** → JSON snapshot (gate with `TOLLBOOTH_STATS_TOKEN` for bearer-auth)
+
+With a `TOLLBOOTH_KV` namespace bound, the stats aggregate across all isolates
+of all Cloudflare colos serving the Worker — one consistent view.
+
+On Next.js / Vercel Edge, middleware can't mount dashboards itself (it'd gate
+them), so a companion **route handler** at `app/__tollbooth/stats/route.js`
+serves the JSON; a static **page** at `app/__tollbooth/page.jsx` renders the
+dashboard HTML. Both are in [`deploy/nextjs/middleware.js`](deploy/nextjs/middleware.js)
+as drop-in copyable snippets.
 
 ## Production checklist (read this)
 
@@ -183,8 +301,9 @@ without a browser-like `Accept` header, or charge everyone on `/api/*`).
   redirect it elsewhere) and **strips client-forged trust/forwarding headers**
   (`X-Tollbooth-Paid`, `X-Forwarded-Host`, etc.) before forwarding.
 - **UA matching is the default, not a security boundary** — a bot can forge a
-  human UA to get the *same free access a human gets* (it gains nothing more). If
-  you need to charge everyone, pass `charge: () => true`.
+  human UA to get the *same free access a human gets* (it gains nothing more). To
+  stop relying on detection entirely, use `mode: "all"` / `mode: "strict"`, and
+  turn on `adaptive` so high-volume abuse pays escalating proof-of-work.
 
 ## Notes
 

package/dashboard.js

@@ -0,0 +1,61 @@
+// A tiny self-contained operator dashboard for the tollbooth, served at
+// /__tollbooth by the reverse-proxy CLI. It polls /__tollbooth/stats and shows
+// how much of your traffic is bots and what the gate is collecting — the visual
+// answer to "I had to upgrade my plan to serve bots I don't monetize."
+// Pure function (no deps) so it's trivially testable.
+export function dashboardHtml() {
+  return `<!doctype html>
+<html lang="en"><head>
+<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<title>agent402-tollbooth — live stats</title>
+<style>
+  :root{--bg:#0b0e14;--card:#131826;--text:#e6e9f0;--muted:#8b93a7;--accent:#4ade80;--line:#1e2638;--mono:ui-monospace,SFMono-Regular,Menlo,monospace}
+  *{box-sizing:border-box;margin:0}body{background:var(--bg);color:var(--text);font:16px/1.5 system-ui,-apple-system,sans-serif}
+  .wrap{max-width:880px;margin:0 auto;padding:40px 20px}
+  h1{font-size:1.4rem;display:flex;align-items:center;gap:10px}
+  .badge{font:700 14px/1 var(--mono);color:var(--accent);background:#000;border:1px solid #1f4a1d;border-radius:8px;padding:8px 10px}
+  .sub{color:var(--muted);margin:6px 0 24px;font-size:.9rem}
+  .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:14px}
+  .card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:18px}
+  .k{color:var(--muted);font-size:.8rem;text-transform:uppercase;letter-spacing:.04em}
+  .v{font:800 1.8rem/1.1 var(--mono);margin-top:8px}
+  .v.accent{color:var(--accent)}
+  footer{color:var(--muted);font-size:.8rem;margin-top:28px}
+  a{color:var(--accent)}
+</style></head>
+<body><div class="wrap">
+  <h1><span class="badge">402</span> agent402-tollbooth <span id="mode"></span></h1>
+  <div class="sub">Live pay-per-crawl stats · refreshes every 5s · <span id="since"></span></div>
+  <div class="grid" id="grid"></div>
+  <footer><b id="botpct">—</b> of requests were classified as AI bots. Aggregate counts only (no per-request data). Raw JSON: <a href="/__tollbooth/stats">/__tollbooth/stats</a>.</footer>
+</div>
+<script>
+const cards=[
+  ["requests","Requests",false],
+  ["freeAllowed","Humans (free)",false],
+  ["wouldCharge","Would charge (observe)","accent"],
+  ["charged","Charged (402)","accent"],
+  ["powSolved","Proof-of-work paid","accent"],
+  ["x402Paid","USDC paid (x402)","accent"],
+  ["difficultyNow","PoW difficulty",false]
+];
+async function tick(){
+  try{
+    const s=await (await fetch("/__tollbooth/stats",{cache:"no-store"})).json();
+    document.getElementById("grid").innerHTML=cards.map(function(c){
+      var k=c[0],label=c[1],acc=c[2];
+      if(k==="wouldCharge" && !(s.wouldCharge>0) && !s.observe) return "";
+      return '<div class="card"><div class="k">'+label+'</div><div class="v '+(acc||"")+'">'+(s[k]??0)+'</div></div>';
+    }).join("");
+    var botish=(s.charged||0)+(s.wouldCharge||0);
+    var pct=s.requests?Math.round((botish/s.requests)*100):0;
+    document.getElementById("botpct").textContent=pct+"%";
+    document.getElementById("mode").textContent=s.observe?" · OBSERVE":"";
+    if(s.since)document.getElementById("since").textContent="since "+new Date(s.since).toLocaleString();
+  }catch(e){/* keep last values */}
+}
+tick();setInterval(tick,5000);
+</script>
+</body></html>`;
+}

package/edge.js

@@ -7,6 +7,9 @@
 //
 // See worker.js for a Cloudflare Worker entry, and the README for Next.js.
 import { makeBotMatcher, AI_BOTS } from "./bots.js";
+import { memorySink } from "./sinks.js";
+
+export { memorySink, kvStatsSink, httpStatsSink } from "./sinks.js";
 
 const te = new TextEncoder();
 
@@ -121,34 +124,59 @@ export function createEdgeTollbooth(config = {}) {
     secret,
     store,
     botUserAgents = AI_BOTS,
+    // "bots" (default, charge AI crawler UAs) | "all" (charge all but free()) |
+    // "strict" (charge anything that isn't a real-browser request). An explicit
+    // charge()/free() still wins. Default preserves the original behavior.
+    mode = "bots",
     charge,
     free,
     resourceBaseUrl = "",
+    // Observe-only mode: count what would have been charged but never 402. Lets
+    // operators run the gate on a live site for a week before flipping on enforcement.
+    observe = false,
+    // Pluggable durable-stats sink. Default = in-memory (per-isolate, so on the
+    // edge it dies with the isolate — pass kvStatsSink(env.TOLLBOOTH_KV) for
+    // cross-isolate aggregation).
+    statsSink,
     message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
   } = config;
 
   const isBot = makeBotMatcher(botUserAgents);
   const powEngine = pow ? createEdgePow({ secret, difficulty: powDifficulty, store }) : null;
+  const mem = memorySink();
+  const sink = statsSink || mem;
+  const writeThrough = statsSink && statsSink !== mem;
+  const incr = (k, n = 1) => { mem.incr(k, n); if (writeThrough) sink.incr(k, n); };
 
+  const looksHuman = (request) => {
+    const ua = request.headers.get("user-agent") || "";
+    const accept = request.headers.get("accept") || "";
+    return /mozilla\/5\.0/i.test(ua) && /text\/html/i.test(accept);
+  };
   const shouldCharge = (request) => {
     try {
       if (typeof free === "function" && free(request)) return false;
       if (typeof charge === "function") return Boolean(charge(request));
     } catch { return true; /* fail closed: charge on predicate error */ }
-    return isBot(request.headers.get("user-agent") || "");
+    if (mode === "all") return true;
+    if (mode === "strict") return !looksHuman(request);
+    return isBot(request.headers.get("user-agent") || ""); // "bots" (default)
   };
 
-  return async function gate(request) {
-    if (!shouldCharge(request)) return null;
+  async function gate(request) {
+    incr("requests");
+    if (!shouldCharge(request)) { incr("freeAllowed"); return null; }
+    if (observe) { incr("wouldCharge"); return null; }
     const u = new URL(request.url);
     const resource = (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + u.pathname + u.search;
 
     const sol = request.headers.get("x-pow-solution");
     if (powEngine && sol) {
       const r = await powEngine.verify(sol, resource);
-      if (r.ok) return null;
+      if (r.ok) { incr("powSolved"); return null; }
     }
 
+    incr("charged");
     const body = {
       error: "Payment Required",
       message,
@@ -159,5 +187,14 @@ export function createEdgeTollbooth(config = {}) {
       status: 402,
       headers: { "content-type": "application/json", "x-tollbooth": "pay-per-crawl" },
     });
-  };
+  }
+
+  // Operator surface: in-process mirror (sync), durable snapshot (async),
+  // and an explicit flush() for edge runtimes — the Worker should call this
+  // inside ctx.waitUntil so buffered KV writes actually happen.
+  gate.observe = observe;
+  gate.stats = () => ({ ...mem.snapshot(), observe });
+  gate.snapshot = async () => ({ ...(await sink.snapshot()), observe });
+  gate.flush = () => Promise.resolve(sink.flush && sink.flush());
+  return gate;
 }

package/index.js

@@ -16,9 +16,11 @@ import { fileURLToPath } from "node:url";
 import { Readable } from "node:stream";
 import { createPow } from "./pow.js";
 import { makeBotMatcher, AI_BOTS } from "./bots.js";
+import { memorySink } from "./sinks.js";
 
 export { AI_BOTS, makeBotMatcher } from "./bots.js";
 export { createPow, leadingZeroBits } from "./pow.js";
+export { memorySink, kvStatsSink, httpStatsSink } from "./sinks.js";
 
 const VERIFY_TIMEOUT_MS = Number(process.env.TOLLBOOTH_VERIFY_TIMEOUT_MS) || 10_000;
 // Headers a client must never be able to forge through the proxy: the gate's own
@@ -44,6 +46,8 @@ const STRIP_INBOUND = new Set([
  * @param {(req, requirements)=>boolean|Promise<boolean>} [config.verifyX402]  USDC settlement check
  * @param {string} [config.resourceBaseUrl]    absolute base for the x402 `resource`/PoW binding
  * @param {string} [config.message]            human-readable note included in the 402
+ * @param {boolean} [config.observe]           observe-only: classify, count, never 402 (deploy a week before enforcing)
+ * @param {object} [config.statsSink]          pluggable durable-stats sink (default: in-memory). See sinks.js.
  */
 export function createTollbooth(config = {}) {
   const {
@@ -55,22 +59,71 @@ export function createTollbooth(config = {}) {
     powDifficulty,
     powSecret,
     botUserAgents = AI_BOTS,
+    // Who pays. Default "bots" = the original behavior (charge AI crawler UAs).
+    //  "all"    — charge every client except a `free()` match (UA detection is
+    //             not a security boundary; this stops relying on it).
+    //  "strict" — charge anything that isn't a real-browser request (browser-like
+    //             UA + an HTML Accept). Raises the bar on naive scrapers.
+    // An explicit charge()/free() still wins over the mode.
+    mode = process.env.TOLLBOOTH_MODE || "bots",
     charge,
     free,
     verifyX402,
     resourceBaseUrl = process.env.TOLLBOOTH_RESOURCE_BASE || "",
+    // Adaptive proof-of-work: raise difficulty as charged-request load climbs, so
+    // high-volume abuse pays escalating CPU regardless of how it disguises itself.
+    // Off by default — behavior is unchanged unless explicitly enabled.
+    adaptive = config.adaptive ?? (process.env.TOLLBOOTH_ADAPTIVE === "true"),
+    adaptivePerBit = Number(process.env.TOLLBOOTH_ADAPTIVE_PER_BIT) || 300, // +1 bit per N charged req/min
+    maxDifficulty,
+    // Observe-only mode: classify every request as charge-vs-free and count it,
+    // but always let it through (never return 402). Use it to measure a site's
+    // bot traffic for a week before flipping the meter on. Off by default.
+    observe = config.observe ?? (process.env.TOLLBOOTH_OBSERVE === "true"),
+    // Pluggable durable-stats sink. Default = in-memory (current behavior).
+    // See sinks.js for kvStatsSink (Cloudflare KV) and httpStatsSink.
+    statsSink,
     message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
   } = config;
 
   const isBot = makeBotMatcher(botUserAgents);
   const powEngine = pow ? createPow({ difficulty: powDifficulty, secret: powSecret }) : null;
 
+  // Passive analytics — never affects request handling, just counts what happens.
+  // `mem` is an always-on in-process mirror so `.stats()` stays synchronous for
+  // single-process Node deployments. A durable `statsSink` (KV/HTTP) is written
+  // through alongside and is the source of truth for `.snapshot()`.
+  const mem = memorySink();
+  const sink = statsSink || mem;
+  const writeThrough = statsSink && statsSink !== mem;
+  const incr = (k, n = 1) => { mem.incr(k, n); if (writeThrough) sink.incr(k, n); };
+
+  const looksHuman = (req) => {
+    const ua = req.headers["user-agent"] || "";
+    const accept = req.headers["accept"] || "";
+    return /mozilla\/5\.0/i.test(ua) && /text\/html/i.test(accept);
+  };
   const shouldCharge = (req) => {
     try {
       if (typeof free === "function" && free(req)) return false;
       if (typeof charge === "function") return Boolean(charge(req));
     } catch { return true; /* fail closed: charge on predicate error */ }
-    return isBot(req.headers["user-agent"] || "");
+    if (mode === "all") return true;
+    if (mode === "strict") return !looksHuman(req);
+    return isBot(req.headers["user-agent"] || ""); // "bots" (default)
+  };
+
+  // Sliding-window of recent charged requests → adaptive PoW difficulty.
+  const baseDifficulty = powEngine?.difficulty ?? (Number(process.env.TOLLBOOTH_POW_BITS) || 18);
+  const ceilDifficulty = Math.min(Number(maxDifficulty) || baseDifficulty + 6, 32);
+  const ADAPT_WINDOW_MS = 60_000;
+  let chargedWindow = [];
+  const difficultyNow = () => {
+    if (!adaptive) return baseDifficulty;
+    const cut = Date.now() - ADAPT_WINDOW_MS;
+    if (chargedWindow.length > 100_000) chargedWindow = chargedWindow.filter((t) => t > cut); // hard bound
+    else while (chargedWindow.length && chargedWindow[0] < cut) chargedWindow.shift();
+    return Math.min(baseDifficulty + Math.floor(chargedWindow.length / Math.max(1, adaptivePerBit)), ceilDifficulty);
   };
   const resourceOf = (req) => {
     // Canonicalize to path+search (matches the edge impl) so the PoW binding is
@@ -82,10 +135,16 @@ export function createTollbooth(config = {}) {
   };
 
   function tollbooth(req, res, next) {
-    if (!shouldCharge(req)) return next();
+    incr("requests");
+    if (!shouldCharge(req)) { incr("freeAllowed"); return next(); }
+    // Observe-only: classify as would-charge but let it through. Lets operators
+    // measure bot traffic on a live site before turning on enforcement.
+    if (observe) { incr("wouldCharge"); res.setHeader("X-Tollbooth-Observed", "would-charge"); return next(); }
     const resource = resourceOf(req);
 
     const send402 = (extra = {}) => {
+      incr("charged");
+      if (adaptive) chargedWindow.push(Date.now());
       const body = {
         error: "Payment Required",
         message,
@@ -94,7 +153,7 @@ export function createTollbooth(config = {}) {
           : [],
         ...extra,
       };
-      if (powEngine) body.proofOfWork = powEngine.challenge(resource);
+      if (powEngine) body.proofOfWork = powEngine.challenge(resource, difficultyNow());
       res.status(402).json(body);
     };
 
@@ -102,7 +161,7 @@ export function createTollbooth(config = {}) {
     const powHeader = req.headers["x-pow-solution"];
     if (powEngine && powHeader) {
       const r = powEngine.verify(powHeader, resource);
-      if (r.ok) { res.setHeader("X-Tollbooth-Paid", "pow"); return next(); }
+      if (r.ok) { incr("powSolved"); res.setHeader("X-Tollbooth-Paid", "pow"); return next(); }
       res.setHeader("X-Pow-Error", r.reason);
     }
 
@@ -114,7 +173,7 @@ export function createTollbooth(config = {}) {
       const timeout = new Promise((resolve) => setTimeout(() => resolve(false), VERIFY_TIMEOUT_MS));
       return Promise.race([Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource })), timeout])
         .then((ok) => {
-          if (ok) { res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
+          if (ok) { incr("x402Paid"); res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
           send402();
         })
         .catch(() => { res.setHeader("X-Tollbooth-Error", "x402-verify-failed"); send402(); });
@@ -125,6 +184,13 @@ export function createTollbooth(config = {}) {
 
   tollbooth.shouldCharge = shouldCharge;
   tollbooth.pow = powEngine;
+  tollbooth.observe = observe;
+  // Live counters for operators: how much traffic, how much was charged, and how
+  // it was settled. A point-in-time snapshot (never mutated by the caller).
+  // .stats() is sync (in-process mirror). .snapshot() is async (durable sink).
+  tollbooth.stats = () => ({ ...mem.snapshot(), difficultyNow: difficultyNow(), observe });
+  tollbooth.snapshot = async () => ({ ...(await sink.snapshot()), difficultyNow: difficultyNow(), observe });
+  tollbooth.flush = () => Promise.resolve(sink.flush && sink.flush());
   return tollbooth;
 }
 
@@ -187,6 +253,11 @@ async function startCli() {
   const { default: express } = await import("express");
   const app = express();
   const gate = createTollbooth({ resourceBaseUrl: process.env.TOLLBOOTH_RESOURCE_BASE || upstream || "" });
+  // Operator analytics — aggregate counts only (no per-request data), mounted
+  // before the gate so they're always reachable and never themselves charged.
+  const { dashboardHtml } = await import("./dashboard.js");
+  app.get("/__tollbooth", (_req, res) => res.type("html").send(dashboardHtml()));
+  app.get("/__tollbooth/stats", (_req, res) => res.json(gate.stats()));
   app.use(gate);
   if (upstream) {
     app.use(createProxy(upstream));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent402-tollbooth",
-  "version": "0.1.4",
+  "version": "0.3.0",
   "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
   "type": "module",
   "main": "index.js",
@@ -8,7 +8,13 @@
     "agent402-tollbooth": "index.js"
   },
   "exports": {
-    ".": "./index.js"
+    ".": "./index.js",
+    "./edge": "./edge.js",
+    "./edge.js": "./edge.js",
+    "./worker": "./worker.js",
+    "./worker.js": "./worker.js",
+    "./dashboard.js": "./dashboard.js",
+    "./sinks.js": "./sinks.js"
   },
   "files": [
     "index.js",
@@ -16,6 +22,8 @@
     "worker.js",
     "bots.js",
     "pow.js",
+    "dashboard.js",
+    "sinks.js",
     "example.js",
     "demo.js",
     "README.md"
@@ -24,7 +32,7 @@
     "node": ">=20"
   },
   "scripts": {
-    "test": "node test.js && node edge.test.js",
+    "test": "node test.js && node edge.test.js && node features.test.js",
     "demo": "node demo.js",
     "example": "node example.js"
   },

package/pow.js

@@ -35,19 +35,22 @@ export function createPow({
 
   const sign = (payload) => createHmac("sha256", secret).update(payload).digest("base64url");
 
-  /** Mint a signed, resource-scoped challenge. */
-  function challenge(resource) {
+  /** Mint a signed, resource-scoped challenge. `diff` lets the caller raise the
+   *  difficulty for this one challenge (e.g. adaptive-under-load); verify reads
+   *  the difficulty from the token, so any value is self-describing and safe. */
+  function challenge(resource, diff = difficulty) {
+    const d = Math.max(1, Math.min(Math.floor(Number(diff) || difficulty), 32));
     const chal = randomBytes(16).toString("hex");
     const exp = Date.now() + ttlMs;
-    const payload = `${chal}.${exp}.${difficulty}.${resource}`;
+    const payload = `${chal}.${exp}.${d}.${resource}`;
     const token = `${payload}.${sign(payload)}`;
     return {
       algorithm: "sha256",
       challenge: chal,
-      difficulty,
+      difficulty: d,
       expires: exp,
       token,
-      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${difficulty} leading zero bits, then resend the request with header  X-Pow-Solution: ${token}:<nonce>`,
+      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${d} leading zero bits, then resend the request with header  X-Pow-Solution: ${token}:<nonce>`,
     };
   }
 

package/sinks.js

@@ -0,0 +1,148 @@
+// Pluggable stats sinks for the tollbooth — turn the in-memory counter into
+// something that survives restart and aggregates across processes / isolates.
+//
+// A sink is just:
+//   {
+//     incr(field, n=1)   // fire-and-forget; sync from the gate's perspective
+//     flush()            // optional — run at end of request via ctx.waitUntil
+//     snapshot()         // returns the aggregated counts as { field: number, ... }
+//   }
+//
+// Shipped sinks:
+//   memorySink()                       — default, in-process, zero config.
+//   kvStatsSink(kv, opts)              — Cloudflare KV (append-only, eventually consistent).
+//   httpStatsSink(url, opts)           — POST batched deltas to any collector.
+//
+// Want strong consistency on Cloudflare? Back it with a Durable Object that
+// implements the same { incr, snapshot } interface and pass it as `statsSink`.
+
+const ZERO = () => ({
+  requests: 0,
+  freeAllowed: 0,
+  wouldCharge: 0,
+  charged: 0,
+  powSolved: 0,
+  x402Paid: 0,
+});
+
+/** In-process counter. Synchronous; loses state on restart. */
+export function memorySink() {
+  const c = { since: new Date().toISOString(), ...ZERO() };
+  return {
+    incr(field, n = 1) { c[field] = (c[field] || 0) + n; },
+    flush() { return Promise.resolve(); },
+    snapshot() { return { ...c }; },
+  };
+}
+
+/**
+ * Cloudflare KV-backed sink. Writes are append-only delta records under
+ * `tb:stats:${bucket}:*`; snapshot lists + sums them. Bounded by `ttlSeconds`.
+ *
+ * Pattern notes:
+ *  - Cloudflare Workers don't run timers between requests. The gate calls
+ *    `incr()` synchronously into an in-isolate buffer; the Worker entry must
+ *    call `flush()` (typically inside `ctx.waitUntil(...)`) so the buffered
+ *    delta becomes a KV write. Without that, deltas stay in isolate memory
+ *    and die with the isolate.
+ *  - KV has no atomic increment, so we never RMW a shared counter. Each
+ *    flush writes a brand-new key (unique id + ts) — concurrent isolates
+ *    can't lose each other's deltas. Snapshot sums every key under the prefix.
+ *  - Tradeoff: snapshot reads list+get every key, which is fine for low traffic
+ *    (deltas batched per request) but expensive for very high traffic. For
+ *    that, back with a Durable Object instead.
+ */
+export function kvStatsSink(kv, { bucket = "default", ttlSeconds = 60 * 60 * 24 * 30 } = {}) {
+  const prefix = `tb:stats:${bucket}:`;
+  const buf = {};
+  const isDirty = () => Object.keys(buf).length > 0;
+  return {
+    incr(field, n = 1) { buf[field] = (buf[field] || 0) + n; },
+    flush() {
+      if (!isDirty()) return Promise.resolve();
+      const payload = { ...buf, _ts: Date.now() };
+      for (const k of Object.keys(buf)) delete buf[k];
+      const key = prefix + `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+      return Promise.resolve(kv.put(key, JSON.stringify(payload), { expirationTtl: ttlSeconds })).catch(() => {});
+    },
+    async snapshot() {
+      const result = ZERO();
+      let oldest = Infinity;
+      let cursor;
+      do {
+        const page = await kv.list({ prefix, cursor });
+        for (const { name } of page.keys) {
+          let raw;
+          try { raw = await kv.get(name); } catch { continue; }
+          if (!raw) continue;
+          let obj; try { obj = JSON.parse(raw); } catch { continue; }
+          if (typeof obj._ts === "number" && obj._ts < oldest) oldest = obj._ts;
+          for (const [k, n] of Object.entries(obj)) {
+            if (k === "_ts") continue;
+            if (typeof n === "number") result[k] = (result[k] || 0) + n;
+          }
+        }
+        cursor = page.cursor;
+        if (page.list_complete || !cursor) break;
+      } while (cursor);
+      if (oldest !== Infinity) result.since = new Date(oldest).toISOString();
+      return result;
+    },
+  };
+}
+
+/**
+ * HTTP-backed sink. Batched POSTs of `{ incr: { field: n, ... }, ts }`.
+ * GET on the same URL returns the aggregated snapshot — wire that up on your
+ * collector (a tiny Vercel route hander, an Express app, anything).
+ *
+ *   incr(): buffers locally and schedules a flush via setTimeout (Node) or
+ *           returns immediately on the edge (caller should invoke flush()
+ *           inside ctx.waitUntil so the POST survives the response).
+ *   flush(): force-send the current buffer right now (used by edge entries).
+ *   snapshot(): GET the collector URL.
+ *
+ * Auth: pass `token` to include `Authorization: Bearer <token>`.
+ */
+export function httpStatsSink(url, { token, batchMs = 2000, fetchImpl } = {}) {
+  const f = fetchImpl || globalThis.fetch;
+  if (typeof f !== "function") throw new Error("httpStatsSink: no fetch available (pass fetchImpl or run on Node 18+)");
+  const buf = {};
+  let timer = null;
+  const headers = { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) };
+  async function postBuf() {
+    timer = null;
+    if (Object.keys(buf).length === 0) return;
+    const payload = { ...buf };
+    for (const k of Object.keys(buf)) delete buf[k];
+    try {
+      await f(url, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ incr: payload, ts: Date.now() }),
+        signal: AbortSignal.timeout(5000),
+      });
+    } catch { /* fire-and-forget: stats loss is acceptable here */ }
+  }
+  return {
+    incr(field, n = 1) {
+      buf[field] = (buf[field] || 0) + n;
+      // Batched flush. setTimeout works on Node; on the edge the caller should
+      // still invoke flush() inside ctx.waitUntil because timers don't fire
+      // after the response is returned.
+      if (!timer && typeof setTimeout === "function") timer = setTimeout(postBuf, batchMs);
+    },
+    flush: postBuf,
+    async snapshot() {
+      try {
+        const r = await f(url, {
+          headers: token ? { authorization: `Bearer ${token}` } : {},
+          signal: AbortSignal.timeout(5000),
+        });
+        if (!r.ok) return ZERO();
+        const j = await r.json();
+        return { ...ZERO(), ...j };
+      } catch { return ZERO(); }
+    },
+  };
+}

package/worker.js

@@ -8,9 +8,13 @@
 //   [vars]
 //   TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
 //   TOLLBOOTH_PAYTO    = "0xYourWallet"        # optional (advertises USDC quote)
+//   TOLLBOOTH_OBSERVE  = "true"                # optional: observe-only, never 402
+//   TOLLBOOTH_STATS_TOKEN = "any long string"  # optional: gate /__tollbooth/stats
 //   # secret (required): wrangler secret put TOLLBOOTH_SECRET
 //   # optional single-use store: [[kv_namespaces]] binding = "TOLLBOOTH_KV"
-import { createEdgeTollbooth } from "./edge.js";
+//   # ↑ same TOLLBOOTH_KV is reused for durable stats aggregation across isolates.
+import { createEdgeTollbooth, kvStatsSink } from "./edge.js";
+import { dashboardHtml } from "./dashboard.js";
 
 const kvStore = (kv) => ({
   // Best-effort atomic claim. KV has no native compare-and-set, so this is
@@ -24,7 +28,7 @@ const kvStore = (kv) => ({
 });
 
 export default {
-  async fetch(request, env) {
+  async fetch(request, env, ctx) {
     if (!env.TOLLBOOTH_SECRET) {
       return new Response("Tollbooth misconfigured: set TOLLBOOTH_SECRET (wrangler secret put TOLLBOOTH_SECRET)", { status: 500 });
     }
@@ -33,6 +37,11 @@ export default {
       // can be reused across isolates within its TTL. Bind a KV namespace for prod.
       console.warn("agent402-tollbooth: no TOLLBOOTH_KV bound — proof-of-work replay protection is per-isolate only. Bind a KV namespace for production.");
     }
+    // Durable stats live in KV if a namespace is bound. Without it, the dashboard
+    // is per-isolate (dies on cold start) — fine for dev, useless for prod.
+    const statsSink = env.TOLLBOOTH_KV
+      ? kvStatsSink(env.TOLLBOOTH_KV, { bucket: env.TOLLBOOTH_STATS_BUCKET || "default" })
+      : undefined;
     const gate = createEdgeTollbooth({
       secret: env.TOLLBOOTH_SECRET,
       price: env.TOLLBOOTH_PRICE || "$0.001",
@@ -40,9 +49,30 @@ export default {
       network: env.TOLLBOOTH_NETWORK || "base",
       powDifficulty: env.TOLLBOOTH_POW_BITS ? Number(env.TOLLBOOTH_POW_BITS) : undefined,
       store: env.TOLLBOOTH_KV ? kvStore(env.TOLLBOOTH_KV) : undefined,
+      observe: env.TOLLBOOTH_OBSERVE === "true",
+      statsSink,
     });
 
+    // Free, never-gated operator endpoints. Mounted BEFORE the gate so they
+    // can't be paywalled — and so the dashboard polls work even when the rest
+    // of the origin is fully gated.
+    const u = new URL(request.url);
+    if (u.pathname === "/__tollbooth" || u.pathname === "/__tollbooth/") {
+      return new Response(dashboardHtml(), { headers: { "content-type": "text/html; charset=utf-8" } });
+    }
+    if (u.pathname === "/__tollbooth/stats") {
+      // Optional bearer-token gate — share with your monitoring caller.
+      if (env.TOLLBOOTH_STATS_TOKEN) {
+        const got = (request.headers.get("authorization") || "").replace(/^Bearer\s+/i, "");
+        if (got !== env.TOLLBOOTH_STATS_TOKEN) return new Response("unauthorized", { status: 401 });
+      }
+      const snap = await gate.snapshot();
+      return new Response(JSON.stringify(snap), { headers: { "content-type": "application/json" } });
+    }
+
     const blocked = await gate(request);
+    // Flush any buffered stats to KV after we've replied — survives the response.
+    if (ctx && typeof ctx.waitUntil === "function") ctx.waitUntil(gate.flush());
     if (blocked) return blocked;
 
     // Allowed → proxy to the origin.