agent402-tollbooth 0.1.0

package/README.md

@@ -0,0 +1,118 @@
+# agent402-tollbooth
+
+**Open-source, self-hostable x402 "pay-per-crawl". Put it in front of any site or
+API: humans browse free, AI crawlers and agents pay per request** — either in
+USDC over the [x402 protocol](https://x402.org), or for free by solving a
+proof-of-work. No Cloudflare, no Stripe, no Merchant-of-Record, no signup.
+
+The big platforms ([Cloudflare](https://stackoverflow.blog/2026/02/26/how-pay-per-crawl-is-reshaping-data-monetization/),
+Stack Overflow) shipped pay-per-crawl as a closed, fiat, you-must-be-on-our-CDN
+feature. This is the open, crypto-native, run-it-yourself version — built on the
+same hardened 402 + proof-of-work machinery as [Agent402](https://github.com/MikeyPetrillo/Agent402).
+
+## Install
+
+```bash
+npm install agent402-tollbooth
+```
+
+## Use it as Express middleware
+
+```js
+import express from "express";
+import { createTollbooth } from "agent402-tollbooth";
+
+const app = express();
+
+// Humans pass through; known AI crawlers get 402 and must pay or solve a PoW.
+app.use(createTollbooth({ price: "$0.002" }));
+
+app.get("/article", (_req, res) => res.send("…your content…"));
+app.listen(3000);
+```
+
+```bash
+curl -A "Mozilla/5.0" localhost:3000/article     # human  -> 200, free
+curl -A "ClaudeBot/1.0" localhost:3000/article   # bot    -> 402 Payment Required
+```
+
+The 402 body advertises both rails:
+
+```jsonc
+{
+  "error": "Payment Required",
+  "message": "…humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
+  "accepts": [{ "scheme": "exact", "network": "base", "maxAmountRequired": "$0.002", "asset": "USDC", "payTo": "0x…", "resource": "/article" }],
+  "proofOfWork": { "algorithm": "sha256", "challenge": "…", "difficulty": 18, "token": "…", "rule": "Find a nonce so sha256(challenge+\":\"+nonce) has >= 18 leading zero bits; resend with header X-Pow-Solution: <token>:<nonce>" }
+}
+```
+
+A crawler that can't (or won't) pay USDC solves the puzzle and retries with
+`X-Pow-Solution: <token>:<nonce>` — sub-second of CPU, single-use, bound to that
+exact URL.
+
+## Use it as a reverse proxy (any language/framework)
+
+Point it at your existing site — no code changes there:
+
+```bash
+TOLLBOOTH_UPSTREAM=https://your-site.com \
+TOLLBOOTH_PAYTO=0xYourWallet \
+npx agent402-tollbooth          # listens on :4021, proxies humans free, charges bots
+```
+
+## Accepting USDC (x402)
+
+The proof-of-work rail works with **zero config**. To also settle real USDC,
+set `payTo` and supply `verifyX402` — wire it to the standard, audited x402
+server stack (`@x402/express` / your facilitator) rather than reinventing
+settlement:
+
+```js
+import { paymentMiddleware } from "x402-express"; // or @x402/express
+const x402 = paymentMiddleware(/* your wallet + facilitator config */);
+
+app.use(createTollbooth({
+  payTo: "0xYourWallet",
+  network: "base",
+  // Reuse the standard middleware to verify the X-PAYMENT header:
+  verifyX402: (req) => new Promise((resolve) =>
+    x402(req, { setHeader() {}, status() { return this; }, json() { resolve(false); } }, () => resolve(true))),
+}));
+```
+
+(PoW is checked first, so an agent without a wallet always has a free path.)
+
+## Configuration
+
+| Option | Default | What |
+|---|---|---|
+| `price` | `"$0.001"` | Advertised price per request (x402 quote) |
+| `payTo` | – | Wallet address; set to advertise a USDC x402 quote |
+| `network` | `"base"` | x402 network |
+| `pow` | `true` | Enable the free proof-of-work rail |
+| `powDifficulty` | `18` | PoW difficulty in leading zero bits (~0.1–0.5s of CPU) |
+| `botUserAgents` | `AI_BOTS` | User-agents to charge (AI/LLM crawlers; search indexers excluded) |
+| `charge(req)` | UA match | Custom predicate for "should this client pay?" |
+| `free(req)` | – | Custom force-allow predicate (wins over `charge`) |
+| `verifyX402(req, reqs)` | – | Async USDC settlement check (return `true` to allow) |
+| `resourceBaseUrl` | `""` | Absolute base used for the `resource` field / PoW binding |
+
+Environment variables: `TOLLBOOTH_UPSTREAM`, `TOLLBOOTH_PAYTO`, `TOLLBOOTH_PRICE`,
+`TOLLBOOTH_NETWORK`, `TOLLBOOTH_POW_BITS`, `TOLLBOOTH_SECRET`, `PORT`.
+
+## How it decides who pays
+
+By default it charges requests whose `User-Agent` matches a known **AI/LLM
+crawler** (GPTBot, ClaudeBot, CCBot, PerplexityBot, Bytespider, Google-Extended,
+Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are intentionally
+**not** charged so your SEO indexing stays free. Override with `botUserAgents`,
+or take full control with a `charge(req)` predicate (e.g. charge anything
+without a browser-like `Accept` header, or charge everyone on `/api/*`).
+
+## Notes
+
+- Proof-of-work tokens are HMAC-signed, expiry-checked, single-use, and bound to
+  the exact resource — a solution for `/a` can't be replayed on `/b` or reused.
+- Set a stable `TOLLBOOTH_SECRET` in production so tokens survive restarts.
+- MIT licensed. Part of [Agent402](https://github.com/MikeyPetrillo/Agent402).

package/bots.js

@@ -0,0 +1,30 @@
+// Default set of AI / LLM crawler and agent user-agents to charge.
+//
+// Deliberately AI-focused: traditional search indexers (Googlebot, Bingbot,
+// DuckDuckBot) are NOT here — you almost always want classic SEO indexing to
+// stay free. Override the whole list with `botUserAgents`, or supply your own
+// `charge(req)` predicate for full control.
+export const AI_BOTS = [
+  "GPTBot", "OAI-SearchBot", "ChatGPT-User",
+  "ClaudeBot", "Claude-Web", "anthropic-ai", "Claude-User",
+  "PerplexityBot", "Perplexity-User",
+  "CCBot", "Bytespider", "Google-Extended", "Amazonbot",
+  "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher",
+  "Applebot-Extended", "Diffbot", "Omgilibot", "ImagesiftBot",
+  "YouBot", "Timpibot", "DuckAssistBot", "PetalBot",
+  "FriendlyCrawler", "AI2Bot", "Scrapy", "python-requests",
+];
+
+/**
+ * Build a fast case-insensitive substring matcher over a user-agent list.
+ * @param {string[]} list
+ * @returns {(userAgent?: string) => boolean}
+ */
+export function makeBotMatcher(list = AI_BOTS) {
+  const needles = list.map((s) => s.toLowerCase());
+  return (userAgent = "") => {
+    const ua = String(userAgent).toLowerCase();
+    if (!ua) return false;
+    return needles.some((n) => ua.includes(n));
+  };
+}

package/example.js

@@ -0,0 +1,29 @@
+// Runnable demo: a tiny "content site" sitting behind the tollbooth.
+// Humans read it free; AI crawlers get a 402 and must pay (USDC via x402) or
+// solve a proof-of-work.
+//
+//   node tollbooth/example.js
+//   curl -A "Mozilla/5.0" localhost:4021/article      # human  -> 200
+//   curl -A "ClaudeBot/1.0" localhost:4021/article    # bot    -> 402
+import express from "express";
+import { createTollbooth } from "./index.js";
+
+const app = express();
+
+app.use(createTollbooth({
+  price: "$0.002",
+  // Set payTo to advertise a USDC x402 quote (and wire `verifyX402` to settle).
+  payTo: process.env.TOLLBOOTH_PAYTO || undefined,
+  // Humans pass free; AI crawlers (GPTBot/ClaudeBot/CCBot/PerplexityBot/…) pay.
+}));
+
+app.use((_req, res) =>
+  res.type("html").send("<h1>Premium content</h1><p>Humans read this for free. An AI crawler paid (or solved a proof-of-work) to reach this page.</p>")
+);
+
+const port = Number(process.env.PORT) || 4021;
+app.listen(port, () => {
+  console.log(`Demo site behind the tollbooth → http://localhost:${port}`);
+  console.log(`  human:  curl -A "Mozilla/5.0" localhost:${port}/article`);
+  console.log(`  bot:    curl -A "ClaudeBot/1.0" localhost:${port}/article`);
+});

package/index.js

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+// agent402-tollbooth — an open-source, self-hostable x402 "pay-per-crawl" gate.
+//
+// Put it in front of any site or API: human visitors pass through free, while
+// AI crawlers / agents must pay per request — either in USDC over the x402
+// protocol, or for free by solving a proof-of-work (no wallet, no signup, no
+// Stripe, no Cloudflare). Use it two ways:
+//
+//   1. Express middleware:   app.use(createTollbooth({ ... }))
+//   2. Reverse proxy (CLI):  TOLLBOOTH_UPSTREAM=https://your-site.com npx agent402-tollbooth
+//
+// The proof-of-work rail works out of the box with zero configuration. To also
+// accept USDC, set `payTo` and supply `verifyX402` (wire it to the standard
+// x402 server middleware / your facilitator — see README).
+import { fileURLToPath } from "node:url";
+import { createPow } from "./pow.js";
+import { makeBotMatcher, AI_BOTS } from "./bots.js";
+
+export { AI_BOTS, makeBotMatcher } from "./bots.js";
+export { createPow, leadingZeroBits } from "./pow.js";
+
+/**
+ * Create the tollbooth Express middleware.
+ * @param {object} [config]
+ * @param {string} [config.price="$0.001"]     advertised price per request
+ * @param {string|null} [config.payTo]         wallet for the x402 quote (enables USDC rail)
+ * @param {string} [config.network="base"]     x402 network
+ * @param {string} [config.asset="USDC"]
+ * @param {boolean} [config.pow=true]          enable the free proof-of-work rail
+ * @param {number} [config.powDifficulty]      PoW difficulty in leading zero bits
+ * @param {string[]} [config.botUserAgents]    user-agents to charge (default: AI_BOTS)
+ * @param {(req)=>boolean} [config.charge]     custom "should this client pay?" predicate
+ * @param {(req)=>boolean} [config.free]       custom force-allow predicate (wins over charge)
+ * @param {(req, requirements)=>boolean|Promise<boolean>} [config.verifyX402]  USDC settlement check
+ * @param {string} [config.resourceBaseUrl]    absolute base for the x402 `resource`/PoW binding
+ * @param {string} [config.message]            human-readable note included in the 402
+ */
+export function createTollbooth(config = {}) {
+  const {
+    price = process.env.TOLLBOOTH_PRICE || "$0.001",
+    payTo = process.env.TOLLBOOTH_PAYTO || null,
+    network = process.env.TOLLBOOTH_NETWORK || "base",
+    asset = "USDC",
+    pow = true,
+    powDifficulty,
+    powSecret,
+    botUserAgents = AI_BOTS,
+    charge,
+    free,
+    verifyX402,
+    resourceBaseUrl = process.env.TOLLBOOTH_RESOURCE_BASE || "",
+    message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
+  } = config;
+
+  const isBot = makeBotMatcher(botUserAgents);
+  const powEngine = pow ? createPow({ difficulty: powDifficulty, secret: powSecret }) : null;
+
+  const shouldCharge = (req) => {
+    if (typeof free === "function" && free(req)) return false;
+    if (typeof charge === "function") return Boolean(charge(req));
+    return isBot(req.headers["user-agent"] || "");
+  };
+  const resourceOf = (req) =>
+    (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + (req.originalUrl || req.url || "/");
+
+  function tollbooth(req, res, next) {
+    if (!shouldCharge(req)) return next();
+    const resource = resourceOf(req);
+
+    const send402 = (extra = {}) => {
+      const body = {
+        error: "Payment Required",
+        message,
+        accepts: payTo
+          ? [{ scheme: "exact", network, maxAmountRequired: String(price), asset, payTo, resource }]
+          : [],
+        ...extra,
+      };
+      if (powEngine) body.proofOfWork = powEngine.challenge(resource);
+      res.status(402).json(body);
+    };
+
+    // Free rail: proof-of-work.
+    const powHeader = req.headers["x-pow-solution"];
+    if (powEngine && powHeader) {
+      const r = powEngine.verify(powHeader, resource);
+      if (r.ok) { res.setHeader("X-Tollbooth-Paid", "pow"); return next(); }
+      res.setHeader("X-Pow-Error", r.reason);
+    }
+
+    // Paid rail: x402 (USDC). Settlement verification is operator-supplied so
+    // we reuse the standard, audited x402 stack rather than reinvent it.
+    const payHeader = req.headers["x-payment"] || req.headers["payment-signature"];
+    if (payTo && typeof verifyX402 === "function" && payHeader) {
+      return Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource }))
+        .then((ok) => {
+          if (ok) { res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
+          send402();
+        })
+        .catch(() => { res.setHeader("X-Tollbooth-Error", "x402-verify-failed"); send402(); });
+    }
+
+    return send402();
+  }
+
+  tollbooth.shouldCharge = shouldCharge;
+  tollbooth.pow = powEngine;
+  return tollbooth;
+}
+
+/** Minimal reverse proxy: forward a request to `upstream`, streaming-safe-ish. */
+export function createProxy(upstream) {
+  return async (req, res) => {
+    try {
+      const target = new URL(req.originalUrl || req.url, upstream);
+      const headers = {};
+      for (const [k, v] of Object.entries(req.headers)) {
+        const lk = k.toLowerCase();
+        if (lk === "host" || lk === "connection" || lk === "content-length") continue;
+        headers[k] = v;
+      }
+      const method = req.method;
+      const body = method === "GET" || method === "HEAD" ? undefined : await readBody(req);
+      const up = await fetch(target, { method, headers, body, redirect: "manual" });
+      res.status(up.status);
+      up.headers.forEach((val, key) => {
+        const lk = key.toLowerCase();
+        // fetch already decoded the body; drop hop-by-hop / length/encoding headers.
+        if (["content-encoding", "content-length", "transfer-encoding", "connection"].includes(lk)) return;
+        res.setHeader(key, val);
+      });
+      res.send(Buffer.from(await up.arrayBuffer()));
+    } catch (e) {
+      res.status(502).json({ error: `tollbooth proxy failed: ${e.message}` });
+    }
+  };
+}
+
+function readBody(req, cap = 10 * 1024 * 1024) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let n = 0;
+    req.on("data", (c) => {
+      n += c.length;
+      if (n > cap) { reject(new Error("request body too large")); req.destroy(); return; }
+      chunks.push(c);
+    });
+    req.on("end", () => resolve(Buffer.concat(chunks)));
+    req.on("error", reject);
+  });
+}
+
+async function startCli() {
+  const upstream = process.env.TOLLBOOTH_UPSTREAM;
+  const port = Number(process.env.PORT) || 4021;
+  const { default: express } = await import("express");
+  const app = express();
+  const gate = createTollbooth({ resourceBaseUrl: process.env.TOLLBOOTH_RESOURCE_BASE || upstream || "" });
+  app.use(gate);
+  if (upstream) {
+    app.use(createProxy(upstream));
+  } else {
+    app.use((_req, res) => res.json({ ok: true, note: "Bare tollbooth gate (no TOLLBOOTH_UPSTREAM set). Clients that reach here paid or solved a proof-of-work." }));
+  }
+  app.listen(port, () => {
+    const rails = `${gate.pow ? "proof-of-work" : ""}${process.env.TOLLBOOTH_PAYTO ? (gate.pow ? " + x402(USDC)" : "x402(USDC)") : ""}`;
+    console.log(`agent402-tollbooth listening on :${port} — charging AI bots via ${rails || "proof-of-work"}`);
+    if (upstream) console.log(`  proxying → ${upstream}`);
+  });
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) startCli();

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "agent402-tollbooth",
+  "version": "0.1.0",
+  "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
+  "type": "module",
+  "main": "index.js",
+  "bin": {
+    "agent402-tollbooth": "index.js"
+  },
+  "exports": {
+    ".": "./index.js"
+  },
+  "files": [
+    "index.js",
+    "bots.js",
+    "pow.js",
+    "example.js",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "test": "node test.js",
+    "example": "node example.js"
+  },
+  "dependencies": {
+    "express": "^5.0.0"
+  },
+  "keywords": [
+    "x402",
+    "pay-per-crawl",
+    "ai-bots",
+    "ai-crawler",
+    "bot-monetization",
+    "http-402",
+    "402-payment-required",
+    "proof-of-work",
+    "usdc",
+    "reverse-proxy",
+    "express-middleware",
+    "self-hosted",
+    "mcp",
+    "ai-agents"
+  ],
+  "author": "Mikey Petrillo (https://github.com/MikeyPetrillo)",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MikeyPetrillo/Agent402.git",
+    "directory": "tollbooth"
+  },
+  "homepage": "https://github.com/MikeyPetrillo/Agent402/tree/main/tollbooth"
+}

package/pow.js

@@ -0,0 +1,78 @@
+// Self-contained proof-of-work for the tollbooth: a no-wallet payment rail.
+// A client without USDC proves it spent real CPU instead — the same scheme
+// Agent402 uses, packaged standalone so the tollbooth has zero crypto deps.
+//
+// Tokens are HMAC-signed by this process (stateless to issue), bound to the
+// exact resource they were minted for, expiry-checked, and single-use (a solved
+// token is remembered until it expires). Solving costs the caller CPU; issuing
+// and verifying cost us one hash + one HMAC.
+import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto";
+
+const TTL_MS = 5 * 60 * 1000;
+
+/** Count leading zero bits of a buffer (the proof-of-work difficulty metric). */
+export function leadingZeroBits(buf) {
+  let bits = 0;
+  for (const b of buf) {
+    if (b === 0) { bits += 8; continue; }
+    bits += Math.clz32(b) - 24;
+    break;
+  }
+  return bits;
+}
+
+export function createPow({
+  secret = process.env.TOLLBOOTH_SECRET || randomBytes(32).toString("hex"),
+  difficulty = Number(process.env.TOLLBOOTH_POW_BITS) || 18,
+  ttlMs = TTL_MS,
+} = {}) {
+  const used = new Map(); // token -> expiry(ms)
+  const sweep = setInterval(() => {
+    const now = Date.now();
+    for (const [t, exp] of used) if (exp < now) used.delete(t);
+  }, 60_000);
+  sweep.unref?.();
+
+  const sign = (payload) => createHmac("sha256", secret).update(payload).digest("base64url");
+
+  /** Mint a signed, resource-scoped challenge. */
+  function challenge(resource) {
+    const chal = randomBytes(16).toString("hex");
+    const exp = Date.now() + ttlMs;
+    const payload = `${chal}.${exp}.${difficulty}.${resource}`;
+    const token = `${payload}.${sign(payload)}`;
+    return {
+      algorithm: "sha256",
+      challenge: chal,
+      difficulty,
+      expires: exp,
+      token,
+      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${difficulty} leading zero bits, then resend the request with header  X-Pow-Solution: ${token}:<nonce>`,
+    };
+  }
+
+  /** Verify a "<token>:<nonce>" solution for a given resource. */
+  function verify(headerValue, resource) {
+    if (!headerValue || typeof headerValue !== "string") return { ok: false, reason: "missing solution" };
+    const cut = headerValue.lastIndexOf(":");
+    if (cut < 0) return { ok: false, reason: "malformed solution" };
+    const token = headerValue.slice(0, cut);
+    const nonce = headerValue.slice(cut + 1);
+    const parts = token.split(".");
+    if (parts.length !== 5) return { ok: false, reason: "malformed token" };
+    const [chal, expStr, diffStr, res, sig] = parts;
+    const expected = sign(`${chal}.${expStr}.${diffStr}.${res}`);
+    const a = Buffer.from(sig);
+    const b = Buffer.from(expected);
+    if (a.length !== b.length || !timingSafeEqual(a, b)) return { ok: false, reason: "bad signature" };
+    if (res !== resource) return { ok: false, reason: "wrong resource" };
+    if (Date.now() > Number(expStr)) return { ok: false, reason: "expired" };
+    if (used.has(token)) return { ok: false, reason: "already used" };
+    const hash = createHash("sha256").update(`${chal}:${nonce}`).digest();
+    if (leadingZeroBits(hash) < Number(diffStr)) return { ok: false, reason: "insufficient work" };
+    used.set(token, Number(expStr));
+    return { ok: true };
+  }
+
+  return { challenge, verify, difficulty };
+}