agent402-tollbooth 0.1.0 → 0.1.2

package/README.md

@@ -10,6 +10,31 @@ Stack Overflow) shipped pay-per-crawl as a closed, fiat, you-must-be-on-our-CDN
 feature. This is the open, crypto-native, run-it-yourself version — built on the
 same hardened 402 + proof-of-work machinery as [Agent402](https://github.com/MikeyPetrillo/Agent402).
 
+## See it work (one command)
+
+```bash
+npx agent402-tollbooth   # then, in the repo:  npm run --prefix tollbooth demo
+```
+
+```text
+agent402-tollbooth — live pay-per-crawl demo
+
+① A human opens the page (normal browser)
+   → HTTP 200 FREE  "📄 The Future of Machine Payments — full article text…"
+   Humans are never charged.
+
+② An AI crawler hits the same page (ClaudeBot)
+   → HTTP 402 Payment Required
+   pay with USDC: $0.002 USDC on base → 0x…
+   …or free with proof-of-work: a 18-bit sha256 puzzle
+
+③ The crawler has no wallet, so it spends CPU instead
+   solved in 0.32s (nonce=100208)
+   → HTTP 200 OK (paid via pow)  "📄 The Future of Machine Payments — full article text…"
+
+✓ Pay-per-crawl, end to end — humans free, bots pay (USDC or compute).
+```
+
 ## Install
 
 ```bash
@@ -61,6 +86,41 @@ TOLLBOOTH_PAYTO=0xYourWallet \
 npx agent402-tollbooth          # listens on :4021, proxies humans free, charges bots
 ```
 
+## Run on the edge (Cloudflare Workers, Next.js, Deno, Bun)
+
+The same gate is also built on the Web Crypto + Fetch APIs (`edge.js`), so it runs
+anywhere — no Node required. The gate returns a `402 Response` when the client
+must pay, or `null` to let it through.
+
+**Cloudflare Workers** (poetic: the open pay-per-crawl, on the incumbent's own platform). `wrangler.toml`:
+
+```toml
+name = "tollbooth"
+main = "node_modules/agent402-tollbooth/worker.js"
+[vars]
+TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
+TOLLBOOTH_PAYTO    = "0xYourWallet"   # optional: advertise a USDC x402 quote
+# wrangler secret put TOLLBOOTH_SECRET
+# optional single-use replay store:  [[kv_namespaces]] binding = "TOLLBOOTH_KV"
+```
+
+**Next.js middleware** (`middleware.ts`):
+
+```ts
+import { createEdgeTollbooth } from "agent402-tollbooth/edge";
+const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET! });
+
+export async function middleware(req: Request) {
+  return (await gate(req)) ?? NextResponse.next();
+}
+```
+
+**Any Fetch-API runtime** (Deno, Bun, custom): `const gate = createEdgeTollbooth({ secret }); const blocked = await gate(request); return blocked ?? fetch(request);`
+
+> On the edge, pass a stable `secret` (PoW tokens are HMAC-signed). For
+> single-use replay protection across stateless invocations, supply a `store`
+> (e.g. a Cloudflare KV wrapper — the Worker entry wires this for you).
+
 ## Accepting USDC (x402)
 
 The proof-of-work rail works with **zero config**. To also settle real USDC,

package/demo.js

@@ -0,0 +1,58 @@
+// Live, narrated pay-per-crawl demo. Spins up a "premium content" site behind
+// the tollbooth, then shows a human getting it free and an AI crawler paying to
+// get through (here: by solving a proof-of-work, since that needs no wallet).
+//
+//   node demo.js        (or: npm run demo)
+import express from "express";
+import { createHash } from "node:crypto";
+import { createTollbooth } from "./index.js";
+
+const c = { dim: "\x1b[2m", reset: "\x1b[0m", green: "\x1b[32m", yellow: "\x1b[33m", cyan: "\x1b[36m", bold: "\x1b[1m" };
+const log = (s = "") => console.log(s);
+const lz = (buf) => { let b = 0; for (const x of buf) { if (x === 0) { b += 8; continue; } b += Math.clz32(x) - 24; break; } return b; };
+
+// A "premium content" site, fronted by the tollbooth.
+const app = express();
+app.use(createTollbooth({ price: "$0.002", payTo: "0xYourWalletHere000000000000000000000000", powDifficulty: 18 }));
+app.use((_req, res) => res.send("📄 The Future of Machine Payments — full article text…"));
+const server = app.listen(0);
+const base = `http://localhost:${server.address().port}`;
+const visit = (ua, extra = {}) => fetch(`${base}/article`, { headers: { "user-agent": ua, ...extra } });
+
+const HUMAN = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";
+const CRAWLER = "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)";
+
+try {
+  log(`${c.bold}agent402-tollbooth — live pay-per-crawl demo${c.reset}`);
+  log(`${c.dim}A premium-content site is now running behind the tollbooth.${c.reset}\n`);
+
+  // 1) A human browses — free.
+  log(`${c.cyan}① A human opens the page (normal browser)${c.reset}`);
+  let r = await visit(HUMAN);
+  log(`   → HTTP ${r.status} ${c.green}FREE${c.reset}  "${await r.text()}"`);
+  log(`   ${c.dim}Humans are never charged.${c.reset}\n`);
+
+  // 2) An AI crawler hits the same page — 402.
+  log(`${c.cyan}② An AI crawler hits the same page (ClaudeBot)${c.reset}`);
+  r = await visit(CRAWLER);
+  const q = await r.json();
+  const a = q.accepts?.[0];
+  log(`   → HTTP ${r.status} ${c.yellow}Payment Required${c.reset}`);
+  if (a) log(`   ${c.dim}pay with USDC:${c.reset} ${a.maxAmountRequired} ${a.asset} on ${a.network} → ${a.payTo}`);
+  log(`   ${c.dim}…or free with proof-of-work:${c.reset} a ${q.proofOfWork.difficulty}-bit sha256 puzzle\n`);
+
+  // 3) The crawler has no wallet, so it solves the proof-of-work.
+  log(`${c.cyan}③ The crawler has no wallet, so it spends CPU instead${c.reset}`);
+  const t0 = Date.now();
+  let nonce = 0;
+  while (lz(createHash("sha256").update(`${q.proofOfWork.challenge}:${nonce}`).digest()) < q.proofOfWork.difficulty) nonce++;
+  log(`   ${c.dim}solved in ${((Date.now() - t0) / 1000).toFixed(2)}s (nonce=${nonce})${c.reset}`);
+
+  // 4) Retry with the solution — unlocked.
+  r = await visit(CRAWLER, { "x-pow-solution": `${q.proofOfWork.token}:${nonce}` });
+  log(`   → HTTP ${r.status} ${c.green}OK${c.reset} (paid via ${r.headers.get("x-tollbooth-paid")})  "${await r.text()}"\n`);
+
+  log(`${c.bold}${c.green}✓ Pay-per-crawl, end to end${c.reset} — humans free, bots pay (USDC or compute). No Cloudflare, no Stripe, no signup.`);
+} finally {
+  server.close();
+}

package/edge.js

@@ -0,0 +1,144 @@
+// Edge / runtime-agnostic tollbooth — the same pay-per-crawl gate, built on the
+// Web Crypto + Fetch APIs so it runs anywhere: Cloudflare Workers, Next.js
+// middleware (edge), Deno, Bun, and Node 20+. No node:* imports.
+//
+//   const gate = createEdgeTollbooth({ secret: env.TOLLBOOTH_SECRET });
+//   const blocked = await gate(request);   // Response(402) if must pay, else null
+//
+// See worker.js for a Cloudflare Worker entry, and the README for Next.js.
+import { makeBotMatcher, AI_BOTS } from "./bots.js";
+
+const te = new TextEncoder();
+
+function b64url(buf) {
+  const bytes = new Uint8Array(buf);
+  let s = "";
+  for (let i = 0; i < bytes.length; i++) s += String.fromCharCode(bytes[i]);
+  return btoa(s).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+const toHex = (bytes) => [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
+function leadingZeroBits(bytes) {
+  let n = 0;
+  for (const b of bytes) { if (b === 0) { n += 8; continue; } n += Math.clz32(b) - 24; break; }
+  return n;
+}
+async function hmac(secret, payload) {
+  const key = await crypto.subtle.importKey("raw", te.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
+  return b64url(await crypto.subtle.sign("HMAC", key, te.encode(payload)));
+}
+const sha256 = async (str) => new Uint8Array(await crypto.subtle.digest("SHA-256", te.encode(str)));
+function constEq(a, b) {
+  if (a.length !== b.length) return false;
+  let r = 0;
+  for (let i = 0; i < a.length; i++) r |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return r === 0;
+}
+
+/**
+ * Web Crypto proof-of-work. `secret` is required (edge runtimes can't keep a
+ * random per-process secret across invocations). `store` is an optional
+ * single-use replay backend ({ has(token), add(token, expMs) }, both async) —
+ * e.g. a Cloudflare KV wrapper; defaults to in-isolate memory.
+ */
+export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000, store } = {}) {
+  if (!secret) throw new Error("createEdgePow requires a stable `secret` string");
+  const mem = new Map();
+  const used = store || {
+    has: async (k) => { const e = mem.get(k); if (e && e < Date.now()) { mem.delete(k); return false; } return mem.has(k); },
+    add: async (k, exp) => { mem.set(k, exp); },
+  };
+
+  async function challenge(resource) {
+    const rnd = new Uint8Array(16);
+    crypto.getRandomValues(rnd);
+    const chal = toHex(rnd);
+    const exp = Date.now() + ttlMs;
+    const payload = `${chal}.${exp}.${difficulty}.${resource}`;
+    const token = `${payload}.${await hmac(secret, payload)}`;
+    return {
+      algorithm: "sha256",
+      challenge: chal,
+      difficulty,
+      expires: exp,
+      token,
+      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${difficulty} leading zero bits, then resend with header  X-Pow-Solution: ${token}:<nonce>`,
+    };
+  }
+
+  async function verify(headerValue, resource) {
+    if (!headerValue || typeof headerValue !== "string") return { ok: false, reason: "missing solution" };
+    const cut = headerValue.lastIndexOf(":");
+    if (cut < 0) return { ok: false, reason: "malformed solution" };
+    const token = headerValue.slice(0, cut);
+    const nonce = headerValue.slice(cut + 1);
+    const parts = token.split(".");
+    if (parts.length !== 5) return { ok: false, reason: "malformed token" };
+    const [chal, expStr, diffStr, res, sig] = parts;
+    const expected = await hmac(secret, `${chal}.${expStr}.${diffStr}.${res}`);
+    if (!constEq(sig, expected)) return { ok: false, reason: "bad signature" };
+    if (res !== resource) return { ok: false, reason: "wrong resource" };
+    if (Date.now() > Number(expStr)) return { ok: false, reason: "expired" };
+    if (await used.has(token)) return { ok: false, reason: "already used" };
+    const h = await sha256(`${chal}:${nonce}`);
+    if (leadingZeroBits(h) < Number(diffStr)) return { ok: false, reason: "insufficient work" };
+    await used.add(token, Number(expStr));
+    return { ok: true };
+  }
+
+  return { challenge, verify, difficulty };
+}
+
+/**
+ * Create the edge gate. Returns `async (request) => Response | null`:
+ * a 402 `Response` when the client must pay, or `null` to allow the request
+ * through (proxy it, or `NextResponse.next()`).
+ */
+export function createEdgeTollbooth(config = {}) {
+  const {
+    price = "$0.001",
+    payTo = null,
+    network = "base",
+    asset = "USDC",
+    pow = true,
+    powDifficulty,
+    secret,
+    store,
+    botUserAgents = AI_BOTS,
+    charge,
+    free,
+    resourceBaseUrl = "",
+    message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
+  } = config;
+
+  const isBot = makeBotMatcher(botUserAgents);
+  const powEngine = pow ? createEdgePow({ secret, difficulty: powDifficulty, store }) : null;
+
+  const shouldCharge = (request) => {
+    if (typeof free === "function" && free(request)) return false;
+    if (typeof charge === "function") return Boolean(charge(request));
+    return isBot(request.headers.get("user-agent") || "");
+  };
+
+  return async function gate(request) {
+    if (!shouldCharge(request)) return null;
+    const u = new URL(request.url);
+    const resource = (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + u.pathname + u.search;
+
+    const sol = request.headers.get("x-pow-solution");
+    if (powEngine && sol) {
+      const r = await powEngine.verify(sol, resource);
+      if (r.ok) return null;
+    }
+
+    const body = {
+      error: "Payment Required",
+      message,
+      accepts: payTo ? [{ scheme: "exact", network, maxAmountRequired: String(price), asset, payTo, resource }] : [],
+    };
+    if (powEngine) body.proofOfWork = await powEngine.challenge(resource);
+    return new Response(JSON.stringify(body), {
+      status: 402,
+      headers: { "content-type": "application/json", "x-tollbooth": "pay-per-crawl" },
+    });
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent402-tollbooth",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
   "type": "module",
   "main": "index.js",
@@ -12,16 +12,20 @@
   },
   "files": [
     "index.js",
+    "edge.js",
+    "worker.js",
     "bots.js",
     "pow.js",
     "example.js",
+    "demo.js",
     "README.md"
   ],
   "engines": {
     "node": ">=20"
   },
   "scripts": {
-    "test": "node test.js",
+    "test": "node test.js && node edge.test.js",
+    "demo": "node demo.js",
     "example": "node example.js"
   },
   "dependencies": {
@@ -39,6 +43,9 @@
     "usdc",
     "reverse-proxy",
     "express-middleware",
+    "cloudflare-workers",
+    "nextjs",
+    "edge",
     "self-hosted",
     "mcp",
     "ai-agents"

package/worker.js

@@ -0,0 +1,47 @@
+// Cloudflare Worker entry for the tollbooth — open-source pay-per-crawl on the
+// edge. Deploy in front of your origin: humans pass through, AI crawlers pay
+// (USDC via x402, or free proof-of-work).
+//
+// wrangler.toml:
+//   name = "tollbooth"
+//   main = "node_modules/agent402-tollbooth/worker.js"
+//   [vars]
+//   TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
+//   TOLLBOOTH_PAYTO    = "0xYourWallet"        # optional (advertises USDC quote)
+//   # secret (required): wrangler secret put TOLLBOOTH_SECRET
+//   # optional single-use store: [[kv_namespaces]] binding = "TOLLBOOTH_KV"
+import { createEdgeTollbooth } from "./edge.js";
+
+const kvStore = (kv) => ({
+  has: async (k) => (await kv.get(k)) != null,
+  add: async (k, expMs) => kv.put(k, "1", { expiration: Math.ceil(expMs / 1000) }),
+});
+
+export default {
+  async fetch(request, env) {
+    if (!env.TOLLBOOTH_SECRET) {
+      return new Response("Tollbooth misconfigured: set TOLLBOOTH_SECRET (wrangler secret put TOLLBOOTH_SECRET)", { status: 500 });
+    }
+    const gate = createEdgeTollbooth({
+      secret: env.TOLLBOOTH_SECRET,
+      price: env.TOLLBOOTH_PRICE || "$0.001",
+      payTo: env.TOLLBOOTH_PAYTO || null,
+      network: env.TOLLBOOTH_NETWORK || "base",
+      powDifficulty: env.TOLLBOOTH_POW_BITS ? Number(env.TOLLBOOTH_POW_BITS) : undefined,
+      store: env.TOLLBOOTH_KV ? kvStore(env.TOLLBOOTH_KV) : undefined,
+    });
+
+    const blocked = await gate(request);
+    if (blocked) return blocked;
+
+    // Allowed → proxy to the origin.
+    const upstream = env.TOLLBOOTH_UPSTREAM;
+    if (!upstream) return new Response("Tollbooth: set TOLLBOOTH_UPSTREAM to your origin", { status: 500 });
+    const target = new URL(request.url);
+    const origin = new URL(upstream);
+    target.protocol = origin.protocol;
+    target.hostname = origin.hostname;
+    target.port = origin.port;
+    return fetch(new Request(target.toString(), request));
+  },
+};