agent402-tollbooth 0.1.1 → 0.1.3

package/README.md

@@ -86,6 +86,41 @@ TOLLBOOTH_PAYTO=0xYourWallet \
 npx agent402-tollbooth          # listens on :4021, proxies humans free, charges bots
 ```
 
+## Run on the edge (Cloudflare Workers, Next.js, Deno, Bun)
+
+The same gate is also built on the Web Crypto + Fetch APIs (`edge.js`), so it runs
+anywhere — no Node required. The gate returns a `402 Response` when the client
+must pay, or `null` to let it through.
+
+**Cloudflare Workers** (poetic: the open pay-per-crawl, on the incumbent's own platform). `wrangler.toml`:
+
+```toml
+name = "tollbooth"
+main = "node_modules/agent402-tollbooth/worker.js"
+[vars]
+TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
+TOLLBOOTH_PAYTO    = "0xYourWallet"   # optional: advertise a USDC x402 quote
+# wrangler secret put TOLLBOOTH_SECRET
+# optional single-use replay store:  [[kv_namespaces]] binding = "TOLLBOOTH_KV"
+```
+
+**Next.js middleware** (`middleware.ts`):
+
+```ts
+import { createEdgeTollbooth } from "agent402-tollbooth/edge";
+const gate = createEdgeTollbooth({ secret: process.env.TOLLBOOTH_SECRET! });
+
+export async function middleware(req: Request) {
+  return (await gate(req)) ?? NextResponse.next();
+}
+```
+
+**Any Fetch-API runtime** (Deno, Bun, custom): `const gate = createEdgeTollbooth({ secret }); const blocked = await gate(request); return blocked ?? fetch(request);`
+
+> On the edge, pass a stable `secret` (PoW tokens are HMAC-signed). For
+> single-use replay protection across stateless invocations, supply a `store`
+> (e.g. a Cloudflare KV wrapper — the Worker entry wires this for you).
+
 ## Accepting USDC (x402)
 
 The proof-of-work rail works with **zero config**. To also settle real USDC,
@@ -135,9 +170,25 @@ Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are intentionally
 or take full control with a `charge(req)` predicate (e.g. charge anything
 without a browser-like `Accept` header, or charge everyone on `/api/*`).
 
+## Production checklist (read this)
+
+- **Set a stable `TOLLBOOTH_SECRET`.** Required for any multi-process/clustered
+  Node deploy and for all edge deploys — without it, proof-of-work tokens use a
+  random per-process secret and are rejected across restarts/workers/isolates.
+- **For serverless/edge, supply a durable replay `store`** (e.g. bind a Cloudflare
+  KV namespace as `TOLLBOOTH_KV`). The in-memory default is per-isolate, so a
+  solved token could be reused across isolates within its TTL. The Worker entry
+  warns when no KV is bound.
+- **The reverse proxy pins the host** to your configured upstream (a client can't
+  redirect it elsewhere) and **strips client-forged trust/forwarding headers**
+  (`X-Tollbooth-Paid`, `X-Forwarded-Host`, etc.) before forwarding.
+- **UA matching is the default, not a security boundary** — a bot can forge a
+  human UA to get the *same free access a human gets* (it gains nothing more). If
+  you need to charge everyone, pass `charge: () => true`.
+
 ## Notes
 
 - Proof-of-work tokens are HMAC-signed, expiry-checked, single-use, and bound to
-  the exact resource — a solution for `/a` can't be replayed on `/b` or reused.
-- Set a stable `TOLLBOOTH_SECRET` in production so tokens survive restarts.
+  the exact resource (path + query, dots and all) — a solution for one URL can't
+  be replayed or reused on another.
 - MIT licensed. Part of [Agent402](https://github.com/MikeyPetrillo/Agent402).

package/edge.js

@@ -0,0 +1,156 @@
+// Edge / runtime-agnostic tollbooth — the same pay-per-crawl gate, built on the
+// Web Crypto + Fetch APIs so it runs anywhere: Cloudflare Workers, Next.js
+// middleware (edge), Deno, Bun, and Node 20+. No node:* imports.
+//
+//   const gate = createEdgeTollbooth({ secret: env.TOLLBOOTH_SECRET });
+//   const blocked = await gate(request);   // Response(402) if must pay, else null
+//
+// See worker.js for a Cloudflare Worker entry, and the README for Next.js.
+import { makeBotMatcher, AI_BOTS } from "./bots.js";
+
+const te = new TextEncoder();
+
+function b64url(buf) {
+  const bytes = new Uint8Array(buf);
+  let s = "";
+  for (let i = 0; i < bytes.length; i++) s += String.fromCharCode(bytes[i]);
+  return btoa(s).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+const toHex = (bytes) => [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
+function leadingZeroBits(bytes) {
+  let n = 0;
+  for (const b of bytes) { if (b === 0) { n += 8; continue; } n += Math.clz32(b) - 24; break; }
+  return n;
+}
+async function hmac(secret, payload) {
+  const key = await crypto.subtle.importKey("raw", te.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
+  return b64url(await crypto.subtle.sign("HMAC", key, te.encode(payload)));
+}
+const sha256 = async (str) => new Uint8Array(await crypto.subtle.digest("SHA-256", te.encode(str)));
+function constEq(a, b) {
+  if (a.length !== b.length) return false;
+  let r = 0;
+  for (let i = 0; i < a.length; i++) r |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return r === 0;
+}
+
+/**
+ * Web Crypto proof-of-work. `secret` is required (edge runtimes can't keep a
+ * random per-process secret across invocations). `store` is an optional
+ * single-use replay backend ({ has(token), add(token, expMs) }, both async) —
+ * e.g. a Cloudflare KV wrapper; defaults to in-isolate memory.
+ */
+export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000, store } = {}) {
+  if (!secret) throw new Error("createEdgePow requires a stable `secret` string");
+  const mem = new Map();
+  const MAX = 50_000; // bound memory: edge runtimes have no timer to sweep, so prune on write
+  const used = store || {
+    has: async (k) => { const e = mem.get(k); if (e && e < Date.now()) { mem.delete(k); return false; } return mem.has(k); },
+    add: async (k, exp) => {
+      if (mem.size >= MAX) {
+        const now = Date.now();
+        for (const [kk, ee] of mem) if (ee < now) mem.delete(kk);          // drop expired first
+        if (mem.size >= MAX) mem.delete(mem.keys().next().value);          // hard cap: evict oldest
+      }
+      mem.set(k, exp);
+    },
+  };
+
+  async function challenge(resource) {
+    const rnd = new Uint8Array(16);
+    crypto.getRandomValues(rnd);
+    const chal = toHex(rnd);
+    const exp = Date.now() + ttlMs;
+    const payload = `${chal}.${exp}.${difficulty}.${resource}`;
+    const token = `${payload}.${await hmac(secret, payload)}`;
+    return {
+      algorithm: "sha256",
+      challenge: chal,
+      difficulty,
+      expires: exp,
+      token,
+      rule: `Find an integer nonce so sha256("${chal}:" + nonce) has >= ${difficulty} leading zero bits, then resend with header  X-Pow-Solution: ${token}:<nonce>`,
+    };
+  }
+
+  async function verify(headerValue, resource) {
+    if (!headerValue || typeof headerValue !== "string") return { ok: false, reason: "missing solution" };
+    const cut = headerValue.lastIndexOf(":");
+    if (cut < 0) return { ok: false, reason: "malformed solution" };
+    const token = headerValue.slice(0, cut);
+    const nonce = headerValue.slice(cut + 1);
+    const parts = token.split(".");
+    if (parts.length < 5) return { ok: false, reason: "malformed token" };
+    const sig = parts.pop();
+    const [chal, expStr, diffStr] = parts;
+    const res = parts.slice(3).join("."); // resource may itself contain dots (e.g. /post.html?v=1.2)
+    const expected = await hmac(secret, parts.join("."));
+    if (!constEq(sig, expected)) return { ok: false, reason: "bad signature" };
+    if (res !== resource) return { ok: false, reason: "wrong resource" };
+    if (Date.now() > Number(expStr)) return { ok: false, reason: "expired" };
+    if (await used.has(token)) return { ok: false, reason: "already used" };
+    const h = await sha256(`${chal}:${nonce}`);
+    if (leadingZeroBits(h) < Number(diffStr)) return { ok: false, reason: "insufficient work" };
+    await used.add(token, Number(expStr));
+    return { ok: true };
+  }
+
+  return { challenge, verify, difficulty };
+}
+
+/**
+ * Create the edge gate. Returns `async (request) => Response | null`:
+ * a 402 `Response` when the client must pay, or `null` to allow the request
+ * through (proxy it, or `NextResponse.next()`).
+ */
+export function createEdgeTollbooth(config = {}) {
+  const {
+    price = "$0.001",
+    payTo = null,
+    network = "base",
+    asset = "USDC",
+    pow = true,
+    powDifficulty,
+    secret,
+    store,
+    botUserAgents = AI_BOTS,
+    charge,
+    free,
+    resourceBaseUrl = "",
+    message = "This resource charges automated / AI clients per request. Humans browse free; bots pay in USDC via x402 or by solving a proof-of-work.",
+  } = config;
+
+  const isBot = makeBotMatcher(botUserAgents);
+  const powEngine = pow ? createEdgePow({ secret, difficulty: powDifficulty, store }) : null;
+
+  const shouldCharge = (request) => {
+    try {
+      if (typeof free === "function" && free(request)) return false;
+      if (typeof charge === "function") return Boolean(charge(request));
+    } catch { return true; /* fail closed: charge on predicate error */ }
+    return isBot(request.headers.get("user-agent") || "");
+  };
+
+  return async function gate(request) {
+    if (!shouldCharge(request)) return null;
+    const u = new URL(request.url);
+    const resource = (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + u.pathname + u.search;
+
+    const sol = request.headers.get("x-pow-solution");
+    if (powEngine && sol) {
+      const r = await powEngine.verify(sol, resource);
+      if (r.ok) return null;
+    }
+
+    const body = {
+      error: "Payment Required",
+      message,
+      accepts: payTo ? [{ scheme: "exact", network, maxAmountRequired: String(price), asset, payTo, resource }] : [],
+    };
+    if (powEngine) body.proofOfWork = await powEngine.challenge(resource);
+    return new Response(JSON.stringify(body), {
+      status: 402,
+      headers: { "content-type": "application/json", "x-tollbooth": "pay-per-crawl" },
+    });
+  };
+}

package/index.js

@@ -13,12 +13,22 @@
 // accept USDC, set `payTo` and supply `verifyX402` (wire it to the standard
 // x402 server middleware / your facilitator — see README).
 import { fileURLToPath } from "node:url";
+import { Readable } from "node:stream";
 import { createPow } from "./pow.js";
 import { makeBotMatcher, AI_BOTS } from "./bots.js";
 
 export { AI_BOTS, makeBotMatcher } from "./bots.js";
 export { createPow, leadingZeroBits } from "./pow.js";
 
+const VERIFY_TIMEOUT_MS = Number(process.env.TOLLBOOTH_VERIFY_TIMEOUT_MS) || 10_000;
+// Headers a client must never be able to forge through the proxy: the gate's own
+// trust signals and forwarding/hop-by-hop headers.
+const STRIP_INBOUND = new Set([
+  "host", "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+  "te", "trailer", "transfer-encoding", "upgrade", "content-length",
+  "x-tollbooth-paid", "x-tollbooth-error", "x-pow-error", "x-forwarded-host", "forwarded",
+]);
+
 /**
  * Create the tollbooth Express middleware.
  * @param {object} [config]
@@ -56,12 +66,20 @@ export function createTollbooth(config = {}) {
   const powEngine = pow ? createPow({ difficulty: powDifficulty, secret: powSecret }) : null;
 
   const shouldCharge = (req) => {
-    if (typeof free === "function" && free(req)) return false;
-    if (typeof charge === "function") return Boolean(charge(req));
+    try {
+      if (typeof free === "function" && free(req)) return false;
+      if (typeof charge === "function") return Boolean(charge(req));
+    } catch { return true; /* fail closed: charge on predicate error */ }
     return isBot(req.headers["user-agent"] || "");
   };
-  const resourceOf = (req) =>
-    (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + (req.originalUrl || req.url || "/");
+  const resourceOf = (req) => {
+    // Canonicalize to path+search (matches the edge impl) so the PoW binding is
+    // stable and not confusable via a raw/abnormal request target.
+    const raw = req.originalUrl || req.url || "/";
+    let pathAndSearch = raw;
+    try { const u = new URL(raw, "http://internal.invalid"); pathAndSearch = u.pathname + u.search; } catch { /* keep raw */ }
+    return (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + pathAndSearch;
+  };
 
   function tollbooth(req, res, next) {
     if (!shouldCharge(req)) return next();
@@ -92,7 +110,9 @@ export function createTollbooth(config = {}) {
     // we reuse the standard, audited x402 stack rather than reinvent it.
     const payHeader = req.headers["x-payment"] || req.headers["payment-signature"];
     if (payTo && typeof verifyX402 === "function" && payHeader) {
-      return Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource }))
+      // Bound verification time so a slow/hung verifier can't exhaust resources.
+      const timeout = new Promise((resolve) => setTimeout(() => resolve(false), VERIFY_TIMEOUT_MS));
+      return Promise.race([Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource })), timeout])
         .then((ok) => {
           if (ok) { res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
           send402();
@@ -108,19 +128,25 @@ export function createTollbooth(config = {}) {
   return tollbooth;
 }
 
-/** Minimal reverse proxy: forward a request to `upstream`, streaming-safe-ish. */
-export function createProxy(upstream) {
+/** Minimal reverse proxy: forward to `upstream` with the host PINNED (no SSRF via
+ *  the request target), client trust/forwarding headers stripped, and the
+ *  response STREAMED (no unbounded buffering). */
+export function createProxy(upstream, { maxBody = 10 * 1024 * 1024 } = {}) {
+  const base = new URL(upstream);
   return async (req, res) => {
     try {
-      const target = new URL(req.originalUrl || req.url, upstream);
+      // Take ONLY the path+query from the (possibly hostile) target; the
+      // authority is always the operator's upstream — protocol-relative or
+      // absolute-form targets cannot redirect us to another host.
+      const reqUrl = new URL(req.originalUrl || req.url || "/", base);
+      const target = new URL(reqUrl.pathname + reqUrl.search, base);
       const headers = {};
       for (const [k, v] of Object.entries(req.headers)) {
-        const lk = k.toLowerCase();
-        if (lk === "host" || lk === "connection" || lk === "content-length") continue;
-        headers[k] = v;
+        if (!STRIP_INBOUND.has(k.toLowerCase())) headers[k] = v;
       }
+      headers["x-forwarded-for"] = req.socket?.remoteAddress || ""; // set by us, not the client
       const method = req.method;
-      const body = method === "GET" || method === "HEAD" ? undefined : await readBody(req);
+      const body = method === "GET" || method === "HEAD" ? undefined : await readBody(req, maxBody);
       const up = await fetch(target, { method, headers, body, redirect: "manual" });
       res.status(up.status);
       up.headers.forEach((val, key) => {
@@ -129,9 +155,11 @@ export function createProxy(upstream) {
         if (["content-encoding", "content-length", "transfer-encoding", "connection"].includes(lk)) return;
         res.setHeader(key, val);
       });
-      res.send(Buffer.from(await up.arrayBuffer()));
+      if (up.body) Readable.fromWeb(up.body).pipe(res);
+      else res.end();
     } catch (e) {
-      res.status(502).json({ error: `tollbooth proxy failed: ${e.message}` });
+      if (!res.headersSent) res.status(502).json({ error: `tollbooth proxy failed: ${e.message}` });
+      else res.end();
     }
   };
 }
@@ -153,6 +181,9 @@ function readBody(req, cap = 10 * 1024 * 1024) {
 async function startCli() {
   const upstream = process.env.TOLLBOOTH_UPSTREAM;
   const port = Number(process.env.PORT) || 4021;
+  if (!process.env.TOLLBOOTH_SECRET) {
+    console.warn("⚠ TOLLBOOTH_SECRET not set — proof-of-work tokens use a random per-process secret: they won't survive a restart and will be rejected across multiple workers/instances. Set a stable TOLLBOOTH_SECRET in production.");
+  }
   const { default: express } = await import("express");
   const app = express();
   const gate = createTollbooth({ resourceBaseUrl: process.env.TOLLBOOTH_RESOURCE_BASE || upstream || "" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent402-tollbooth",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
   "type": "module",
   "main": "index.js",
@@ -12,6 +12,8 @@
   },
   "files": [
     "index.js",
+    "edge.js",
+    "worker.js",
     "bots.js",
     "pow.js",
     "example.js",
@@ -22,7 +24,7 @@
     "node": ">=20"
   },
   "scripts": {
-    "test": "node test.js",
+    "test": "node test.js && node edge.test.js",
     "demo": "node demo.js",
     "example": "node example.js"
   },
@@ -41,6 +43,9 @@
     "usdc",
     "reverse-proxy",
     "express-middleware",
+    "cloudflare-workers",
+    "nextjs",
+    "edge",
     "self-hosted",
     "mcp",
     "ai-agents"

package/pow.js

@@ -59,9 +59,11 @@ export function createPow({
     const token = headerValue.slice(0, cut);
     const nonce = headerValue.slice(cut + 1);
     const parts = token.split(".");
-    if (parts.length !== 5) return { ok: false, reason: "malformed token" };
-    const [chal, expStr, diffStr, res, sig] = parts;
-    const expected = sign(`${chal}.${expStr}.${diffStr}.${res}`);
+    if (parts.length < 5) return { ok: false, reason: "malformed token" };
+    const sig = parts.pop();
+    const [chal, expStr, diffStr] = parts;
+    const res = parts.slice(3).join("."); // resource may itself contain dots (e.g. /post.html?v=1.2)
+    const expected = sign(parts.join("."));
     const a = Buffer.from(sig);
     const b = Buffer.from(expected);
     if (a.length !== b.length || !timingSafeEqual(a, b)) return { ok: false, reason: "bad signature" };

package/worker.js

@@ -0,0 +1,58 @@
+// Cloudflare Worker entry for the tollbooth — open-source pay-per-crawl on the
+// edge. Deploy in front of your origin: humans pass through, AI crawlers pay
+// (USDC via x402, or free proof-of-work).
+//
+// wrangler.toml:
+//   name = "tollbooth"
+//   main = "node_modules/agent402-tollbooth/worker.js"
+//   [vars]
+//   TOLLBOOTH_UPSTREAM = "https://your-origin.example.com"
+//   TOLLBOOTH_PAYTO    = "0xYourWallet"        # optional (advertises USDC quote)
+//   # secret (required): wrangler secret put TOLLBOOTH_SECRET
+//   # optional single-use store: [[kv_namespaces]] binding = "TOLLBOOTH_KV"
+import { createEdgeTollbooth } from "./edge.js";
+
+const kvStore = (kv) => ({
+  has: async (k) => (await kv.get(k)) != null,
+  add: async (k, expMs) => kv.put(k, "1", { expiration: Math.ceil(expMs / 1000) }),
+});
+
+export default {
+  async fetch(request, env) {
+    if (!env.TOLLBOOTH_SECRET) {
+      return new Response("Tollbooth misconfigured: set TOLLBOOTH_SECRET (wrangler secret put TOLLBOOTH_SECRET)", { status: 500 });
+    }
+    if (!env.TOLLBOOTH_KV) {
+      // No durable store → replay protection is per-isolate only; a solved token
+      // can be reused across isolates within its TTL. Bind a KV namespace for prod.
+      console.warn("agent402-tollbooth: no TOLLBOOTH_KV bound — proof-of-work replay protection is per-isolate only. Bind a KV namespace for production.");
+    }
+    const gate = createEdgeTollbooth({
+      secret: env.TOLLBOOTH_SECRET,
+      price: env.TOLLBOOTH_PRICE || "$0.001",
+      payTo: env.TOLLBOOTH_PAYTO || null,
+      network: env.TOLLBOOTH_NETWORK || "base",
+      powDifficulty: env.TOLLBOOTH_POW_BITS ? Number(env.TOLLBOOTH_POW_BITS) : undefined,
+      store: env.TOLLBOOTH_KV ? kvStore(env.TOLLBOOTH_KV) : undefined,
+    });
+
+    const blocked = await gate(request);
+    if (blocked) return blocked;
+
+    // Allowed → proxy to the origin.
+    const upstream = env.TOLLBOOTH_UPSTREAM;
+    if (!upstream) return new Response("Tollbooth: set TOLLBOOTH_UPSTREAM to your origin", { status: 500 });
+    const target = new URL(request.url);
+    const origin = new URL(upstream);
+    target.protocol = origin.protocol;
+    target.hostname = origin.hostname;
+    target.port = origin.port;
+    // Strip client-forgeable trust/forwarding headers before forwarding to origin.
+    const headers = new Headers(request.headers);
+    for (const h of ["x-tollbooth-paid", "x-tollbooth-error", "x-pow-error", "x-forwarded-host", "forwarded"]) headers.delete(h);
+    headers.set("x-forwarded-for", request.headers.get("cf-connecting-ip") || "");
+    const init = { method: request.method, headers, redirect: "manual" };
+    if (request.method !== "GET" && request.method !== "HEAD") init.body = request.body;
+    return fetch(target.toString(), init);
+  },
+};