agent402-tollbooth 0.1.2 → 0.1.3

package/README.md

@@ -170,9 +170,25 @@ Amazonbot, …). Classic search indexers (Googlebot, Bingbot) are intentionally
 or take full control with a `charge(req)` predicate (e.g. charge anything
 without a browser-like `Accept` header, or charge everyone on `/api/*`).
 
+## Production checklist (read this)
+
+- **Set a stable `TOLLBOOTH_SECRET`.** Required for any multi-process/clustered
+  Node deploy and for all edge deploys — without it, proof-of-work tokens use a
+  random per-process secret and are rejected across restarts/workers/isolates.
+- **For serverless/edge, supply a durable replay `store`** (e.g. bind a Cloudflare
+  KV namespace as `TOLLBOOTH_KV`). The in-memory default is per-isolate, so a
+  solved token could be reused across isolates within its TTL. The Worker entry
+  warns when no KV is bound.
+- **The reverse proxy pins the host** to your configured upstream (a client can't
+  redirect it elsewhere) and **strips client-forged trust/forwarding headers**
+  (`X-Tollbooth-Paid`, `X-Forwarded-Host`, etc.) before forwarding.
+- **UA matching is the default, not a security boundary** — a bot can forge a
+  human UA to get the *same free access a human gets* (it gains nothing more). If
+  you need to charge everyone, pass `charge: () => true`.
+
 ## Notes
 
 - Proof-of-work tokens are HMAC-signed, expiry-checked, single-use, and bound to
-  the exact resource — a solution for `/a` can't be replayed on `/b` or reused.
-- Set a stable `TOLLBOOTH_SECRET` in production so tokens survive restarts.
+  the exact resource (path + query, dots and all) — a solution for one URL can't
+  be replayed or reused on another.
 - MIT licensed. Part of [Agent402](https://github.com/MikeyPetrillo/Agent402).

package/edge.js

@@ -43,9 +43,17 @@ function constEq(a, b) {
 export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000, store } = {}) {
   if (!secret) throw new Error("createEdgePow requires a stable `secret` string");
   const mem = new Map();
+  const MAX = 50_000; // bound memory: edge runtimes have no timer to sweep, so prune on write
   const used = store || {
     has: async (k) => { const e = mem.get(k); if (e && e < Date.now()) { mem.delete(k); return false; } return mem.has(k); },
-    add: async (k, exp) => { mem.set(k, exp); },
+    add: async (k, exp) => {
+      if (mem.size >= MAX) {
+        const now = Date.now();
+        for (const [kk, ee] of mem) if (ee < now) mem.delete(kk);          // drop expired first
+        if (mem.size >= MAX) mem.delete(mem.keys().next().value);          // hard cap: evict oldest
+      }
+      mem.set(k, exp);
+    },
   };
 
   async function challenge(resource) {
@@ -72,9 +80,11 @@ export function createEdgePow({ secret, difficulty = 18, ttlMs = 5 * 60 * 1000,
     const token = headerValue.slice(0, cut);
     const nonce = headerValue.slice(cut + 1);
     const parts = token.split(".");
-    if (parts.length !== 5) return { ok: false, reason: "malformed token" };
-    const [chal, expStr, diffStr, res, sig] = parts;
-    const expected = await hmac(secret, `${chal}.${expStr}.${diffStr}.${res}`);
+    if (parts.length < 5) return { ok: false, reason: "malformed token" };
+    const sig = parts.pop();
+    const [chal, expStr, diffStr] = parts;
+    const res = parts.slice(3).join("."); // resource may itself contain dots (e.g. /post.html?v=1.2)
+    const expected = await hmac(secret, parts.join("."));
     if (!constEq(sig, expected)) return { ok: false, reason: "bad signature" };
     if (res !== resource) return { ok: false, reason: "wrong resource" };
     if (Date.now() > Number(expStr)) return { ok: false, reason: "expired" };
@@ -114,8 +124,10 @@ export function createEdgeTollbooth(config = {}) {
   const powEngine = pow ? createEdgePow({ secret, difficulty: powDifficulty, store }) : null;
 
   const shouldCharge = (request) => {
-    if (typeof free === "function" && free(request)) return false;
-    if (typeof charge === "function") return Boolean(charge(request));
+    try {
+      if (typeof free === "function" && free(request)) return false;
+      if (typeof charge === "function") return Boolean(charge(request));
+    } catch { return true; /* fail closed: charge on predicate error */ }
     return isBot(request.headers.get("user-agent") || "");
   };
 

package/index.js

@@ -13,12 +13,22 @@
 // accept USDC, set `payTo` and supply `verifyX402` (wire it to the standard
 // x402 server middleware / your facilitator — see README).
 import { fileURLToPath } from "node:url";
+import { Readable } from "node:stream";
 import { createPow } from "./pow.js";
 import { makeBotMatcher, AI_BOTS } from "./bots.js";
 
 export { AI_BOTS, makeBotMatcher } from "./bots.js";
 export { createPow, leadingZeroBits } from "./pow.js";
 
+const VERIFY_TIMEOUT_MS = Number(process.env.TOLLBOOTH_VERIFY_TIMEOUT_MS) || 10_000;
+// Headers a client must never be able to forge through the proxy: the gate's own
+// trust signals and forwarding/hop-by-hop headers.
+const STRIP_INBOUND = new Set([
+  "host", "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+  "te", "trailer", "transfer-encoding", "upgrade", "content-length",
+  "x-tollbooth-paid", "x-tollbooth-error", "x-pow-error", "x-forwarded-host", "forwarded",
+]);
+
 /**
  * Create the tollbooth Express middleware.
  * @param {object} [config]
@@ -56,12 +66,20 @@ export function createTollbooth(config = {}) {
   const powEngine = pow ? createPow({ difficulty: powDifficulty, secret: powSecret }) : null;
 
   const shouldCharge = (req) => {
-    if (typeof free === "function" && free(req)) return false;
-    if (typeof charge === "function") return Boolean(charge(req));
+    try {
+      if (typeof free === "function" && free(req)) return false;
+      if (typeof charge === "function") return Boolean(charge(req));
+    } catch { return true; /* fail closed: charge on predicate error */ }
     return isBot(req.headers["user-agent"] || "");
   };
-  const resourceOf = (req) =>
-    (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + (req.originalUrl || req.url || "/");
+  const resourceOf = (req) => {
+    // Canonicalize to path+search (matches the edge impl) so the PoW binding is
+    // stable and not confusable via a raw/abnormal request target.
+    const raw = req.originalUrl || req.url || "/";
+    let pathAndSearch = raw;
+    try { const u = new URL(raw, "http://internal.invalid"); pathAndSearch = u.pathname + u.search; } catch { /* keep raw */ }
+    return (resourceBaseUrl ? resourceBaseUrl.replace(/\/$/, "") : "") + pathAndSearch;
+  };
 
   function tollbooth(req, res, next) {
     if (!shouldCharge(req)) return next();
@@ -92,7 +110,9 @@ export function createTollbooth(config = {}) {
     // we reuse the standard, audited x402 stack rather than reinvent it.
     const payHeader = req.headers["x-payment"] || req.headers["payment-signature"];
     if (payTo && typeof verifyX402 === "function" && payHeader) {
-      return Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource }))
+      // Bound verification time so a slow/hung verifier can't exhaust resources.
+      const timeout = new Promise((resolve) => setTimeout(() => resolve(false), VERIFY_TIMEOUT_MS));
+      return Promise.race([Promise.resolve(verifyX402(req, { price, network, asset, payTo, resource })), timeout])
         .then((ok) => {
           if (ok) { res.setHeader("X-Tollbooth-Paid", "x402"); return next(); }
           send402();
@@ -108,19 +128,25 @@ export function createTollbooth(config = {}) {
   return tollbooth;
 }
 
-/** Minimal reverse proxy: forward a request to `upstream`, streaming-safe-ish. */
-export function createProxy(upstream) {
+/** Minimal reverse proxy: forward to `upstream` with the host PINNED (no SSRF via
+ *  the request target), client trust/forwarding headers stripped, and the
+ *  response STREAMED (no unbounded buffering). */
+export function createProxy(upstream, { maxBody = 10 * 1024 * 1024 } = {}) {
+  const base = new URL(upstream);
   return async (req, res) => {
     try {
-      const target = new URL(req.originalUrl || req.url, upstream);
+      // Take ONLY the path+query from the (possibly hostile) target; the
+      // authority is always the operator's upstream — protocol-relative or
+      // absolute-form targets cannot redirect us to another host.
+      const reqUrl = new URL(req.originalUrl || req.url || "/", base);
+      const target = new URL(reqUrl.pathname + reqUrl.search, base);
       const headers = {};
       for (const [k, v] of Object.entries(req.headers)) {
-        const lk = k.toLowerCase();
-        if (lk === "host" || lk === "connection" || lk === "content-length") continue;
-        headers[k] = v;
+        if (!STRIP_INBOUND.has(k.toLowerCase())) headers[k] = v;
       }
+      headers["x-forwarded-for"] = req.socket?.remoteAddress || ""; // set by us, not the client
       const method = req.method;
-      const body = method === "GET" || method === "HEAD" ? undefined : await readBody(req);
+      const body = method === "GET" || method === "HEAD" ? undefined : await readBody(req, maxBody);
       const up = await fetch(target, { method, headers, body, redirect: "manual" });
       res.status(up.status);
       up.headers.forEach((val, key) => {
@@ -129,9 +155,11 @@ export function createProxy(upstream) {
         if (["content-encoding", "content-length", "transfer-encoding", "connection"].includes(lk)) return;
         res.setHeader(key, val);
       });
-      res.send(Buffer.from(await up.arrayBuffer()));
+      if (up.body) Readable.fromWeb(up.body).pipe(res);
+      else res.end();
     } catch (e) {
-      res.status(502).json({ error: `tollbooth proxy failed: ${e.message}` });
+      if (!res.headersSent) res.status(502).json({ error: `tollbooth proxy failed: ${e.message}` });
+      else res.end();
     }
   };
 }
@@ -153,6 +181,9 @@ function readBody(req, cap = 10 * 1024 * 1024) {
 async function startCli() {
   const upstream = process.env.TOLLBOOTH_UPSTREAM;
   const port = Number(process.env.PORT) || 4021;
+  if (!process.env.TOLLBOOTH_SECRET) {
+    console.warn("⚠ TOLLBOOTH_SECRET not set — proof-of-work tokens use a random per-process secret: they won't survive a restart and will be rejected across multiple workers/instances. Set a stable TOLLBOOTH_SECRET in production.");
+  }
   const { default: express } = await import("express");
   const app = express();
   const gate = createTollbooth({ resourceBaseUrl: process.env.TOLLBOOTH_RESOURCE_BASE || upstream || "" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent402-tollbooth",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Open-source, self-hostable x402 pay-per-crawl tollbooth: charge AI bots per request (USDC via x402, or free proof-of-work) while humans browse free. Drop-in Express middleware or reverse proxy — no Cloudflare, no Stripe, no signup.",
   "type": "module",
   "main": "index.js",

package/pow.js

@@ -59,9 +59,11 @@ export function createPow({
     const token = headerValue.slice(0, cut);
     const nonce = headerValue.slice(cut + 1);
     const parts = token.split(".");
-    if (parts.length !== 5) return { ok: false, reason: "malformed token" };
-    const [chal, expStr, diffStr, res, sig] = parts;
-    const expected = sign(`${chal}.${expStr}.${diffStr}.${res}`);
+    if (parts.length < 5) return { ok: false, reason: "malformed token" };
+    const sig = parts.pop();
+    const [chal, expStr, diffStr] = parts;
+    const res = parts.slice(3).join("."); // resource may itself contain dots (e.g. /post.html?v=1.2)
+    const expected = sign(parts.join("."));
     const a = Buffer.from(sig);
     const b = Buffer.from(expected);
     if (a.length !== b.length || !timingSafeEqual(a, b)) return { ok: false, reason: "bad signature" };

package/worker.js

@@ -22,6 +22,11 @@ export default {
     if (!env.TOLLBOOTH_SECRET) {
       return new Response("Tollbooth misconfigured: set TOLLBOOTH_SECRET (wrangler secret put TOLLBOOTH_SECRET)", { status: 500 });
     }
+    if (!env.TOLLBOOTH_KV) {
+      // No durable store → replay protection is per-isolate only; a solved token
+      // can be reused across isolates within its TTL. Bind a KV namespace for prod.
+      console.warn("agent402-tollbooth: no TOLLBOOTH_KV bound — proof-of-work replay protection is per-isolate only. Bind a KV namespace for production.");
+    }
     const gate = createEdgeTollbooth({
       secret: env.TOLLBOOTH_SECRET,
       price: env.TOLLBOOTH_PRICE || "$0.001",
@@ -42,6 +47,12 @@ export default {
     target.protocol = origin.protocol;
     target.hostname = origin.hostname;
     target.port = origin.port;
-    return fetch(new Request(target.toString(), request));
+    // Strip client-forgeable trust/forwarding headers before forwarding to origin.
+    const headers = new Headers(request.headers);
+    for (const h of ["x-tollbooth-paid", "x-tollbooth-error", "x-pow-error", "x-forwarded-host", "forwarded"]) headers.delete(h);
+    headers.set("x-forwarded-for", request.headers.get("cf-connecting-ip") || "");
+    const init = { method: request.method, headers, redirect: "manual" };
+    if (request.method !== "GET" && request.method !== "HEAD") init.body = request.body;
+    return fetch(target.toString(), init);
   },
 };