@profullstack/x402-gateway 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Profullstack, LLC
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,110 @@
1
+ # @profullstack/x402-gateway
2
+
3
+ Sell crawl access to AI training crawlers, by the day, over [x402](https://x402.org), settled by [CoinPay](https://coinpayportal.com).
4
+
5
+ People read your site free. So do search engines and the retrieval crawlers behind AI answers, because they send readers back. A crawler that copies pages into a training corpus sends nobody back, so it pays: every page answers `402 Payment Required` with an x402 offer, paying the offer returns a signed pass, and the pass opens the site for a day.
6
+
7
+ One middleware. No database. Runs in Node, Bun and at the edge.
8
+
9
+ ```
10
+ npm install @profullstack/x402-gateway
11
+ ```
12
+
13
+ ## What it does
14
+
15
+ | Visitor | Gets |
16
+ | --- | --- |
17
+ | A person, Googlebot, Bingbot, Applebot, OAI-SearchBot, Claude-SearchBot, PerplexityBot… | the site, untouched |
18
+ | GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended… | `402` with an x402 offer, or the HTML sales page if it asked for HTML |
19
+ | Anyone at `/crawl` | the sales page (HTML) or the offer (JSON), and the place to pay |
20
+ | A request with a valid pass | the site |
21
+
22
+ The sales page explains the price, how to pay with an x402 client, and how to pay with the CoinPay CLI:
23
+
24
+ ```
25
+ npm install -g @profullstack/coinpay
26
+ coinpay x402 pay https://your-site.com/crawl --output pass.json
27
+ curl -H "x-crawl-pass: $(node -p "require('./pass.json').pass")" https://your-site.com/
28
+ ```
29
+
30
+ ## Hono
31
+
32
+ ```js
33
+ import { Hono } from 'hono';
34
+ import { createGateway } from '@profullstack/x402-gateway';
35
+ import { x402Gateway } from '@profullstack/x402-gateway/hono';
36
+
37
+ const gateway = createGateway({
38
+ siteUrl: 'https://your-site.com',
39
+ coinpay: { apiKey: process.env.COINPAY_X402_KEY },
40
+ payTo: process.env.CRAWL_PAY_TO,
41
+ });
42
+
43
+ const app = new Hono();
44
+ app.use('*', x402Gateway(gateway));
45
+ app.get('/robots.txt', (c) => c.text(gateway.robotsTxt({ disallow: ['/login', '/api/'] })));
46
+ ```
47
+
48
+ ## Next.js
49
+
50
+ `src/proxy.ts` on Next 16, `middleware.ts` before that. One per app; if you already have one, compose.
51
+
52
+ ```ts
53
+ import { createGateway } from '@profullstack/x402-gateway';
54
+ import { x402Proxy } from '@profullstack/x402-gateway/next';
55
+
56
+ export const gateway = createGateway({
57
+ siteUrl: 'https://your-site.com',
58
+ coinpay: { apiKey: process.env.COINPAY_X402_KEY },
59
+ payTo: process.env.CRAWL_PAY_TO,
60
+ });
61
+
62
+ export const proxy = x402Proxy(gateway);
63
+ export const config = { matcher: ['/((?!_next/static|_next/image|favicon.ico).*)'] };
64
+ ```
65
+
66
+ ```ts
67
+ // app/robots.txt/route.ts
68
+ import { robotsRoute } from '@profullstack/x402-gateway/next';
69
+ import { gateway } from '../../proxy';
70
+ export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
71
+ ```
72
+
73
+ ## Anything else
74
+
75
+ `gateway.handle(request)` takes a Fetch `Request` and resolves to a `Response` to send, or `null` to carry on. Wrap it in ten lines for whatever you run.
76
+
77
+ ## Options
78
+
79
+ | Option | Default | |
80
+ | --- | --- | --- |
81
+ | `siteUrl` | required | canonical origin, no trailing slash |
82
+ | `coinpay.apiKey` | | a **scoped** CoinPay key (`cp_live_…`, from the business's API Keys tab) with `payments:create`. The legacy business key is refused by CoinPay's x402 routes. |
83
+ | `payTo` | | EVM address that receives the USDC, on Base, Polygon and Ethereum alike |
84
+ | `priceCents` | `100` | |
85
+ | `passMinutes` | `1440` | a day |
86
+ | `header` | `x-crawl-pass` | where the pass goes; `Authorization: Bearer` works too |
87
+ | `path` | `/crawl` | the sales page |
88
+ | `openPaths` | `[]` | extra paths a refused crawler may read (`robots.txt`, the sales page, `security.txt` and `.well-known/` always are) |
89
+ | `isPaidAgent` | training list | `(userAgent) => boolean` |
90
+ | `training`, `retrieval` | the lists in `./agents` | |
91
+ | `secret` | the CoinPay key | pass signing secret |
92
+ | `page` | built in | `(ctx) => html` |
93
+ | `contact` | | mailto: or URL for bulk deals |
94
+ | `onSale` | | `({ payer, ref, token, expiresAt, userAgent, priceCents, currency }) => …`, for accounting |
95
+
96
+ Without `coinpay.apiKey` and `payTo` the gateway still answers training crawlers with 402 and the page says payments are off. Nothing is sold, but nothing is given away either.
97
+
98
+ ## How the money moves
99
+
100
+ The offer is x402 v2 in CoinPay's dialect: USDC under the `exact` scheme on Base, Polygon or Ethereum, EIP-3009 `transferWithAuthorization`. The buyer signs, the gateway sends the proof to CoinPay's `/api/x402/verify` and `/api/x402/settle`, and CoinPay's relayer broadcasts the transfer, paying the gas. The USDC goes straight to `payTo`.
101
+
102
+ A pass is `cp_<payload>.<hmac>`: its own expiry and the payment's nonce, signed with HMAC-SHA256. Verifying one is a hash, not a query. A proof is single-use; retrying with the same proof returns a pass bounded by the proof's own validity window, so a lost response is not a lost dollar and a replayed header is not a free day.
103
+
104
+ ## Who is on which list
105
+
106
+ `TRAINING_AGENTS` is the documented corpus-crawl token of each operator; `RETRIEVAL_AGENTS` the search half of the same pairs: GPTBot / OAI-SearchBot, ClaudeBot / Claude-SearchBot, meta-externalagent / Meta-ExternalFetcher, Applebot-Extended / Applebot. Google-Extended stays welcome because Google documents it as also gating Gemini app grounding. Matching is a substring of the user agent, which identifies a self-declared crawler and nothing more: a crawler wearing a browser's user agent walks past this, and that one needs blocking at the edge.
107
+
108
+ ## Licence
109
+
110
+ MIT
package/index.d.ts ADDED
@@ -0,0 +1,133 @@
1
+ /** A Fetch-API request handler that answers, or returns null to let the request through. */
2
+ export type Handle = (request: Request) => Promise<Response | null>;
3
+
4
+ export interface Sale {
5
+ payer: string | null;
6
+ ref: string | null;
7
+ token: string;
8
+ expiresAt: string;
9
+ userAgent: string;
10
+ priceCents: number;
11
+ currency: string;
12
+ }
13
+
14
+ export interface PageContext {
15
+ siteName: string;
16
+ siteUrl: string;
17
+ buyUrl: string;
18
+ price: string;
19
+ minutes: number;
20
+ header: string;
21
+ enabled: boolean;
22
+ offer: Offer;
23
+ training: string[];
24
+ retrieval: string[];
25
+ contact: string;
26
+ }
27
+
28
+ export interface GatewayOptions {
29
+ /** Canonical origin, no trailing slash. */
30
+ siteUrl: string;
31
+ /** Shown on the sales page; defaults to the hostname. */
32
+ siteName?: string;
33
+ /** A SCOPED CoinPay key (cp_live_… from the business's API Keys tab) with payments:create. */
34
+ coinpay?: { apiKey?: string; baseUrl?: string };
35
+ /** EVM address that receives the USDC. */
36
+ payTo?: string;
37
+ /** Default 100 ($1). */
38
+ priceCents?: number;
39
+ currency?: string;
40
+ /** What a payment buys. Default 1440 (a day). */
41
+ passMinutes?: number;
42
+ /** Request header the pass is presented in. Default 'x-crawl-pass'. */
43
+ header?: string;
44
+ /** The sales page. Default '/crawl'. */
45
+ path?: string;
46
+ /** Extra paths a refused crawler may still read. */
47
+ openPaths?: string[];
48
+ training?: string[];
49
+ retrieval?: string[];
50
+ /** Who is charged. Default: the training list, substring-matched on the user agent. */
51
+ isPaidAgent?: (userAgent: string) => boolean;
52
+ /** Pass signing secret. Defaults to the CoinPay key. */
53
+ secret?: string;
54
+ page?: (ctx: PageContext) => string;
55
+ contact?: string;
56
+ onSale?: (sale: Sale) => void | Promise<void>;
57
+ fetch?: typeof fetch;
58
+ }
59
+
60
+ export interface AcceptEntry {
61
+ scheme: 'exact';
62
+ network: string;
63
+ amount: string;
64
+ asset: string;
65
+ payTo: string;
66
+ resource: string;
67
+ description: string;
68
+ mimeType: string;
69
+ maxTimeoutSeconds: number;
70
+ extra: { name: string; version: string };
71
+ }
72
+
73
+ export interface Offer {
74
+ x402Version: 2;
75
+ accepts: AcceptEntry[];
76
+ }
77
+
78
+ export interface RobotsOptions {
79
+ siteUrl?: string;
80
+ disallow?: string[];
81
+ allow?: string[];
82
+ sitemap?: string;
83
+ path?: string;
84
+ refused?: string[];
85
+ training?: string[];
86
+ retrieval?: string[];
87
+ comments?: string[];
88
+ }
89
+
90
+ export interface Gateway {
91
+ handle: Handle;
92
+ sell: (request: Request) => Promise<Response>;
93
+ enabled: boolean;
94
+ options: Required<Omit<GatewayOptions, 'onSale' | 'fetch' | 'page' | 'isPaidAgent'>> & {
95
+ onSale: GatewayOptions['onSale'] | null;
96
+ fetch: typeof fetch;
97
+ page: (ctx: PageContext) => string;
98
+ isPaidAgent: (userAgent: string) => boolean;
99
+ };
100
+ robotsTxt: (extra?: RobotsOptions) => string;
101
+ page: () => string;
102
+ }
103
+
104
+ export function createGateway(options: GatewayOptions): Gateway;
105
+ export function wantsHtml(accept?: string | null): boolean;
106
+
107
+ export const TRAINING_AGENTS: string[];
108
+ export const RETRIEVAL_AGENTS: string[];
109
+ export function isTrainingAgent(userAgent?: string | null, agents?: string[]): boolean;
110
+
111
+ export function robotsTxt(options: RobotsOptions & { siteUrl: string }): string;
112
+ export function renderPage(ctx: PageContext): string;
113
+
114
+ export function mintPass(args: { secret: string; ref: string | null; expiresAt: number; now?: number }): Promise<{ token: string; expiresAt: number; ref: string | null }>;
115
+ export function readPass(token: string, args: { secret: string; now?: number }): Promise<{ exp: number; iat: number | null; ref: string | null } | null>;
116
+
117
+ export const METHODS: Array<{ key: string; network: string; asset: string; label: string }>;
118
+ export const X402_METHODS: typeof METHODS;
119
+ export function buildOffer(args: { payTo: string; priceCents: number; resource: string; description?: string; maxTimeoutSeconds?: number; methods?: typeof METHODS }): Offer;
120
+ export function decodePayment(header: string | null | undefined): Record<string, unknown> | null;
121
+ export function expectedFor(payment: unknown, offer: Offer): { amount: string; resource: string; payTo: string; asset: string } | null;
122
+ export function verifyAndSettle(
123
+ payment: unknown,
124
+ expected: { amount: string; resource: string; payTo: string; asset: string },
125
+ coinpay: { apiKey: string; baseUrl: string; fetch?: typeof fetch },
126
+ ): Promise<{ ok: true; payer: string | null; ref: string | null } | { ok: false; reason: string; replay: boolean }>;
127
+
128
+ /** ./hono */
129
+ export function x402Gateway(gatewayOrOptions: Gateway | GatewayOptions): (c: { req: { raw: Request } }, next: () => Promise<void>) => Promise<Response | undefined>;
130
+
131
+ /** ./next */
132
+ export function x402Proxy(gatewayOrOptions: Gateway | GatewayOptions): (request: Request) => Promise<Response | undefined>;
133
+ export function robotsRoute(gateway: Gateway, extra?: RobotsOptions): () => Response;
package/package.json ADDED
@@ -0,0 +1,42 @@
1
+ {
2
+ "name": "@profullstack/x402-gateway",
3
+ "version": "0.1.0",
4
+ "type": "module",
5
+ "description": "Sell crawl access to AI training crawlers by the day over x402, settled by CoinPay. One middleware: 402 with an offer, a sales page with CLI instructions, signed passes, and a robots.txt that keeps search crawlers welcome.",
6
+ "keywords": [
7
+ "x402",
8
+ "402",
9
+ "payment-required",
10
+ "crawler",
11
+ "ai-crawler",
12
+ "gptbot",
13
+ "claudebot",
14
+ "meta-externalagent",
15
+ "robots.txt",
16
+ "coinpay",
17
+ "usdc",
18
+ "hono",
19
+ "nextjs",
20
+ "middleware",
21
+ "paywall"
22
+ ],
23
+ "repository": { "type": "git", "url": "git+https://github.com/profullstack/x402-gateway.git" },
24
+ "homepage": "https://github.com/profullstack/x402-gateway#readme",
25
+ "bugs": { "url": "https://github.com/profullstack/x402-gateway/issues" },
26
+ "license": "MIT",
27
+ "author": "Profullstack, LLC",
28
+ "exports": {
29
+ ".": { "types": "./index.d.ts", "import": "./src/index.js" },
30
+ "./hono": { "types": "./index.d.ts", "import": "./src/hono.js" },
31
+ "./next": { "types": "./index.d.ts", "import": "./src/next.js" },
32
+ "./robots": { "types": "./index.d.ts", "import": "./src/robots.js" },
33
+ "./agents": { "types": "./index.d.ts", "import": "./src/agents.js" }
34
+ },
35
+ "types": "./index.d.ts",
36
+ "files": ["src", "index.d.ts", "README.md", "LICENSE"],
37
+ "engines": { "node": ">=20.11" },
38
+ "sideEffects": false,
39
+ "scripts": {
40
+ "test": "node --test"
41
+ }
42
+ }
package/src/agents.js ADDED
@@ -0,0 +1,59 @@
1
+ /**
2
+ * Which crawlers pay, and which read free.
3
+ *
4
+ * Two kinds of AI crawler visit a site and only one of them ever sends a reader
5
+ * back. RETRIEVAL crawlers feed the live index that ChatGPT search, Perplexity,
6
+ * Bing and Siri cite from; they are welcome everywhere a reader may go.
7
+ * TRAINING crawlers copy pages into a corpus that is baked into weights months
8
+ * later with no link back. Those are the ones robots.txt refuses and the
9
+ * gateway charges.
10
+ *
11
+ * Each entry is the token its operator documents, and the pairs are the point:
12
+ * GPTBot vs OAI-SearchBot, ClaudeBot vs Claude-SearchBot, meta-externalagent vs
13
+ * Meta-ExternalFetcher, Applebot-Extended vs Applebot. Refusing the wrong half
14
+ * of a pair cuts off citations while the corpus crawl carries on.
15
+ */
16
+
17
+ /** Training-only crawlers: refused in robots.txt, charged by the gateway. */
18
+ export const TRAINING_AGENTS = [
19
+ 'GPTBot',
20
+ 'ClaudeBot',
21
+ 'anthropic-ai',
22
+ 'CCBot',
23
+ 'meta-externalagent',
24
+ 'FacebookBot',
25
+ 'Bytespider',
26
+ 'Applebot-Extended',
27
+ ];
28
+
29
+ /**
30
+ * Retrieval crawlers, named in robots.txt so their operators can see they are
31
+ * welcome. Google-Extended is Google's training token but Google documents it
32
+ * as also gating grounding in the Gemini app, so it stays on this side.
33
+ */
34
+ export const RETRIEVAL_AGENTS = [
35
+ 'OAI-SearchBot',
36
+ 'ChatGPT-User',
37
+ 'Claude-SearchBot',
38
+ 'Claude-User',
39
+ 'PerplexityBot',
40
+ 'Perplexity-User',
41
+ 'Google-Extended',
42
+ 'Bingbot',
43
+ ];
44
+
45
+ const lower = (list) => list.map((t) => t.toLowerCase());
46
+
47
+ /**
48
+ * Whether a user agent is one of `agents` (default: the training list).
49
+ *
50
+ * Substring match on the documented token, which a determined caller can lie
51
+ * about. That is fine and worth being clear about: this identifies a
52
+ * self-declared corpus crawler so it can be charged. It does not stop anything
53
+ * hostile, and a crawler wearing a browser's user agent walks straight past it.
54
+ */
55
+ export function isTrainingAgent(userAgent = '', agents = TRAINING_AGENTS) {
56
+ const ua = String(userAgent ?? '').toLowerCase();
57
+ if (!ua) return false;
58
+ return lower(agents).some((t) => ua.includes(t));
59
+ }
package/src/hono.js ADDED
@@ -0,0 +1,24 @@
1
+ import { createGateway } from './index.js';
2
+
3
+ /**
4
+ * The gateway as Hono middleware.
5
+ *
6
+ * import { x402Gateway } from '@profullstack/x402-gateway/hono';
7
+ * app.use('*', x402Gateway({ siteUrl, coinpay: { apiKey }, payTo }));
8
+ *
9
+ * Register it before the routes and after any outright user-agent block: a
10
+ * crawler that is refused everywhere should not be sold anything. Takes either
11
+ * options or a gateway already created with `createGateway`, so an app can
12
+ * keep the gateway around for `robotsTxt()` and `page()`.
13
+ */
14
+ export function x402Gateway(gatewayOrOptions) {
15
+ const gateway =
16
+ gatewayOrOptions && typeof gatewayOrOptions.handle === 'function'
17
+ ? gatewayOrOptions
18
+ : createGateway(gatewayOrOptions);
19
+ return async (c, next) => {
20
+ const answer = await gateway.handle(c.req.raw);
21
+ if (answer) return answer;
22
+ await next();
23
+ };
24
+ }
package/src/index.js ADDED
@@ -0,0 +1,262 @@
1
+ import { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
2
+ import { renderPage } from './page.js';
3
+ import { mintPass, readPass } from './pass.js';
4
+ import { robotsTxt } from './robots.js';
5
+ import {
6
+ buildOffer,
7
+ decodePayment,
8
+ expectedFor,
9
+ METHODS,
10
+ nonceOf,
11
+ settleAgain,
12
+ validBeforeOf,
13
+ verifyAndSettle,
14
+ } from './x402.js';
15
+
16
+ export { isTrainingAgent, RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
17
+ export { renderPage } from './page.js';
18
+ export { mintPass, readPass } from './pass.js';
19
+ export { robotsTxt } from './robots.js';
20
+ export { buildOffer, decodePayment, expectedFor, METHODS, verifyAndSettle } from './x402.js';
21
+
22
+ /**
23
+ * A gateway that sells crawl access to training crawlers, by the day, over x402.
24
+ *
25
+ * People and search crawlers pass through untouched. A crawler on the training
26
+ * list is answered with 402 Payment Required carrying an x402 offer -- as JSON,
27
+ * or as an HTML sales page if it asked for HTML -- on every path but the few it
28
+ * needs to read to comply. Paying the offer, at the sales page or on any 402'd
29
+ * URL, returns a signed pass good for `passMinutes`, presented in `header` on
30
+ * every request after that.
31
+ *
32
+ * Framework-agnostic: `handle(request)` takes a Fetch `Request` and resolves to
33
+ * a `Response` to send, or null to let the request through. The adapters in
34
+ * ./hono and ./next are one line each on top of it.
35
+ *
36
+ * @param {object} options
37
+ * @param {string} options.siteUrl canonical origin, no trailing slash
38
+ * @param {string} [options.siteName] for the page; defaults to the hostname
39
+ * @param {{apiKey: string, baseUrl?: string}} [options.coinpay] a SCOPED CoinPay key (cp_live_…) with payments:create
40
+ * @param {string} [options.payTo] EVM address that receives the USDC
41
+ * @param {number} [options.priceCents=100]
42
+ * @param {string} [options.currency='USD']
43
+ * @param {number} [options.passMinutes=1440] a day
44
+ * @param {string} [options.header='x-crawl-pass']
45
+ * @param {string} [options.path='/crawl'] the sales page
46
+ * @param {string[]} [options.openPaths] extra paths a refused crawler may read
47
+ * @param {(ua: string) => boolean} [options.isPaidAgent]
48
+ * @param {string} [options.secret] pass signing secret; defaults to the CoinPay key
49
+ * @param {(ctx: object) => string} [options.page] custom sales page renderer
50
+ * @param {string} [options.contact] mailto: or URL for bulk deals
51
+ * @param {(sale: object) => void|Promise<void>} [options.onSale] accounting hook, never awaited for the answer
52
+ * @param {typeof fetch} [options.fetch] for tests
53
+ */
54
+ export function createGateway(options = {}) {
55
+ const o = normalise(options);
56
+ const enabled = Boolean(o.coinpay.apiKey && o.payTo);
57
+ const secret = o.secret || o.coinpay.apiKey || null;
58
+
59
+ const openPaths = ['/robots.txt', o.path, '/security.txt', '/.well-known/', ...o.openPaths];
60
+ const isOpen = (path) => openPaths.some((p) => (p.endsWith('/') ? path.startsWith(p) : path === p));
61
+
62
+ const price = `${(o.priceCents / 100).toFixed(2)} ${o.currency}`;
63
+ const buyUrl = `${o.siteUrl}${o.path}`;
64
+
65
+ const offer = () =>
66
+ enabled
67
+ ? buildOffer({
68
+ payTo: o.payTo,
69
+ priceCents: o.priceCents,
70
+ resource: buyUrl,
71
+ description: `${o.passMinutes} minutes of crawl access to ${o.siteUrl}`,
72
+ })
73
+ : { x402Version: 2, accepts: [] };
74
+
75
+ const receipt = (extra = {}) => ({
76
+ ...offer(),
77
+ pass: { price, minutes: o.passMinutes, header: o.header, buy: buyUrl },
78
+ ...extra,
79
+ });
80
+
81
+ const noStore = {
82
+ 'cache-control': 'no-store',
83
+ vary: 'Accept, User-Agent, X-Payment',
84
+ };
85
+ const json = (body, status, headers = {}) =>
86
+ new Response(JSON.stringify(body, null, 2), {
87
+ status,
88
+ headers: { 'content-type': 'application/json; charset=utf-8', ...noStore, ...headers },
89
+ });
90
+ const html = (body, status) =>
91
+ new Response(body, { status, headers: { 'content-type': 'text/html; charset=utf-8', ...noStore } });
92
+
93
+ const pageCtx = () => ({
94
+ siteName: o.siteName,
95
+ siteUrl: o.siteUrl,
96
+ buyUrl,
97
+ price,
98
+ minutes: o.passMinutes,
99
+ header: o.header,
100
+ enabled,
101
+ offer: offer(),
102
+ training: o.training,
103
+ retrieval: o.retrieval,
104
+ contact: o.contact,
105
+ });
106
+
107
+ /** The pass a request presents, from the named header or a bearer token. */
108
+ const passFrom = (request) => {
109
+ const direct = request.headers.get(o.header);
110
+ if (direct) return direct.trim();
111
+ const m = /^Bearer\s+(cp_[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$/i.exec(request.headers.get('authorization') ?? '');
112
+ return m ? m[1] : null;
113
+ };
114
+
115
+ /**
116
+ * Answer one request with the sale.
117
+ *
118
+ * The pass comes back as the BODY of a 200, not as the page that was asked
119
+ * for: the buyer is a program reading stdout, `coinpay x402 pay` prints the
120
+ * body and not the headers, and a crawler that wanted the page can fetch it
121
+ * again a moment later with the pass.
122
+ */
123
+ async function sell(request) {
124
+ const ua = request.headers.get('user-agent') ?? '';
125
+ const proofHeader = request.headers.get('x-payment');
126
+
127
+ if (proofHeader) {
128
+ if (!enabled) return json(receipt({ error: 'Payments are not switched on here.' }), 402);
129
+ const payment = decodePayment(proofHeader);
130
+ if (!payment) return json(receipt({ error: 'X-PAYMENT is not base64 JSON.' }), 402);
131
+ const current = offer();
132
+ const expected = expectedFor(payment, current);
133
+ if (!expected) return json(receipt({ error: 'Proof does not match an offered network.' }), 402);
134
+
135
+ const now = Math.floor(Date.now() / 1000);
136
+ const coinpay = { apiKey: o.coinpay.apiKey, baseUrl: o.coinpay.baseUrl, fetch: o.fetch };
137
+ const result = await verifyAndSettle(payment, expected, coinpay);
138
+
139
+ let expiresAt = null;
140
+ let replayed = false;
141
+ if (result.ok) {
142
+ expiresAt = now + o.passMinutes * 60;
143
+ } else if (result.replay) {
144
+ /*
145
+ * Paid once, lost the answer, asked again with the same proof. Answered
146
+ * with a pass -- but only if the money really moved, and only within
147
+ * the proof's own validity window plus one term, so the same header
148
+ * cannot be replayed for day after day. validBefore is set by the
149
+ * payer at signing time, typically ten minutes out; nothing else about
150
+ * "when was this bought" survives without a table.
151
+ */
152
+ const paid = await settleAgain(payment, coinpay);
153
+ const validBefore = validBeforeOf(payment);
154
+ if (paid && validBefore) {
155
+ expiresAt = Math.min(now + o.passMinutes * 60, validBefore + o.passMinutes * 60);
156
+ replayed = true;
157
+ }
158
+ }
159
+ if (!expiresAt || expiresAt <= now) {
160
+ return json(receipt({ error: result.reason ?? 'Payment could not be settled.' }), 402);
161
+ }
162
+
163
+ const ref = nonceOf(payment) ?? result.ref ?? null;
164
+ const pass = await mintPass({ secret, ref, expiresAt, now });
165
+ const expires = new Date(pass.expiresAt * 1000).toISOString();
166
+ if (o.onSale && !replayed) {
167
+ try {
168
+ await o.onSale({
169
+ payer: result.payer ?? null,
170
+ ref,
171
+ token: pass.token,
172
+ expiresAt: expires,
173
+ userAgent: ua,
174
+ priceCents: o.priceCents,
175
+ currency: o.currency,
176
+ });
177
+ } catch {
178
+ // Accounting must never cost a buyer the pass it paid for.
179
+ }
180
+ }
181
+ return json(
182
+ {
183
+ ok: true,
184
+ pass: pass.token,
185
+ expires_at: expires,
186
+ header: o.header,
187
+ replayed,
188
+ use: `curl -H "${o.header}: ${pass.token}" ${o.siteUrl}/`,
189
+ },
190
+ 200,
191
+ { [o.header]: pass.token, [`${o.header}-expires`]: expires },
192
+ );
193
+ }
194
+
195
+ if (wantsHtml(request.headers.get('accept'))) return html(o.page(pageCtx()), 402);
196
+ return json(receipt({ error: `Payment required for training crawlers. Read ${buyUrl} for how.` }), 402);
197
+ }
198
+
199
+ /**
200
+ * The gate. Null means "not for me, carry on".
201
+ *
202
+ * The sales page is answered for EVERYONE, so an operator can read it and a
203
+ * client can pay at it whatever user agent it wears. Every other decision
204
+ * starts from the user agent.
205
+ */
206
+ async function handle(request) {
207
+ const path = new URL(request.url).pathname;
208
+ if (path === o.path) return sell(request);
209
+ if (!o.isPaidAgent(request.headers.get('user-agent') ?? '')) return null;
210
+ if (isOpen(path)) return null;
211
+
212
+ const token = passFrom(request);
213
+ if (token && (await readPass(token, { secret }))) return null;
214
+ return sell(request);
215
+ }
216
+
217
+ return {
218
+ handle,
219
+ sell,
220
+ enabled,
221
+ options: o,
222
+ /** robots.txt with this gateway's lists and sales path. */
223
+ robotsTxt: (extra = {}) =>
224
+ robotsTxt({ siteUrl: o.siteUrl, path: o.path, training: o.training, retrieval: o.retrieval, ...extra }),
225
+ /** The sales page as HTML, for a site that mounts it on a route of its own. */
226
+ page: () => o.page(pageCtx()),
227
+ };
228
+ }
229
+
230
+ /** Whether the caller would rather read a page than a JSON offer. */
231
+ export const wantsHtml = (accept = '') => String(accept ?? '').toLowerCase().includes('text/html');
232
+
233
+ function normalise(options) {
234
+ const siteUrl = String(options.siteUrl ?? '').replace(/\/+$/, '');
235
+ if (!siteUrl) throw new Error('createGateway needs siteUrl');
236
+ const training = options.training ?? TRAINING_AGENTS;
237
+ return {
238
+ siteUrl,
239
+ siteName: options.siteName || new URL(siteUrl).hostname,
240
+ coinpay: {
241
+ apiKey: options.coinpay?.apiKey ?? '',
242
+ baseUrl: (options.coinpay?.baseUrl ?? 'https://coinpayportal.com').replace(/\/+$/, ''),
243
+ },
244
+ payTo: options.payTo ?? '',
245
+ priceCents: Number.isFinite(options.priceCents) ? options.priceCents : 100,
246
+ currency: options.currency ?? 'USD',
247
+ passMinutes: Number.isFinite(options.passMinutes) && options.passMinutes > 0 ? options.passMinutes : 1440,
248
+ header: String(options.header ?? 'x-crawl-pass').toLowerCase(),
249
+ path: options.path ?? '/crawl',
250
+ openPaths: options.openPaths ?? [],
251
+ training,
252
+ retrieval: options.retrieval ?? RETRIEVAL_AGENTS,
253
+ isPaidAgent: options.isPaidAgent ?? ((ua) => isTrainingAgent(ua, training)),
254
+ secret: options.secret ?? '',
255
+ page: options.page ?? renderPage,
256
+ contact: options.contact ?? '',
257
+ onSale: options.onSale ?? null,
258
+ fetch: options.fetch ?? globalThis.fetch,
259
+ };
260
+ }
261
+
262
+ export { METHODS as X402_METHODS };
package/src/next.js ADDED
@@ -0,0 +1,36 @@
1
+ import { createGateway } from './index.js';
2
+
3
+ /**
4
+ * The gateway for Next.js, in `proxy.ts` (Next 16) or `middleware.ts` (earlier).
5
+ *
6
+ * import { x402Proxy } from '@profullstack/x402-gateway/next';
7
+ * export const proxy = x402Proxy({ siteUrl, coinpay: { apiKey }, payTo });
8
+ * export const config = { matcher: ['/((?!_next/static|_next/image|favicon.ico).*)'] };
9
+ *
10
+ * Returning `undefined` from a Next proxy means "carry on", which is what the
11
+ * gateway's null becomes. When the app already has a proxy for something
12
+ * else, compose: `const answer = await gate(request); if (answer) return answer;`
13
+ * and then do whatever it did before. Runs at the edge: nothing here imports
14
+ * node:, and the pass is verified with Web Crypto rather than a database.
15
+ */
16
+ export function x402Proxy(gatewayOrOptions) {
17
+ const gateway =
18
+ gatewayOrOptions && typeof gatewayOrOptions.handle === 'function'
19
+ ? gatewayOrOptions
20
+ : createGateway(gatewayOrOptions);
21
+ return async (request) => (await gateway.handle(request)) ?? undefined;
22
+ }
23
+
24
+ /**
25
+ * A Route Handler for `app/robots.txt/route.js`, so robots.txt and the gateway
26
+ * are generated from one set of lists.
27
+ *
28
+ * import { robotsRoute } from '@profullstack/x402-gateway/next';
29
+ * export const GET = robotsRoute(gateway, { disallow: ['/login', '/api/'] });
30
+ */
31
+ export function robotsRoute(gateway, extra = {}) {
32
+ return () =>
33
+ new Response(gateway.robotsTxt(extra), {
34
+ headers: { 'content-type': 'text/plain; charset=utf-8' },
35
+ });
36
+ }
package/src/page.js ADDED
@@ -0,0 +1,133 @@
1
+ /**
2
+ * The sales page: what a refused crawler is shown, and what its operator reads.
3
+ *
4
+ * Plain HTML with no script, because the reader is a program that renders
5
+ * nothing, or a person who was sent the link by one. Every number on it comes
6
+ * from the gateway's options; nothing is typed in twice.
7
+ */
8
+
9
+ const esc = (s) =>
10
+ String(s ?? '')
11
+ .replace(/&/g, '&amp;')
12
+ .replace(/</g, '&lt;')
13
+ .replace(/>/g, '&gt;')
14
+ .replace(/"/g, '&quot;');
15
+
16
+ const CSS = `
17
+ :root{color-scheme:light dark;--fg:#1a1a1a;--bg:#fff;--mut:#666;--line:#e5e5e5;--code:#f4f4f4;--acc:#0a5}
18
+ @media(prefers-color-scheme:dark){:root{--fg:#eee;--bg:#111;--mut:#aaa;--line:#333;--code:#1c1c1c;--acc:#3c9}}
19
+ body{margin:0;background:var(--bg);color:var(--fg);font:16px/1.55 system-ui,-apple-system,Segoe UI,Roboto,sans-serif}
20
+ main{max-width:44rem;margin:0 auto;padding:2.5rem 1.25rem 4rem}
21
+ h1{font-size:1.8rem;line-height:1.2;margin:0 0 .5rem}h2{font-size:1.15rem;margin:2rem 0 .5rem}
22
+ p{margin:.5rem 0}.mut{color:var(--mut)}
23
+ pre{background:var(--code);border:1px solid var(--line);border-radius:6px;padding:.9rem 1rem;overflow-x:auto;font-size:.88rem;line-height:1.45}
24
+ code{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.92em}
25
+ ol,ul{padding-left:1.3rem}li{margin:.3rem 0}
26
+ .price{font-size:2.2rem;font-weight:700;color:var(--acc);margin:.25rem 0}
27
+ table{border-collapse:collapse;margin:.5rem 0;font-size:.95rem}td,th{border-bottom:1px solid var(--line);padding:.35rem .6rem;text-align:left}
28
+ footer{margin-top:3rem;color:var(--mut);font-size:.85rem}
29
+ `;
30
+
31
+ /**
32
+ * @param {object} ctx
33
+ * @param {string} ctx.siteName
34
+ * @param {string} ctx.siteUrl
35
+ * @param {string} ctx.buyUrl the URL to pay at (the sales page itself)
36
+ * @param {string} ctx.price e.g. "1.00 USD"
37
+ * @param {number} ctx.minutes
38
+ * @param {string} ctx.header request header the pass goes in
39
+ * @param {boolean} ctx.enabled whether a payment can be taken right now
40
+ * @param {object} ctx.offer the x402 body, for the curious
41
+ * @param {string[]} ctx.training agents charged
42
+ * @param {string[]} ctx.retrieval agents welcome free
43
+ * @param {string} [ctx.contact] mailto or URL for bulk deals
44
+ */
45
+ export function renderPage(ctx) {
46
+ const {
47
+ siteName,
48
+ siteUrl,
49
+ buyUrl,
50
+ price,
51
+ minutes,
52
+ header,
53
+ enabled,
54
+ offer,
55
+ training = [],
56
+ retrieval = [],
57
+ contact,
58
+ } = ctx;
59
+ const window =
60
+ minutes === 1440
61
+ ? 'one day'
62
+ : minutes % 1440 === 0
63
+ ? `${minutes / 1440} days`
64
+ : minutes === 60
65
+ ? 'one hour'
66
+ : minutes % 60 === 0
67
+ ? `${minutes / 60} hours`
68
+ : `${minutes} minutes`;
69
+ const networks = (offer?.accepts ?? []).map((a) => a.network).join(', ');
70
+
71
+ return `<!doctype html>
72
+ <html lang="en">
73
+ <head>
74
+ <meta charset="utf-8">
75
+ <meta name="viewport" content="width=device-width,initial-scale=1">
76
+ <meta name="robots" content="noindex">
77
+ <title>Crawl access · ${esc(siteName)}</title>
78
+ <style>${CSS}</style>
79
+ </head>
80
+ <body>
81
+ <main>
82
+ <h1>Training crawlers pay for access here.</h1>
83
+ <p class="mut">People read <a href="${esc(siteUrl)}">${esc(siteName)}</a> free. So do search engines and the retrieval crawlers behind AI answers, because they send readers back. A crawler that copies pages into a training corpus sends nobody back, so it pays for the time it spends.</p>
84
+
85
+ <div class="price">${esc(price)} <span class="mut" style="font-size:1rem;font-weight:400">for ${esc(window)} of requests</span></div>
86
+ ${
87
+ enabled
88
+ ? ''
89
+ : '<p><strong>Payments are not switched on here yet.</strong> The offer below is empty until the operator configures a payout address, so for now this crawler is simply refused.</p>'
90
+ }
91
+
92
+ <h2>How it works</h2>
93
+ <ol>
94
+ <li>Any page you fetch answers <code>402 Payment Required</code>. This page, fetched with <code>Accept: application/json</code>, returns the x402 offer: USDC, <code>exact</code> scheme, on ${esc(networks || 'Base, Polygon or Ethereum')}.</li>
95
+ <li>Sign the payment and retry with the proof in an <code>X-PAYMENT</code> header. The response is a JSON receipt carrying a pass.</li>
96
+ <li>Send the pass in <code>${esc(header)}</code> on every request for the next ${esc(window)}. When it expires, buy another. The sale is the pass, not the page: fetch the page again with the pass.</li>
97
+ </ol>
98
+
99
+ <h2>Pay with the CoinPay CLI</h2>
100
+ <p>Settlement is by CoinPay: the buyer's USDC goes straight to the site's wallet and CoinPay's relayer pays the gas, so you need USDC and nothing else.</p>
101
+ <pre><code>npm install -g @profullstack/coinpay
102
+ coinpay x402 pay ${esc(buyUrl)} --output pass.json</code></pre>
103
+ <p>The command fetches this page, reads the offer, opens a browser tab to approve the payment with the CoinPay Wallet extension or any EIP-6963 wallet (MetaMask, Rabby, Coinbase Wallet), and writes the receipt to <code>pass.json</code>. Then:</p>
104
+ <pre><code>PASS=$(node -p "require('./pass.json').pass")
105
+ curl -H "${esc(header)}: $PASS" ${esc(siteUrl)}/</code></pre>
106
+
107
+ <h2>Pay from your own x402 client</h2>
108
+ <pre><code>curl -sS -H "Accept: application/json" ${esc(buyUrl)}
109
+ # 402 with { "x402Version": 2, "accepts": [ ... ] }
110
+ # sign an EIP-3009 transferWithAuthorization for one entry, then:
111
+ curl -sS -H "X-PAYMENT: &lt;base64 proof&gt;" ${esc(buyUrl)}
112
+ # 200 with { "ok": true, "pass": "cp_...", "expires_at": "...", "header": "${esc(header)}" }</code></pre>
113
+ <p class="mut">The proof is x402 v2 in CoinPay's dialect: <code>{ x402Version: 2, scheme: "exact", network: "&lt;CAIP-2&gt;", payload: { signature, authorization } }</code>, base64-encoded. A proof is single-use; retrying with the same one returns the same pass, not a second charge.</p>
114
+
115
+ <h2>Who pays and who does not</h2>
116
+ <table>
117
+ <tr><th>Charged</th><td>${training.map(esc).join(', ')}</td></tr>
118
+ <tr><th>Free, named in robots.txt</th><td>${retrieval.map(esc).join(', ')}</td></tr>
119
+ <tr><th>Free</th><td>Everyone else: people, Googlebot, Applebot, Bingbot and any crawler not on the first line.</td></tr>
120
+ </table>
121
+ <p class="mut">If your crawler is on the first line and you believe it should not be, or you want more than an hour at a time${
122
+ contact ? `, <a href="${esc(contact)}">get in touch</a>` : ', contact the site'
123
+ }.</p>
124
+
125
+ <h2>The offer, verbatim</h2>
126
+ <pre><code>${esc(JSON.stringify(offer, null, 2))}</code></pre>
127
+
128
+ <footer>Served by @profullstack/x402-gateway. This page is <code>noindex</code> and is the one URL a refused crawler may read.</footer>
129
+ </main>
130
+ </body>
131
+ </html>
132
+ `;
133
+ }
package/src/pass.js ADDED
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Passes: what a payment buys.
3
+ *
4
+ * A pass is a signed, self-describing token -- `cp_<payload>.<signature>` --
5
+ * carrying its own expiry and the reference of the payment that bought it.
6
+ * There is no table behind it. The gateway runs in front of six sites on two
7
+ * frameworks, some of them at the edge with no database in reach, and a token
8
+ * that proves itself needs none of them to agree on a schema.
9
+ *
10
+ * HMAC-SHA256 over Web Crypto, which is what Node, Bun and every edge runtime
11
+ * have in common. The key defaults to the CoinPay API key: it is already the
12
+ * one secret every deployment holds, and using it as key material never sends
13
+ * it anywhere.
14
+ */
15
+
16
+ const enc = new TextEncoder();
17
+
18
+ function toBase64Url(bytes) {
19
+ let bin = '';
20
+ for (const b of bytes) bin += String.fromCharCode(b);
21
+ return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
22
+ }
23
+
24
+ function fromBase64Url(s) {
25
+ const bin = atob(String(s).replace(/-/g, '+').replace(/_/g, '/'));
26
+ return Uint8Array.from(bin, (ch) => ch.charCodeAt(0));
27
+ }
28
+
29
+ async function hmacKey(secret) {
30
+ return crypto.subtle.importKey('raw', enc.encode(String(secret)), { name: 'HMAC', hash: 'SHA-256' }, false, [
31
+ 'sign',
32
+ 'verify',
33
+ ]);
34
+ }
35
+
36
+ async function sign(secret, data) {
37
+ const key = await hmacKey(secret);
38
+ const sig = await crypto.subtle.sign('HMAC', key, enc.encode(data));
39
+ return toBase64Url(new Uint8Array(sig));
40
+ }
41
+
42
+ /** Constant-time compare of two short strings. */
43
+ function same(a, b) {
44
+ if (a.length !== b.length) return false;
45
+ let diff = 0;
46
+ for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
47
+ return diff === 0;
48
+ }
49
+
50
+ /**
51
+ * Mint a pass.
52
+ *
53
+ * @param {object} args
54
+ * @param {string} args.secret
55
+ * @param {string|null} args.ref payment reference, for the accounting question
56
+ * @param {number} args.expiresAt unix seconds
57
+ * @param {number} [args.now] unix seconds, for tests
58
+ */
59
+ export async function mintPass({ secret, ref, expiresAt, now = Math.floor(Date.now() / 1000) }) {
60
+ if (!secret) throw new Error('a pass needs a signing secret');
61
+ if (!Number.isFinite(expiresAt) || expiresAt <= now) throw new Error('a pass needs a future expiry');
62
+ const payload = toBase64Url(enc.encode(JSON.stringify({ v: 1, iat: now, exp: Math.floor(expiresAt), ref: ref ?? null })));
63
+ const sig = await sign(secret, payload);
64
+ return { token: `cp_${payload}.${sig}`, expiresAt: Math.floor(expiresAt), ref: ref ?? null };
65
+ }
66
+
67
+ /**
68
+ * Read a pass. Resolves to `{ exp, iat, ref }` when the signature holds and the
69
+ * pass has not expired, null otherwise. Never throws on garbage input: a token
70
+ * is untrusted text from a crawler.
71
+ */
72
+ export async function readPass(token, { secret, now = Math.floor(Date.now() / 1000) }) {
73
+ if (!secret || typeof token !== 'string' || !token.startsWith('cp_')) return null;
74
+ const dot = token.indexOf('.');
75
+ if (dot < 0) return null;
76
+ const payload = token.slice(3, dot);
77
+ const sig = token.slice(dot + 1);
78
+ if (!payload || !sig) return null;
79
+ try {
80
+ const expect = await sign(secret, payload);
81
+ if (!same(expect, sig)) return null;
82
+ const claims = JSON.parse(new TextDecoder().decode(fromBase64Url(payload)));
83
+ if (!claims || claims.v !== 1 || !Number.isFinite(claims.exp)) return null;
84
+ if (claims.exp <= now) return null;
85
+ return { exp: claims.exp, iat: claims.iat ?? null, ref: claims.ref ?? null };
86
+ } catch {
87
+ return null;
88
+ }
89
+ }
package/src/robots.js ADDED
@@ -0,0 +1,64 @@
1
+ import { RETRIEVAL_AGENTS, TRAINING_AGENTS } from './agents.js';
2
+
3
+ /**
4
+ * robots.txt with the crawlers sorted the way the gateway sorts them.
5
+ *
6
+ * Training crawlers are refused everywhere except the page that sells them a
7
+ * way in; retrieval crawlers are named so their operators can see they are
8
+ * welcome; everything else gets the wildcard rules.
9
+ *
10
+ * The trap this avoids: a crawler that finds a group matching its own name
11
+ * obeys THAT group and ignores `User-agent: *` entirely. Naming one and listing
12
+ * the sign-in page only under the wildcard would invite it straight into the
13
+ * sign-in page. So every named group repeats the rules, generated rather than
14
+ * typed.
15
+ *
16
+ * @param {object} options
17
+ * @param {string} options.siteUrl no trailing slash
18
+ * @param {string[]} [options.disallow] paths nobody should index, e.g. ['/login', '/api/']
19
+ * @param {string[]} [options.allow] exceptions that beat a disallow by being longer, e.g. ['/api/v1']
20
+ * @param {string} [options.sitemap] defaults to `${siteUrl}/sitemap.xml`; '' to omit
21
+ * @param {string} [options.path] the sales page training crawlers may still read; default '/crawl'
22
+ * @param {string[]} [options.refused] extra agents refused outright, e.g. a rude SEO bot
23
+ * @param {string[]} [options.training] override the training list
24
+ * @param {string[]} [options.retrieval] override the retrieval list
25
+ * @param {string[]} [options.comments] lines written at the top, without the leading '# '
26
+ */
27
+ export function robotsTxt({
28
+ siteUrl,
29
+ disallow = [],
30
+ allow = [],
31
+ sitemap,
32
+ path = '/crawl',
33
+ refused = [],
34
+ training = TRAINING_AGENTS,
35
+ retrieval = RETRIEVAL_AGENTS,
36
+ comments = [],
37
+ } = {}) {
38
+ if (!siteUrl) throw new Error('robotsTxt needs siteUrl');
39
+ const base = siteUrl.replace(/\/+$/, '');
40
+ const map = sitemap === undefined ? `${base}/sitemap.xml` : sitemap;
41
+
42
+ const welcome = (agent) =>
43
+ [
44
+ `User-agent: ${agent}`,
45
+ 'Allow: /',
46
+ ...allow.map((p) => `Allow: ${p}`),
47
+ ...disallow.map((p) => `Disallow: ${p}`),
48
+ ].join('\n');
49
+ const refuse = (agent) => `User-agent: ${agent}\nDisallow: /`;
50
+ // Longest match wins, so `Allow: /crawl` beats `Disallow: /` for that one page.
51
+ const charge = (agent) => `${refuse(agent)}\nAllow: ${path}`;
52
+
53
+ const lines = [
54
+ ...comments.map((c) => `# ${c}`),
55
+ ...(comments.length ? [''] : []),
56
+ ...refused.map((a) => `${refuse(a)}\n`),
57
+ ...training.map((a) => `${charge(a)}\n`),
58
+ ...retrieval.map((a) => `${welcome(a)}\n`),
59
+ welcome('*'),
60
+ '',
61
+ ];
62
+ if (map) lines.push(`Sitemap: ${map}`, '');
63
+ return lines.join('\n');
64
+ }
package/src/x402.js ADDED
@@ -0,0 +1,178 @@
1
+ /**
2
+ * x402 v2, in CoinPayPortal's dialect.
3
+ *
4
+ * Checked against @profullstack/coinpay/x402-v2 rather than remembered: the 402
5
+ * body is JSON `{ x402Version: 2, accepts: [...] }`, the proof arrives as base64
6
+ * JSON in an `X-PAYMENT` request header, and CoinPay's /api/x402/verify then
7
+ * /api/x402/settle do the cryptography and the on-chain transfer (EIP-3009,
8
+ * with CoinPay's relayer paying the gas, so the buyer needs no ETH).
9
+ *
10
+ * Reimplemented here in a few dozen lines because the SDK also carries the
11
+ * CLI's interactive-prompt dependency, which a web server has no business
12
+ * installing, and because a gateway that runs at the edge cannot import node:.
13
+ */
14
+
15
+ /**
16
+ * The three things CoinPay can settle under the `exact` scheme. USDC only:
17
+ * EIP-3009 is an ERC-20 extension, so native ETH cannot be paid this way.
18
+ *
19
+ * CAIP-2 network ids, 6-decimal amounts, and the token's own EIP-712 domain in
20
+ * `extra`, which is not derivable from the contract and was read over JSON-RPC
21
+ * once upstream: all three USDC deployments answer "USD Coin" / "2".
22
+ *
23
+ * Base first. Merchant order is the payer's preference order, and Base is where
24
+ * the relayer's gas is cheapest.
25
+ */
26
+ export const METHODS = [
27
+ {
28
+ key: 'usdc_base',
29
+ network: 'eip155:8453',
30
+ asset: '0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913',
31
+ label: 'USDC on Base',
32
+ },
33
+ {
34
+ key: 'usdc_polygon',
35
+ network: 'eip155:137',
36
+ asset: '0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359',
37
+ label: 'USDC on Polygon',
38
+ },
39
+ {
40
+ key: 'usdc_eth',
41
+ network: 'eip155:1',
42
+ asset: '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48',
43
+ label: 'USDC on Ethereum',
44
+ },
45
+ ];
46
+
47
+ const DECIMALS = 6;
48
+ const DOMAIN = { name: 'USD Coin', version: '2' };
49
+
50
+ /**
51
+ * A v2 402 body.
52
+ *
53
+ * `amount` is the price in the token's smallest unit, rounded UP: a fraction of
54
+ * a cent rounded down would quote less than the asking price and then verify as
55
+ * underpayment.
56
+ */
57
+ export function buildOffer({
58
+ payTo,
59
+ priceCents,
60
+ resource,
61
+ description = 'Payment required',
62
+ maxTimeoutSeconds = 300,
63
+ methods = METHODS,
64
+ }) {
65
+ if (!payTo) throw new Error('an offer needs a payTo address');
66
+ const amount = String(Math.ceil((Number(priceCents) / 100) * 10 ** DECIMALS));
67
+ return {
68
+ x402Version: 2,
69
+ accepts: methods.map((m) => ({
70
+ scheme: 'exact',
71
+ network: m.network,
72
+ amount,
73
+ asset: m.asset,
74
+ payTo,
75
+ resource,
76
+ description,
77
+ mimeType: 'application/json',
78
+ maxTimeoutSeconds,
79
+ extra: { ...DOMAIN },
80
+ })),
81
+ };
82
+ }
83
+
84
+ /** base64 -> utf8, without Buffer, so it runs at the edge. */
85
+ function fromBase64(s) {
86
+ const bin = atob(String(s).trim().replace(/-/g, '+').replace(/_/g, '/'));
87
+ const bytes = Uint8Array.from(bin, (ch) => ch.charCodeAt(0));
88
+ return new TextDecoder().decode(bytes);
89
+ }
90
+
91
+ /** The proof out of an X-PAYMENT header. Null if it is not base64 JSON. */
92
+ export function decodePayment(header) {
93
+ if (!header) return null;
94
+ try {
95
+ const parsed = JSON.parse(fromBase64(header));
96
+ return parsed && typeof parsed === 'object' ? parsed : null;
97
+ } catch {
98
+ return null;
99
+ }
100
+ }
101
+
102
+ /**
103
+ * What CoinPay must hold the proof to.
104
+ *
105
+ * Its v2 verify refuses without all four of amount, resource, payTo and asset,
106
+ * and takes them from the OFFERED entry for the proof's network -- never from
107
+ * the proof itself, which is the payer's claim about what it paid.
108
+ */
109
+ export function expectedFor(payment, offer) {
110
+ const network = String(payment?.network ?? '').toLowerCase();
111
+ const entry = (offer?.accepts ?? []).find((a) => a.network.toLowerCase() === network);
112
+ if (!entry) return null;
113
+ return { amount: entry.amount, resource: entry.resource, payTo: entry.payTo, asset: entry.asset };
114
+ }
115
+
116
+ /** The single-use nonce, which is what CoinPay keys replay detection on. */
117
+ export const nonceOf = (payment) => payment?.payload?.authorization?.nonce ?? null;
118
+
119
+ /** When the payer's signature stops being valid, as a unix timestamp in seconds. */
120
+ export const validBeforeOf = (payment) => {
121
+ const v = Number(payment?.payload?.authorization?.validBefore);
122
+ return Number.isFinite(v) && v > 0 ? v : null;
123
+ };
124
+
125
+ /**
126
+ * Verify, then settle. Two calls because verify moves no money: it checks the
127
+ * signature and records the proof, and settle broadcasts the transfer.
128
+ *
129
+ * Resolves to `{ ok: true, payer, ref }` or `{ ok: false, reason, replay }`.
130
+ * `replay` is set when CoinPay has already seen this proof -- the case where a
131
+ * crawler paid, lost our answer, and is retrying with the same header. The
132
+ * gateway answers that with a pass rather than a second charge, bounded by the
133
+ * proof's own validity window so the same header cannot buy hour after hour.
134
+ */
135
+ export async function verifyAndSettle(payment, expected, { apiKey, baseUrl, fetch: f = globalThis.fetch }) {
136
+ const call = async (path, body) => {
137
+ const res = await f(`${baseUrl}${path}`, {
138
+ method: 'POST',
139
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey },
140
+ body: JSON.stringify(body),
141
+ signal: AbortSignal.timeout(20000),
142
+ });
143
+ const text = await res.text();
144
+ let json = {};
145
+ try {
146
+ json = JSON.parse(text);
147
+ } catch {}
148
+ return { status: res.status, json };
149
+ };
150
+
151
+ const v = await call('/api/x402/verify', { payment, expected });
152
+ if (!v.json?.valid) {
153
+ const reason = String(v.json?.error ?? v.json?.reason ?? `verify failed (${v.status})`);
154
+ return { ok: false, reason, replay: /already used|replay/i.test(reason) };
155
+ }
156
+ const s = await call('/api/x402/settle', { payment });
157
+ if (!s.json?.settled) {
158
+ const reason = String(s.json?.error ?? `settle failed (${s.status})`);
159
+ return { ok: false, reason, replay: /already settled|already being settled/i.test(reason) };
160
+ }
161
+ return { ok: true, payer: v.json.payment?.from ?? null, ref: s.json.txHash ?? nonceOf(payment) };
162
+ }
163
+
164
+ /** Whether the proof has already been paid, when a settle is asked about twice. */
165
+ export async function settleAgain(payment, { apiKey, baseUrl, fetch: f = globalThis.fetch }) {
166
+ const res = await f(`${baseUrl}/api/x402/settle`, {
167
+ method: 'POST',
168
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey },
169
+ body: JSON.stringify({ payment }),
170
+ signal: AbortSignal.timeout(20000),
171
+ });
172
+ let json = {};
173
+ try {
174
+ json = JSON.parse(await res.text());
175
+ } catch {}
176
+ const reason = String(json?.error ?? '');
177
+ return Boolean(json?.settled) || /already settled/i.test(reason);
178
+ }