@profullstack/throttle 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +206 -0
- package/index.d.ts +93 -0
- package/package.json +62 -0
- package/src/hono.js +22 -0
- package/src/identify.js +57 -0
- package/src/index.js +12 -0
- package/src/next.js +29 -0
- package/src/rules.js +82 -0
- package/src/store.js +13 -0
- package/src/throttle.js +213 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Profullstack, LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# @profullstack/throttle
|
|
2
|
+
|
|
3
|
+
An app-wide rate limit whose answer to going over is a price, not a 429.
|
|
4
|
+
|
|
5
|
+
**100 requests a minute, free. Then `402 Payment Required` with an x402 offer**,
|
|
6
|
+
settled by CoinPay in USDC. It meters *every* route, so a scraper wearing a
|
|
7
|
+
browser user agent is charged like any other crawler.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm i @profullstack/throttle
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Why
|
|
14
|
+
|
|
15
|
+
Every site in our fleet had a rate limiter in its middleware. Each one was
|
|
16
|
+
hand-rolled, and each one was scoped to `/api/`.
|
|
17
|
+
|
|
18
|
+
Then a headless browser in Singapore found a block explorer that had shipped
|
|
19
|
+
fifteen hours earlier, and spent two days walking it: **19,000 URLs a day**, a
|
|
20
|
+
fresh visitor id on every hit, a plain `Chrome/145` user agent, every page
|
|
21
|
+
rendered server-side against a metered upstream API.
|
|
22
|
+
|
|
23
|
+
Nothing stopped it.
|
|
24
|
+
|
|
25
|
+
- It was not on a crawler list, because it declared nothing.
|
|
26
|
+
- It was not rate limited, because it never touched `/api/`.
|
|
27
|
+
- The analytics counted every hit as a **human visit**.
|
|
28
|
+
|
|
29
|
+
The limiter we had was protecting the routes nobody was attacking. So: meter
|
|
30
|
+
everything, and when a caller goes over, sell it a pass.
|
|
31
|
+
|
|
32
|
+
429 is the right answer when you have nothing to sell. We do. The moment the
|
|
33
|
+
allowance runs out is the best sales pitch a site will ever get, because the
|
|
34
|
+
caller has just demonstrated it wants more than the free tier and is still
|
|
35
|
+
holding the request open.
|
|
36
|
+
|
|
37
|
+
## Use it
|
|
38
|
+
|
|
39
|
+
```js
|
|
40
|
+
// src/proxy.ts — Next 16 middleware
|
|
41
|
+
import { createGateway } from '@profullstack/x402-gateway';
|
|
42
|
+
import { createThrottle } from '@profullstack/throttle';
|
|
43
|
+
|
|
44
|
+
const gateway = createGateway({
|
|
45
|
+
siteUrl: 'https://example.com',
|
|
46
|
+
coinpay: { apiKey: process.env.COINPAY_X402_KEY },
|
|
47
|
+
payTo: process.env.CRAWL_PAY_TO,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const throttle = createThrottle({
|
|
51
|
+
gateway, // over the limit → 402 with an offer
|
|
52
|
+
limit: 100, // the default; per caller, per minute
|
|
53
|
+
exempt: (request) => hasSession(request),
|
|
54
|
+
rules: [
|
|
55
|
+
{ path: '/api/auth/', limit: 10, credential: false },
|
|
56
|
+
{ path: '/health', open: true },
|
|
57
|
+
],
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
export async function proxy(request) {
|
|
61
|
+
const answer = await throttle.handle(request);
|
|
62
|
+
if (answer) return answer;
|
|
63
|
+
return NextResponse.next();
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`handle` returns a `Response` to send, or `null` to carry on. Adapters for
|
|
68
|
+
[Next](#nextjs) and [Hono](#hono) are one line on top of it.
|
|
69
|
+
|
|
70
|
+
Without a `gateway` the same throttle answers `429` with `Retry-After`. That is
|
|
71
|
+
the fallback, not the goal.
|
|
72
|
+
|
|
73
|
+
## What a refused caller sees
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
HTTP/1.1 402 Payment Required
|
|
77
|
+
ratelimit-limit: 100
|
|
78
|
+
ratelimit-remaining: 0
|
|
79
|
+
ratelimit-reset: 41
|
|
80
|
+
```
|
|
81
|
+
```json
|
|
82
|
+
{
|
|
83
|
+
"x402Version": 2,
|
|
84
|
+
"accepts": [{ "scheme": "exact", "network": "base", "amount": "1000000", "…": "…" }],
|
|
85
|
+
"error": "Free allowance used: 100 requests per 60s. It resets in 41s. A pass removes the limit for 1.00 USD a day.",
|
|
86
|
+
"pass": { "price": "1.00 USD", "minutes": 1440, "buy": "https://example.com/crawl" }
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Ask for `text/html` and you get the sales page instead, which is what makes
|
|
91
|
+
this survivable for a person who tripped it by accident.
|
|
92
|
+
|
|
93
|
+
Pay it, and the pass comes back as the body of a `200`:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
coinpay x402 pay https://example.com/crawl
|
|
97
|
+
curl -H "x-crawl-pass: $PASS" https://example.com/whatever
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
A caller holding a live pass is **not metered at all**. The pass is the thing
|
|
101
|
+
being sold; counting it against an allowance it bought its way out of would be
|
|
102
|
+
selling the same minute twice.
|
|
103
|
+
|
|
104
|
+
## Options
|
|
105
|
+
|
|
106
|
+
| Option | Default | |
|
|
107
|
+
|---|---|---|
|
|
108
|
+
| `limit` | `100` | requests per window, per caller |
|
|
109
|
+
| `windowSeconds` | `60` | |
|
|
110
|
+
| `gateway` | — | an `@profullstack/x402-gateway`; makes over-limit a 402 |
|
|
111
|
+
| `rules` | `[]` | per-path allowances; most specific match wins |
|
|
112
|
+
| `credential` | `{ limit: 600, ceiling: 1200 }` | budget for callers presenting one |
|
|
113
|
+
| `exempt` | — | never metered, e.g. a signed-in session |
|
|
114
|
+
| `openPaths` | — | extra paths never metered |
|
|
115
|
+
| `identify` | the caller's address | what to count against |
|
|
116
|
+
| `store` | per-process | supply one to share a count across a fleet |
|
|
117
|
+
| `onThrottle` | — | told about every refusal; its errors are swallowed |
|
|
118
|
+
|
|
119
|
+
### Rules
|
|
120
|
+
|
|
121
|
+
A rule is a path pattern plus the numbers that replace the defaults for it.
|
|
122
|
+
Patterns ending in `/` match by prefix, everything else exactly. The most
|
|
123
|
+
specific match wins **whatever order they are written in** — the alternative is
|
|
124
|
+
a config where moving two lines silently removes the brute-force limit.
|
|
125
|
+
|
|
126
|
+
```js
|
|
127
|
+
rules: [
|
|
128
|
+
{ path: '/api/auth/', limit: 10, credential: false }, // sign-in
|
|
129
|
+
{ path: '/feed/', limit: 30 },
|
|
130
|
+
{ path: '/health', open: true }, // never metered
|
|
131
|
+
]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Credentialed callers
|
|
135
|
+
|
|
136
|
+
One host running an integration is not one browser, so a request presenting a
|
|
137
|
+
credential gets a larger budget. **Both** buckets are charged: the per-address
|
|
138
|
+
`ceiling` is what a caller rotating junk credentials cannot escape, since
|
|
139
|
+
nothing here has authenticated any of them.
|
|
140
|
+
|
|
141
|
+
Any auth scheme counts, not just `Bearer` — matching Bearer alone once dropped
|
|
142
|
+
a signed wallet's bulk payout into the anonymous bucket, which it exhausted in
|
|
143
|
+
seconds.
|
|
144
|
+
|
|
145
|
+
`credential: false` on a rule keeps it address-bucketed however the request is
|
|
146
|
+
authenticated. **The sign-in routes need this**, or a brute-force attempt bolts
|
|
147
|
+
an `Authorization` header onto every guess and buys itself the integration
|
|
148
|
+
budget.
|
|
149
|
+
|
|
150
|
+
### Never metered
|
|
151
|
+
|
|
152
|
+
`/robots.txt`, `/favicon.ico`, `/.well-known/`, and — when a gateway is
|
|
153
|
+
supplied — its sales page and open paths, read off the gateway rather than
|
|
154
|
+
restated. A paywall you cannot reach to pay is a wall.
|
|
155
|
+
|
|
156
|
+
A caller we cannot identify is also within the allowance. Our own edge failing
|
|
157
|
+
to hand us an address is our gap, and charging someone for it would be charging
|
|
158
|
+
them for our gap.
|
|
159
|
+
|
|
160
|
+
## Nextjs
|
|
161
|
+
|
|
162
|
+
```js
|
|
163
|
+
import { throttleProxy } from '@profullstack/throttle/next';
|
|
164
|
+
export const proxy = throttleProxy({ gateway });
|
|
165
|
+
export const config = { matcher: ['/((?!_next/static|_next/image|favicon.ico).*)'] };
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Put it **after** the crawl gateway and before everything else: the gateway
|
|
169
|
+
charges crawlers that declare themselves, the throttle charges the ones that do
|
|
170
|
+
not, and there is no point running CORS on a request about to be refused.
|
|
171
|
+
|
|
172
|
+
Runs at the edge — nothing imports `node:`, and the pass is verified with Web
|
|
173
|
+
Crypto rather than a database.
|
|
174
|
+
|
|
175
|
+
## Hono
|
|
176
|
+
|
|
177
|
+
```js
|
|
178
|
+
import { throttle } from '@profullstack/throttle/hono';
|
|
179
|
+
app.use('*', throttle({ gateway }));
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## On rotating addresses
|
|
183
|
+
|
|
184
|
+
Rotating addresses defeats an address bucket. That is not a hole to be plugged,
|
|
185
|
+
it is the argument the 402 makes: residential proxy bandwidth is sold by the
|
|
186
|
+
gigabyte at prices that pass a dollar inside the first day of any serious
|
|
187
|
+
crawl, and a rotation still fetches every page one at a time.
|
|
188
|
+
|
|
189
|
+
Anyone spending more on proxies than the pass costs, in order to avoid the
|
|
190
|
+
pass, has made an arithmetic mistake rather than beaten a detector. Detection
|
|
191
|
+
is a race. Price is not.
|
|
192
|
+
|
|
193
|
+
## Counting
|
|
194
|
+
|
|
195
|
+
The default counter lives in the process, so several instances each grant their
|
|
196
|
+
own allowance. That errs toward generosity, which is the right direction to be
|
|
197
|
+
wrong in for a free tier. Pass a `store` to share one count across a fleet — the
|
|
198
|
+
interface is a single method, so Redis or Postgres is a dozen lines:
|
|
199
|
+
|
|
200
|
+
```js
|
|
201
|
+
{ hit(key, windowSeconds) { return { count, resetSeconds } } }
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
## License
|
|
205
|
+
|
|
206
|
+
MIT © Profullstack, LLC
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Gateway, QuotaStore } from '@profullstack/x402-gateway';
|
|
2
|
+
|
|
3
|
+
/** The house default: a hundred requests a minute, per caller, everywhere. */
|
|
4
|
+
export const DEFAULT_LIMIT: 100;
|
|
5
|
+
export const DEFAULT_WINDOW_SECONDS: 60;
|
|
6
|
+
export const ALWAYS_OPEN: string[];
|
|
7
|
+
|
|
8
|
+
export interface ThrottleRule {
|
|
9
|
+
/** Prefix when it ends in `/`, exact match otherwise. */
|
|
10
|
+
path: string;
|
|
11
|
+
/** Requests per window on this rule. Inherits the default when absent. */
|
|
12
|
+
limit?: number;
|
|
13
|
+
windowSeconds?: number;
|
|
14
|
+
/** Never metered. */
|
|
15
|
+
open?: boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Credentials on this rule. `false` keeps it address-bucketed however the
|
|
18
|
+
* request is authenticated — what the sign-in routes need.
|
|
19
|
+
*/
|
|
20
|
+
credential?: false | { limit?: number; ceiling?: number };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface CredentialBudget {
|
|
24
|
+
/** Per credential. Default 600. */
|
|
25
|
+
limit?: number;
|
|
26
|
+
/** Per address, whatever credentials it rotates through. Default 1200. */
|
|
27
|
+
ceiling?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface ThrottleEvent {
|
|
31
|
+
url: string;
|
|
32
|
+
userAgent: string | null;
|
|
33
|
+
limit: number;
|
|
34
|
+
windowSeconds: number;
|
|
35
|
+
count: number;
|
|
36
|
+
/** Whether the refusal came with something to buy. */
|
|
37
|
+
sold: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ThrottleOptions {
|
|
41
|
+
/** Default 100. */
|
|
42
|
+
limit?: number;
|
|
43
|
+
/** Default 60. */
|
|
44
|
+
windowSeconds?: number;
|
|
45
|
+
/** Over-limit is answered 402 with this gateway's offer instead of 429. */
|
|
46
|
+
gateway?: Gateway | null;
|
|
47
|
+
/** Shared counter. Default: one per process, so each instance grants its own. */
|
|
48
|
+
store?: QuotaStore;
|
|
49
|
+
/** What to count against. Default: the caller's address, as the edge reported it. */
|
|
50
|
+
identify?: (request: Request) => string | null;
|
|
51
|
+
/** Never metered, e.g. a request carrying a signed-in session cookie. */
|
|
52
|
+
exempt?: (request: Request) => boolean;
|
|
53
|
+
/** Extra paths never metered. The gateway's sales page is added for you. */
|
|
54
|
+
openPaths?: string[];
|
|
55
|
+
/** Per-path allowances. Most specific match wins, whatever order they are in. */
|
|
56
|
+
rules?: ThrottleRule[];
|
|
57
|
+
/** Budget for callers presenting a credential. `false` to treat them as anonymous. */
|
|
58
|
+
credential?: false | CredentialBudget;
|
|
59
|
+
/** Told about every refusal. Its errors are swallowed. */
|
|
60
|
+
onThrottle?: (event: ThrottleEvent) => void | Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface Throttle {
|
|
64
|
+
/** A Response to send, or null to let the request through. */
|
|
65
|
+
handle: (request: Request) => Promise<Response | null>;
|
|
66
|
+
store: QuotaStore;
|
|
67
|
+
options: {
|
|
68
|
+
limit: number;
|
|
69
|
+
windowSeconds: number;
|
|
70
|
+
rules: Required<ThrottleRule>[];
|
|
71
|
+
credential: { limit: number; ceiling: number } | null;
|
|
72
|
+
openPaths: string[];
|
|
73
|
+
gateway: boolean;
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function createThrottle(options?: ThrottleOptions): Throttle;
|
|
78
|
+
|
|
79
|
+
export function memoryStore(options?: {
|
|
80
|
+
now?: () => number;
|
|
81
|
+
sweepEvery?: number;
|
|
82
|
+
}): QuotaStore & { sweep(): void; readonly size: number };
|
|
83
|
+
|
|
84
|
+
export function fingerprint(value: string): string;
|
|
85
|
+
export function presentedCredential(headers: Headers): string | null;
|
|
86
|
+
export function addressOf(request: Request): string | null;
|
|
87
|
+
export function matches(pattern: string, path: string): boolean;
|
|
88
|
+
export function matchesAny(patterns: string[], path: string): boolean;
|
|
89
|
+
export function compileRules(
|
|
90
|
+
rules: ThrottleRule[],
|
|
91
|
+
defaults: { limit: number; windowSeconds: number },
|
|
92
|
+
): Required<ThrottleRule>[];
|
|
93
|
+
export function ruleFor<T>(compiled: Required<ThrottleRule>[], path: string, defaults: T): T;
|
package/package.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@profullstack/throttle",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "An app-wide rate limit whose answer to going over is a price, not a 429. 100 requests a minute free, then x402 Payment Required with an offer, settled by CoinPay. Meters every route, so a scraper wearing a browser user agent is charged like any other crawler.",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"rate-limit",
|
|
8
|
+
"throttle",
|
|
9
|
+
"x402",
|
|
10
|
+
"402",
|
|
11
|
+
"payment-required",
|
|
12
|
+
"scraping",
|
|
13
|
+
"anti-scraping",
|
|
14
|
+
"crawler",
|
|
15
|
+
"nextjs",
|
|
16
|
+
"hono",
|
|
17
|
+
"middleware",
|
|
18
|
+
"coinpay",
|
|
19
|
+
"usdc"
|
|
20
|
+
],
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/profullstack/throttle.git"
|
|
24
|
+
},
|
|
25
|
+
"homepage": "https://github.com/profullstack/throttle#readme",
|
|
26
|
+
"bugs": {
|
|
27
|
+
"url": "https://github.com/profullstack/throttle/issues"
|
|
28
|
+
},
|
|
29
|
+
"license": "MIT",
|
|
30
|
+
"author": "Profullstack, LLC",
|
|
31
|
+
"exports": {
|
|
32
|
+
".": {
|
|
33
|
+
"types": "./index.d.ts",
|
|
34
|
+
"import": "./src/index.js"
|
|
35
|
+
},
|
|
36
|
+
"./next": {
|
|
37
|
+
"types": "./index.d.ts",
|
|
38
|
+
"import": "./src/next.js"
|
|
39
|
+
},
|
|
40
|
+
"./hono": {
|
|
41
|
+
"types": "./index.d.ts",
|
|
42
|
+
"import": "./src/hono.js"
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"types": "./index.d.ts",
|
|
46
|
+
"files": [
|
|
47
|
+
"src",
|
|
48
|
+
"index.d.ts",
|
|
49
|
+
"README.md",
|
|
50
|
+
"LICENSE"
|
|
51
|
+
],
|
|
52
|
+
"engines": {
|
|
53
|
+
"node": ">=20.11"
|
|
54
|
+
},
|
|
55
|
+
"sideEffects": false,
|
|
56
|
+
"dependencies": {
|
|
57
|
+
"@profullstack/x402-gateway": "^0.5.0"
|
|
58
|
+
},
|
|
59
|
+
"scripts": {
|
|
60
|
+
"test": "node --test"
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/hono.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { createThrottle } from './throttle.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The throttle as Hono middleware.
|
|
5
|
+
*
|
|
6
|
+
* import { throttle } from '@profullstack/throttle/hono';
|
|
7
|
+
* app.use('*', throttle({ gateway }));
|
|
8
|
+
*
|
|
9
|
+
* Register it before the routes and after the crawl gateway, so a crawler that
|
|
10
|
+
* declares itself is charged by its own list rather than metered first.
|
|
11
|
+
*/
|
|
12
|
+
export function throttle(throttleOrOptions) {
|
|
13
|
+
const t =
|
|
14
|
+
throttleOrOptions && typeof throttleOrOptions.handle === 'function'
|
|
15
|
+
? throttleOrOptions
|
|
16
|
+
: createThrottle(throttleOrOptions);
|
|
17
|
+
return async (c, next) => {
|
|
18
|
+
const answer = await t.handle(c.req.raw);
|
|
19
|
+
if (answer) return answer;
|
|
20
|
+
await next();
|
|
21
|
+
};
|
|
22
|
+
}
|
package/src/identify.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Who a request is counted as.
|
|
3
|
+
*
|
|
4
|
+
* The address, by default. The gateway's `clientIp` is what reads it, because
|
|
5
|
+
* it already knows to take the edge's own hop rather than whatever the client
|
|
6
|
+
* put in `X-Forwarded-For` -- a header the client writes, and which a scraper
|
|
7
|
+
* will happily fill with a fresh address per request if we let it decide.
|
|
8
|
+
*
|
|
9
|
+
* ON ROTATION. Rotating addresses defeats an address bucket. That is not a
|
|
10
|
+
* hole to be plugged, it is the argument the 402 makes: residential proxy
|
|
11
|
+
* bandwidth is sold by the gigabyte, and a rotation still fetches every page
|
|
12
|
+
* one at a time. Anyone spending more on proxies than the pass costs, to avoid
|
|
13
|
+
* the pass, has made an arithmetic mistake rather than beaten a detector.
|
|
14
|
+
* Detection is a race; price is not.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { clientIp } from '@profullstack/x402-gateway';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* FNV-1a. A credential must never become a map key we might log or dump, and
|
|
21
|
+
* the counter only ever needs to know that two requests carried the same one.
|
|
22
|
+
*/
|
|
23
|
+
export function fingerprint(value) {
|
|
24
|
+
let hash = 0x811c9dc5;
|
|
25
|
+
const text = String(value);
|
|
26
|
+
for (let i = 0; i < text.length; i++) {
|
|
27
|
+
hash ^= text.charCodeAt(i);
|
|
28
|
+
hash = Math.imul(hash, 0x01000193);
|
|
29
|
+
}
|
|
30
|
+
return (hash >>> 0).toString(36);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The credential a request presents, unverified.
|
|
35
|
+
*
|
|
36
|
+
* Unverified on purpose: this layer decides how much budget a caller gets, and
|
|
37
|
+
* the route handler is what actually authenticates. Any auth scheme counts,
|
|
38
|
+
* not just Bearer -- the CoinPay wallet extension signs with
|
|
39
|
+
* `Authorization: Wallet <id>:<sig>:<ts>`, and matching Bearer alone dropped a
|
|
40
|
+
* bulk payout into the anonymous per-address bucket, which it exhausted in
|
|
41
|
+
* seconds.
|
|
42
|
+
*/
|
|
43
|
+
export function presentedCredential(headers) {
|
|
44
|
+
const authorization = headers.get('authorization');
|
|
45
|
+
if (authorization) {
|
|
46
|
+
const match = /^(\S+)\s+(\S+)/.exec(authorization.trim());
|
|
47
|
+
if (match) return match[2];
|
|
48
|
+
}
|
|
49
|
+
const apiKey = headers.get('x-api-key')?.trim();
|
|
50
|
+
return apiKey ? apiKey : null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The address bucket for a request, or null when the edge gave us nothing. */
|
|
54
|
+
export function addressOf(request) {
|
|
55
|
+
const ip = clientIp(request);
|
|
56
|
+
return ip ? String(ip) : null;
|
|
57
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { createThrottle } from './throttle.js';
|
|
2
|
+
export { memoryStore } from './store.js';
|
|
3
|
+
export { addressOf, fingerprint, presentedCredential } from './identify.js';
|
|
4
|
+
export {
|
|
5
|
+
ALWAYS_OPEN,
|
|
6
|
+
DEFAULT_LIMIT,
|
|
7
|
+
DEFAULT_WINDOW_SECONDS,
|
|
8
|
+
compileRules,
|
|
9
|
+
matches,
|
|
10
|
+
matchesAny,
|
|
11
|
+
ruleFor,
|
|
12
|
+
} from './rules.js';
|
package/src/next.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { createThrottle } from './throttle.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The throttle for Next.js, in `proxy.ts` (Next 16) or `middleware.ts`.
|
|
5
|
+
*
|
|
6
|
+
* import { throttleProxy } from '@profullstack/throttle/next';
|
|
7
|
+
* export const proxy = throttleProxy({ gateway });
|
|
8
|
+
*
|
|
9
|
+
* Returning `undefined` means "carry on", which is what the throttle's null
|
|
10
|
+
* becomes. Most apps already have a proxy doing CORS and security headers, so
|
|
11
|
+
* compose instead:
|
|
12
|
+
*
|
|
13
|
+
* const answer = await throttle.handle(request);
|
|
14
|
+
* if (answer) return answer;
|
|
15
|
+
*
|
|
16
|
+
* Put it after the crawl gateway and before everything else. The gateway
|
|
17
|
+
* charges crawlers that declare themselves; the throttle charges the ones that
|
|
18
|
+
* do not, and there is no point running CORS on a request about to be refused.
|
|
19
|
+
*
|
|
20
|
+
* Runs at the edge: nothing here imports node:, and the pass is verified with
|
|
21
|
+
* Web Crypto rather than a database.
|
|
22
|
+
*/
|
|
23
|
+
export function throttleProxy(throttleOrOptions) {
|
|
24
|
+
const throttle =
|
|
25
|
+
throttleOrOptions && typeof throttleOrOptions.handle === 'function'
|
|
26
|
+
? throttleOrOptions
|
|
27
|
+
: createThrottle(throttleOrOptions);
|
|
28
|
+
return async (request) => (await throttle.handle(request)) ?? undefined;
|
|
29
|
+
}
|
package/src/rules.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which allowance applies to a request.
|
|
3
|
+
*
|
|
4
|
+
* A site does not have one rate limit, it has a few. Signing in has to be
|
|
5
|
+
* tighter than reading, because ten guesses a minute is a brute-force budget
|
|
6
|
+
* and a hundred is a giveaway. A server-to-server integration has to be looser
|
|
7
|
+
* than a browser, because one host legitimately sends a burst that would look
|
|
8
|
+
* like abuse from a person. Everything else -- every page, every feed, every
|
|
9
|
+
* lookup -- shares one default, and that default is the whole point of this
|
|
10
|
+
* package: it meters routes nobody thought to protect, which is where the
|
|
11
|
+
* scrapers actually are.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** The house default: a hundred requests a minute, per caller, everywhere. */
|
|
15
|
+
export const DEFAULT_LIMIT = 100;
|
|
16
|
+
export const DEFAULT_WINDOW_SECONDS = 60;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Paths that are never metered, whatever the rules say.
|
|
20
|
+
*
|
|
21
|
+
* A refused caller has to be able to read the terms and buy its way out, or
|
|
22
|
+
* the 402 is a dead end: the sales page, robots.txt and the well-known paths
|
|
23
|
+
* are how a crawler complies. The gateway already knows which those are, so
|
|
24
|
+
* when one is supplied they are read off it rather than restated here.
|
|
25
|
+
*/
|
|
26
|
+
export const ALWAYS_OPEN = ['/robots.txt', '/favicon.ico', '/.well-known/'];
|
|
27
|
+
|
|
28
|
+
const trim = (path) => String(path ?? '').trim();
|
|
29
|
+
|
|
30
|
+
/** Prefix when it ends in `/`, exact match otherwise -- the gateway's rule. */
|
|
31
|
+
export function matches(pattern, path) {
|
|
32
|
+
const p = trim(pattern);
|
|
33
|
+
if (!p) return false;
|
|
34
|
+
return p.endsWith('/') ? path.startsWith(p) : path === p;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function matchesAny(patterns, path) {
|
|
38
|
+
return patterns.some((p) => matches(p, path));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Normalise one rule. A rule is a path pattern plus the numbers that replace
|
|
43
|
+
* the defaults for it; anything it leaves out it inherits, so `{ path:
|
|
44
|
+
* '/api/auth/', limit: 10 }` is a complete rule.
|
|
45
|
+
*/
|
|
46
|
+
function normaliseRule(rule, defaults) {
|
|
47
|
+
const limit = Number(rule?.limit);
|
|
48
|
+
const windowSeconds = Number(rule?.windowSeconds);
|
|
49
|
+
return {
|
|
50
|
+
path: trim(rule?.path),
|
|
51
|
+
limit: Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : defaults.limit,
|
|
52
|
+
windowSeconds:
|
|
53
|
+
Number.isFinite(windowSeconds) && windowSeconds > 0
|
|
54
|
+
? Math.floor(windowSeconds)
|
|
55
|
+
: defaults.windowSeconds,
|
|
56
|
+
/**
|
|
57
|
+
* A rule can opt out of metering entirely (`{ path: '/health', open: true }`),
|
|
58
|
+
* which is how an app exempts a route without having to restate the whole
|
|
59
|
+
* skip list.
|
|
60
|
+
*/
|
|
61
|
+
open: Boolean(rule?.open),
|
|
62
|
+
/** Credentialed callers on this rule, if it wants different numbers. */
|
|
63
|
+
credential: rule?.credential ?? null,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Compile the rule list. Longest pattern first, so `/api/auth/` is consulted
|
|
69
|
+
* before `/api/` however the site happened to order them -- the alternative is
|
|
70
|
+
* a config where moving two lines silently removes the brute-force limit.
|
|
71
|
+
*/
|
|
72
|
+
export function compileRules(rules = [], defaults) {
|
|
73
|
+
return rules
|
|
74
|
+
.filter((rule) => trim(rule?.path))
|
|
75
|
+
.map((rule) => normaliseRule(rule, defaults))
|
|
76
|
+
.sort((a, b) => b.path.length - a.path.length);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The rule for a path: the most specific match, or the default allowance. */
|
|
80
|
+
export function ruleFor(compiled, path, defaults) {
|
|
81
|
+
return compiled.find((rule) => matches(rule.path, path)) ?? defaults;
|
|
82
|
+
}
|
package/src/store.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the counts live.
|
|
3
|
+
*
|
|
4
|
+
* The default counts in this process, which means several instances each grant
|
|
5
|
+
* their own allowance. That errs toward generosity, which is the right
|
|
6
|
+
* direction to be wrong in for a free tier -- and a site that wants one shared
|
|
7
|
+
* count across a fleet passes a store of its own. The interface is a single
|
|
8
|
+
* method, so Redis, Postgres or Cloudflare KV is a dozen lines.
|
|
9
|
+
*
|
|
10
|
+
* { hit(key, windowSeconds): { count, resetSeconds } }
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export { memoryQuotaStore as memoryStore } from '@profullstack/x402-gateway';
|
package/src/throttle.js
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An app-wide rate limit whose answer to going over is a price.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS. Every site in the fleet had a rate limiter in its
|
|
5
|
+
* middleware, each one hand-rolled, each one scoped to `/api/`. Then a
|
|
6
|
+
* headless browser in Singapore found a newly shipped block explorer and spent
|
|
7
|
+
* two days walking it: 19,000 URLs a day, one fresh visitor id per hit, a
|
|
8
|
+
* plain `Chrome/145` user agent, every page rendered server-side against a
|
|
9
|
+
* paid upstream API. Nothing stopped it. It was not on a crawler list, because
|
|
10
|
+
* it declared nothing; it was not rate limited, because it never touched
|
|
11
|
+
* `/api/`; and the analytics counted every hit as a human visit. The limiter
|
|
12
|
+
* we had protected the routes nobody was attacking.
|
|
13
|
+
*
|
|
14
|
+
* So: meter everything, and when a caller goes over, sell it a pass. 429 is
|
|
15
|
+
* the right answer when you have nothing to sell. We do. The moment the
|
|
16
|
+
* allowance runs out is the best sales pitch the site will ever get, because
|
|
17
|
+
* the caller has just demonstrated it wants more than the free tier and is
|
|
18
|
+
* still holding the request open.
|
|
19
|
+
*
|
|
20
|
+
* A hundred requests a minute is the default. A person reading a site produces
|
|
21
|
+
* a few dozen; a scraper produces that in a second.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { quotaHeaders, spend } from '@profullstack/x402-gateway';
|
|
25
|
+
import { addressOf, fingerprint, presentedCredential } from './identify.js';
|
|
26
|
+
import {
|
|
27
|
+
ALWAYS_OPEN,
|
|
28
|
+
DEFAULT_LIMIT,
|
|
29
|
+
DEFAULT_WINDOW_SECONDS,
|
|
30
|
+
compileRules,
|
|
31
|
+
matchesAny,
|
|
32
|
+
ruleFor,
|
|
33
|
+
} from './rules.js';
|
|
34
|
+
import { memoryStore } from './store.js';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {object} options
|
|
38
|
+
* @param {number} [options.limit=100] requests per window, per caller
|
|
39
|
+
* @param {number} [options.windowSeconds=60]
|
|
40
|
+
* @param {object} [options.gateway] a @profullstack/x402-gateway; over-limit is answered 402 with its offer
|
|
41
|
+
* @param {object} [options.store] shared counter; default is per-process
|
|
42
|
+
* @param {(request: Request) => string|null} [options.identify] what to count against; default the caller's address
|
|
43
|
+
* @param {(request: Request) => boolean} [options.exempt] never metered, e.g. a signed-in session
|
|
44
|
+
* @param {string[]} [options.openPaths] extra paths never metered
|
|
45
|
+
* @param {Array} [options.rules] per-path allowances, most specific wins
|
|
46
|
+
* @param {{limit?: number, ceiling?: number}} [options.credential] budget for callers presenting a credential
|
|
47
|
+
* @param {(event: object) => void|Promise<void>} [options.onThrottle] told about every refusal; its errors are swallowed
|
|
48
|
+
*/
|
|
49
|
+
export function createThrottle(options = {}) {
|
|
50
|
+
const defaults = {
|
|
51
|
+
path: '',
|
|
52
|
+
limit: positive(options.limit, DEFAULT_LIMIT),
|
|
53
|
+
windowSeconds: positive(options.windowSeconds, DEFAULT_WINDOW_SECONDS),
|
|
54
|
+
open: false,
|
|
55
|
+
credential: null,
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const store = options.store?.hit ? options.store : memoryStore();
|
|
59
|
+
const rules = compileRules(options.rules, defaults);
|
|
60
|
+
const gateway = options.gateway ?? null;
|
|
61
|
+
const identify = options.identify ?? addressOf;
|
|
62
|
+
const exempt = options.exempt ?? null;
|
|
63
|
+
const onThrottle = options.onThrottle ?? null;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* A credentialed caller gets its own, larger budget -- one host running an
|
|
67
|
+
* integration is not one browser. Both buckets are charged: the per-address
|
|
68
|
+
* ceiling is what a caller rotating junk credentials cannot escape, since
|
|
69
|
+
* this layer has not authenticated any of them.
|
|
70
|
+
*/
|
|
71
|
+
const credential = options.credential === false ? null : {
|
|
72
|
+
limit: positive(options.credential?.limit, 600),
|
|
73
|
+
ceiling: positive(options.credential?.ceiling, 1200),
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/*
|
|
77
|
+
* The paths a refused caller must still be able to read. Taken off the
|
|
78
|
+
* gateway when there is one, so the sales page it points at in the 402 is
|
|
79
|
+
* never itself throttled -- a paywall you cannot reach to pay is a wall.
|
|
80
|
+
*/
|
|
81
|
+
const openPaths = [
|
|
82
|
+
...ALWAYS_OPEN,
|
|
83
|
+
...(gateway ? [gateway.options.path, ...(gateway.options.openPaths ?? [])] : []),
|
|
84
|
+
...(options.openPaths ?? []),
|
|
85
|
+
];
|
|
86
|
+
|
|
87
|
+
const quotaFor = (rule) => ({
|
|
88
|
+
requests: rule.limit,
|
|
89
|
+
windowSeconds: rule.windowSeconds,
|
|
90
|
+
store,
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
/** 429, for a site with nothing to sell. */
|
|
94
|
+
const tooMany = (rule, usage) => {
|
|
95
|
+
const headers = {
|
|
96
|
+
'content-type': 'application/json; charset=utf-8',
|
|
97
|
+
'cache-control': 'no-store',
|
|
98
|
+
'retry-after': String(usage.resetSeconds),
|
|
99
|
+
...quotaHeaders(quotaFor(rule), usage),
|
|
100
|
+
};
|
|
101
|
+
return new Response(
|
|
102
|
+
JSON.stringify(
|
|
103
|
+
{
|
|
104
|
+
error: 'Too many requests',
|
|
105
|
+
limit: rule.limit,
|
|
106
|
+
windowSeconds: rule.windowSeconds,
|
|
107
|
+
retryAfter: usage.resetSeconds,
|
|
108
|
+
},
|
|
109
|
+
null,
|
|
110
|
+
2,
|
|
111
|
+
),
|
|
112
|
+
{ status: 429, headers },
|
|
113
|
+
);
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* What a caller over the limit is answered with. A gateway that can take
|
|
118
|
+
* money answers 402 with an offer -- or the sales page, if the caller asked
|
|
119
|
+
* for HTML, which is what makes this survivable for a person who tripped it
|
|
120
|
+
* by accident. Without one there is nothing to sell, so 429.
|
|
121
|
+
*/
|
|
122
|
+
async function refuse(request, rule, usage) {
|
|
123
|
+
if (onThrottle) {
|
|
124
|
+
try {
|
|
125
|
+
await onThrottle({
|
|
126
|
+
url: request.url,
|
|
127
|
+
userAgent: request.headers.get('user-agent') ?? null,
|
|
128
|
+
limit: rule.limit,
|
|
129
|
+
windowSeconds: rule.windowSeconds,
|
|
130
|
+
count: usage.count,
|
|
131
|
+
sold: Boolean(gateway?.enabled),
|
|
132
|
+
});
|
|
133
|
+
} catch {
|
|
134
|
+
// Telling someone must never cost the caller its answer.
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (!gateway) return tooMany(rule, usage);
|
|
138
|
+
return gateway.sell(request, {
|
|
139
|
+
usage,
|
|
140
|
+
quota: { requests: rule.limit, windowSeconds: rule.windowSeconds },
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The throttle. Null means "not for me, carry on".
|
|
146
|
+
*/
|
|
147
|
+
async function handle(request) {
|
|
148
|
+
const path = new URL(request.url).pathname;
|
|
149
|
+
|
|
150
|
+
if (matchesAny(openPaths, path)) return null;
|
|
151
|
+
if (exempt && exempt(request)) return null;
|
|
152
|
+
|
|
153
|
+
const rule = ruleFor(rules, path, defaults);
|
|
154
|
+
if (rule.open) return null;
|
|
155
|
+
|
|
156
|
+
/*
|
|
157
|
+
* Whoever already paid is not metered. The pass is the thing being sold;
|
|
158
|
+
* charging a caller that holds one, or counting it toward an allowance it
|
|
159
|
+
* bought its way out of, is selling the same minute twice.
|
|
160
|
+
*/
|
|
161
|
+
if (gateway) {
|
|
162
|
+
const token = gateway.passFrom(request);
|
|
163
|
+
if (token && (await gateway.verifyPass(token))) return null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const key = identify(request);
|
|
167
|
+
/*
|
|
168
|
+
* A caller we cannot identify is within the allowance. Our own edge
|
|
169
|
+
* failing to hand us an address is our gap, and charging someone for it
|
|
170
|
+
* would be charging them for our gap.
|
|
171
|
+
*/
|
|
172
|
+
if (!key) return null;
|
|
173
|
+
|
|
174
|
+
/*
|
|
175
|
+
* A rule can refuse to honour credentials at all (`credential: false`),
|
|
176
|
+
* and the sign-in routes must: auth endpoints stay address-bucketed
|
|
177
|
+
* whatever headers accompany them, or a brute-force attempt would bolt an
|
|
178
|
+
* Authorization header onto every guess and buy itself the integration
|
|
179
|
+
* budget.
|
|
180
|
+
*/
|
|
181
|
+
const honoursCredential = Boolean(credential) && rule.credential !== false;
|
|
182
|
+
const presented = honoursCredential ? presentedCredential(request.headers) : null;
|
|
183
|
+
if (presented) {
|
|
184
|
+
const ceilingRule = { ...rule, limit: rule.credential?.ceiling ?? credential.ceiling };
|
|
185
|
+
const perKeyRule = { ...rule, limit: rule.credential?.limit ?? credential.limit };
|
|
186
|
+
const ceiling = await spend(quotaFor(ceilingRule), `ceiling:${key}`);
|
|
187
|
+
const perKey = await spend(quotaFor(perKeyRule), `cred:${fingerprint(presented)}`);
|
|
188
|
+
/*
|
|
189
|
+
* The host ceiling is what a credential-rotating caller cannot escape,
|
|
190
|
+
* so a denial there beats a healthy per-credential budget.
|
|
191
|
+
*/
|
|
192
|
+
if (ceiling?.overLimit) return refuse(request, ceilingRule, ceiling);
|
|
193
|
+
if (perKey?.overLimit) return refuse(request, perKeyRule, perKey);
|
|
194
|
+
return null;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const usage = await spend(quotaFor(rule), `${rule.path || '*'}:${key}`);
|
|
198
|
+
if (usage?.overLimit) return refuse(request, rule, usage);
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
handle,
|
|
204
|
+
/** For tests and for a site that wants to reclaim memory on demand. */
|
|
205
|
+
store,
|
|
206
|
+
options: { ...defaults, rules, credential, openPaths, gateway: Boolean(gateway) },
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function positive(value, fallback) {
|
|
211
|
+
const n = Number(value);
|
|
212
|
+
return Number.isFinite(n) && n >= 1 ? Math.floor(n) : fallback;
|
|
213
|
+
}
|