@apideck/agent-analytics 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -0
- package/dist/adapters/posthog.cjs +2 -2
- package/dist/adapters/posthog.cjs.map +1 -1
- package/dist/adapters/posthog.js +2 -2
- package/dist/adapters/posthog.js.map +1 -1
- package/dist/firewall.cjs +4 -0
- package/dist/firewall.cjs.map +1 -0
- package/dist/firewall.d.cts +115 -0
- package/dist/firewall.d.ts +115 -0
- package/dist/firewall.js +4 -0
- package/dist/firewall.js.map +1 -0
- package/dist/gateway-CYTQwayu.d.cts +317 -0
- package/dist/gateway-YJL5J0LR.d.ts +317 -0
- package/dist/index.cjs +2 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -76
- package/dist/index.d.ts +5 -76
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/pay.cjs +8 -0
- package/dist/pay.cjs.map +1 -0
- package/dist/pay.d.cts +112 -0
- package/dist/pay.d.ts +112 -0
- package/dist/pay.js +8 -0
- package/dist/pay.js.map +1 -0
- package/dist/policy-B3AakjOJ.d.cts +96 -0
- package/dist/policy-DMTBUe4F.d.ts +96 -0
- package/package.json +15 -2
package/README.md
CHANGED
|
@@ -82,6 +82,87 @@ Now you can build:
|
|
|
82
82
|
|
|
83
83
|
---
|
|
84
84
|
|
|
85
|
+
## Charging for training crawls (experimental)
|
|
86
|
+
|
|
87
|
+
> **⚠️ Experimental.** The payment surface — `paymentRequired`, `paymentGate`,
|
|
88
|
+
> `x402Gateway`, `mppxGateway` — may change without a major version bump. The
|
|
89
|
+
> protocols are weeks old and still moving: x402 and MPP are both live but their
|
|
90
|
+
> specs are unstable, MPP had not publicly pinned a settlement-confirmation
|
|
91
|
+
> header at the time of writing, and no agent in our own production traffic has
|
|
92
|
+
> yet presented a payment credential. Detection, verification and policy are
|
|
93
|
+
> stable; this is not. Do not put it on a revenue-critical path yet.
|
|
94
|
+
|
|
95
|
+
### Meter first. Charge later, if at all.
|
|
96
|
+
|
|
97
|
+
Per-request 402 is what x402 and MPP define, and it is the wrong shape for a
|
|
98
|
+
training sweep. On one production site that is ~199,000 training requests a
|
|
99
|
+
month: three times the traffic once you add pay-and-retry, 199,000 settlements
|
|
100
|
+
whose per-transaction cost exceeds any sane per-page price, and — decisively —
|
|
101
|
+
**no crawler in the wild retries a 402**. Charging per request is blocking with
|
|
102
|
+
extra steps.
|
|
103
|
+
|
|
104
|
+
So start by counting:
|
|
105
|
+
|
|
106
|
+
```ts
|
|
107
|
+
import { paymentGate } from '@apideck/agent-analytics/payments'
|
|
108
|
+
import { combinedVerifier } from '@apideck/agent-analytics/verify'
|
|
109
|
+
|
|
110
|
+
const gate = await paymentGate(req, {
|
|
111
|
+
verify: combinedVerifier(),
|
|
112
|
+
meter: { record: (e) => warehouse.insert(e) } // training only
|
|
113
|
+
})
|
|
114
|
+
if (gate.response) return gate.response
|
|
115
|
+
return gate.decorate(await serve(req))
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`meter` fires only for training traffic. Retrieval and search are served free
|
|
119
|
+
and never counted, because charging the channel that sends you readers is the
|
|
120
|
+
one outcome this design exists to prevent.
|
|
121
|
+
|
|
122
|
+
### Then sell a licence, not a page
|
|
123
|
+
|
|
124
|
+
When you know the number, switch to an entitlement: one 402 advertising a bulk
|
|
125
|
+
offer, one settlement, a reusable credential.
|
|
126
|
+
|
|
127
|
+
```ts
|
|
128
|
+
import { entitlementGateway } from '@apideck/agent-analytics/payments'
|
|
129
|
+
|
|
130
|
+
const gate = await paymentGate(req, {
|
|
131
|
+
onTraining: 'charge',
|
|
132
|
+
gateway: entitlementGateway({
|
|
133
|
+
store: myKV, // lookup + consume; quota state is yours
|
|
134
|
+
offer: { units: 1_000_000, unit: 'pages', validForSeconds: 2_592_000, price: '$400' },
|
|
135
|
+
challenges: [{ protocol: 'mpp', id, realm: 'example.com', method: 'tempo' }]
|
|
136
|
+
})
|
|
137
|
+
})
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
402 once, advertising the licence
|
|
142
|
+
200 every request after, quota −1
|
|
143
|
+
402 again when it runs out
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Unknown, expired and exhausted credentials all return the same challenge —
|
|
147
|
+
distinguishing them would turn the endpoint into an oracle for probing quota.
|
|
148
|
+
|
|
149
|
+
MPP's reusable `Authorization: Payment` credential suits this better than
|
|
150
|
+
x402's per-resource signature, which proves payment for a single URL.
|
|
151
|
+
|
|
152
|
+
Measured against real traffic shapes:
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
402 training charge GPTBot
|
|
156
|
+
serve retrieval allow ChatGPT-User
|
|
157
|
+
403 training block ClaudeBot from an unpublished IP
|
|
158
|
+
402 training charge ClaudeBot from a real Anthropic IP
|
|
159
|
+
serve search allow Googlebot
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Settlement is never ours. `mppxGateway` wraps Stripe's MPP SDK; `x402Gateway`
|
|
163
|
+
calls a facilitator you supply. The library emits challenges and reads
|
|
164
|
+
credentials — holding money would drag PCI scope into edge middleware.
|
|
165
|
+
|
|
85
166
|
## Cryptographic verification (Web Bot Auth)
|
|
86
167
|
|
|
87
168
|
Published IP ranges were always the weak form of identity. [Web Bot
|
|
@@ -170,6 +251,69 @@ without Web Crypto now fail with an explicit message rather than a confusing
|
|
|
170
251
|
- Outbound captures carry a 3s `AbortSignal` (`timeoutMs` to change it).
|
|
171
252
|
- Root bundle is 65% smaller (27.7 kB → 9.6 kB, 3.8 kB gzipped).
|
|
172
253
|
|
|
254
|
+
## Recommending firewall rules
|
|
255
|
+
|
|
256
|
+
Turn observed traffic into staged Vercel WAF proposals. It emits *proposals* —
|
|
257
|
+
every rule comes out in `log` mode and Vercel stages rule changes as drafts, so
|
|
258
|
+
nothing is live until you run `vercel firewall publish` yourself.
|
|
259
|
+
|
|
260
|
+
```ts
|
|
261
|
+
import { recommendFirewallRules, firewallScript } from '@apideck/agent-analytics/firewall'
|
|
262
|
+
|
|
263
|
+
const rules = recommendFirewallRules(observations) // aggregate from your warehouse
|
|
264
|
+
console.log(firewallScript(rules)) // runnable, commented bash
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
Two rules it will not break, both from measurement rather than taste:
|
|
268
|
+
|
|
269
|
+
- **Retrieval and search agents are never proposed for blocking**, and a `bypass`
|
|
270
|
+
rule protecting them is emitted *first* so later rules cannot catch them.
|
|
271
|
+
Rules evaluate top to bottom, and 60% of AI traffic on one production site is
|
|
272
|
+
a person asking a question.
|
|
273
|
+
- **Training crawlers get rate limits, not denials.** Denying them removes you
|
|
274
|
+
from future training sets, which is a discoverability decision rather than a
|
|
275
|
+
default.
|
|
276
|
+
|
|
277
|
+
Only a failed verification earns a proposed `deny`. Every recommendation carries
|
|
278
|
+
its `evidence`, a `risk` rating, and a `caveat` where over-blocking is plausible
|
|
279
|
+
— the datacenter-ASN rule is marked `high` risk because corporate VPNs and
|
|
280
|
+
privacy relays egress from hosting networks.
|
|
281
|
+
|
|
282
|
+
See [`docs/TESTING-PAYMENTS.md`](./docs/TESTING-PAYMENTS.md) for testing the
|
|
283
|
+
payment path end to end.
|
|
284
|
+
|
|
285
|
+
## Entry points
|
|
286
|
+
|
|
287
|
+
The root carries detection, classification, policy and capture — what every
|
|
288
|
+
consumer needs. Everything optional lives behind a subpath, so it only reaches
|
|
289
|
+
your bundle if you import it.
|
|
290
|
+
|
|
291
|
+
| Import | Contains | Root bundle cost |
|
|
292
|
+
| --- | --- | ---: |
|
|
293
|
+
| `@apideck/agent-analytics` | detection, classification, `agentPolicy`, `trackVisit` | 11.6 kB / **4.5 kB gz** |
|
|
294
|
+
| `…/verify` | Web Bot Auth + published IP range tables | 19.0 kB |
|
|
295
|
+
| `…/payments` | 402 challenges, gateways, entitlements | 10.9 kB |
|
|
296
|
+
| `…/firewall` | WAF rule recommendations (offline tool) | 6.8 kB |
|
|
297
|
+
| `…/markdown` | Markdown-twin content negotiation | 2.0 kB |
|
|
298
|
+
|
|
299
|
+
This split is load-bearing rather than tidy-minded. Exporting the payment and
|
|
300
|
+
firewall surfaces from the root once pushed it from 9.6 kB to 22.5 kB — every
|
|
301
|
+
site paid for a firewall recommender that will never run in middleware. Nothing
|
|
302
|
+
failed; the number just drifted for weeks until someone looked.
|
|
303
|
+
|
|
304
|
+
So CI now enforces it. `npm run size` checks each entry against
|
|
305
|
+
[`size-budget.json`](./size-budget.json) and fails the build on a regression:
|
|
306
|
+
|
|
307
|
+
```
|
|
308
|
+
entry gzipped budget used
|
|
309
|
+
dist/index.js 4.44 kB 4.88 kB 91%
|
|
310
|
+
dist/verify.js 6.25 kB 7.42 kB 84%
|
|
311
|
+
dist/pay.js 4.05 kB 4.49 kB 90%
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
Raising a budget is deliberate — `npm run size -- --update`, and say why in the
|
|
315
|
+
commit.
|
|
316
|
+
|
|
173
317
|
## Install
|
|
174
318
|
|
|
175
319
|
```bash
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
'use strict';var
|
|
2
|
-
exports.posthogAnalytics=
|
|
1
|
+
'use strict';var o=class extends Error{status;body;constructor(s,i,p){super(s),this.name="CaptureTransportError",this.status=i,this.body=p;}};function y(t){let s=t.host??"https://us.i.posthog.com",i=(/^https?:\/\//.test(s)?s:`https://${s}`).replace(/\/$/,""),p=(t.path??"/i/v0/e/").replace(/^(?!\/)/,"/"),c=`${i}${p}`,u=t.fetchImpl??fetch;return {async capture(e){let a=e.properties.user_agent,n=e.properties.client_ip,d={api_key:t.apiKey,event:e.event,distinct_id:e.distinctId,timestamp:e.timestamp,properties:{...e.properties,...typeof a=="string"&&a?{$raw_user_agent:a}:{},...typeof n=="string"&&n?{$ip:n}:{}}},r=await u(c,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d),keepalive:true,signal:AbortSignal.timeout(t.timeoutMs??3e3)});if(!r.ok)throw new o(`PostHog capture failed: ${r.status} ${r.statusText}`,r.status,await r.text().catch(()=>{}))}}}
|
|
2
|
+
exports.posthogAnalytics=y;//# sourceMappingURL=posthog.cjs.map
|
|
3
3
|
//# sourceMappingURL=posthog.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/errors.ts","../../src/adapters/posthog.ts"],"names":["CaptureTransportError","message","status","body","posthogAnalytics","config","hostRaw","base","path","endpoint","fetchImpl","event","payload","res"],"mappings":"aACO,IAAMA,
|
|
1
|
+
{"version":3,"sources":["../../src/errors.ts","../../src/adapters/posthog.ts"],"names":["CaptureTransportError","message","status","body","posthogAnalytics","config","hostRaw","base","path","endpoint","fetchImpl","event","ua","ip","payload","res"],"mappings":"aACO,IAAMA,EAAN,cAAoC,KAAM,CACtC,MAAA,CACA,IAAA,CACT,YAAYC,CAAAA,CAAiBC,CAAAA,CAAiBC,CAAAA,CAAe,CAC3D,MAAMF,CAAO,CAAA,CACb,KAAK,IAAA,CAAO,uBAAA,CACZ,KAAK,MAAA,CAASC,CAAAA,CACd,IAAA,CAAK,IAAA,CAAOC,EACd,CACF,CAAA,CC2BO,SAASC,CAAAA,CAAiBC,CAAAA,CAAgD,CAC/E,IAAMC,CAAAA,CAAUD,EAAO,IAAA,EAAQ,0BAAA,CACzBE,GAAQ,cAAA,CAAe,IAAA,CAAKD,CAAO,CAAA,CAAIA,CAAAA,CAAU,WAAWA,CAAO,CAAA,CAAA,EAAI,OAAA,CAAQ,KAAA,CAAO,EAAE,CAAA,CACxFE,CAAAA,CAAAA,CAAQH,EAAO,IAAA,EAAQ,UAAA,EAAY,QAAQ,SAAA,CAAW,GAAG,CAAA,CACzDI,CAAAA,CAAW,GAAGF,CAAI,CAAA,EAAGC,CAAI,CAAA,CAAA,CACzBE,CAAAA,CAAYL,EAAO,SAAA,EAAa,KAAA,CAEtC,OAAO,CACL,MAAM,OAAA,CAAQM,CAAAA,CAAoC,CAchD,IAAMC,CAAAA,CAAKD,EAAM,UAAA,CAAW,UAAA,CACtBE,EAAKF,CAAAA,CAAM,UAAA,CAAW,UACtBG,CAAAA,CAAU,CACd,QAAST,CAAAA,CAAO,MAAA,CAChB,MAAOM,CAAAA,CAAM,KAAA,CACb,WAAA,CAAaA,CAAAA,CAAM,WACnB,SAAA,CAAWA,CAAAA,CAAM,UACjB,UAAA,CAAY,CACV,GAAGA,CAAAA,CAAM,UAAA,CACT,GAAI,OAAOC,GAAO,QAAA,EAAYA,CAAAA,CAAK,CAAE,eAAA,CAAiBA,CAAG,EAAI,EAAC,CAC9D,GAAI,OAAOC,GAAO,QAAA,EAAYA,CAAAA,CAAK,CAAE,GAAA,CAAKA,CAAG,EAAI,EACnD,CACF,CAAA,CAIME,CAAAA,CAAM,MAAML,CAAAA,CAAUD,CAAAA,CAAU,CACpC,MAAA,CAAQ,MAAA,CACR,QAAS,CAAE,cAAA,CAAgB,kBAAmB,CAAA,CAC9C,KAAM,IAAA,CAAK,SAAA,CAAUK,CAAO,CAAA,CAC5B,SAAA,CAAW,KACX,MAAA,CAAQ,WAAA,CAAY,QAAQT,CAAAA,CAAO,SAAA,EAAa,GAAI,CACtD,CAAC,EACD,GAAI,CAACU,EAAI,EAAA,CACP,MAAM,IAAIf,CAAAA,CACR,2BAA2Be,CAAAA,CAAI,MAAM,IAAIA,CAAAA,CAAI,UAAU,GACvDA,CAAAA,CAAI,MAAA,CACJ,MAAMA,CAAAA,CAAI,IAAA,GAAO,KAAA,CAAM,IAAG,EAAY,CACxC,CAEJ,CACF,CACF","file":"posthog.cjs","sourcesContent":["/** Thrown when the analytics backend rejects, errors, or times out a capture. */\nexport class CaptureTransportError extends Error {\n readonly status: number | undefined\n readonly body: string | undefined\n constructor(message: string, status?: number, body?: string) {\n super(message)\n this.name = 'CaptureTransportError'\n this.status = status\n this.body = body\n }\n}\n","import type { AnalyticsAdapter, CaptureEvent } from '../types.js'\nimport { CaptureTransportError } from '../errors.js'\n\n\nexport interface PostHogAdapterConfig {\n /** PostHog project API key (the public one used by the JS SDK). */\n apiKey: string\n /**\n * PostHog host, with or without scheme. Defaults to `https://us.i.posthog.com`.\n * Use `https://eu.i.posthog.com` for EU cloud, or your own reverse-proxy\n * domain (e.g. `https://svc.example.com`).\n */\n host?: string\n /**\n * Path on the host that accepts single-event captures. Defaults to\n * `/i/v0/e/` which is PostHog's current endpoint for this.\n */\n path?: string\n /**\n * Override the `fetch` implementation (useful for tests or custom runtimes\n * that need a pinned fetch).\n */\n fetchImpl?: typeof fetch\n /**\n * Abort the capture after this many milliseconds. Defaults to 3000. Without\n * a bound, a hung backend leaves a pending promise for the lifetime of an\n * edge invocation.\n */\n timeoutMs?: number\n}\n\n/**\n * Adapter that posts each event to the PostHog capture endpoint. Uses\n * `keepalive: true` so the request survives after a serverless response\n * returns — events aren't guaranteed (fire-and-forget), but that's the\n * trade we want to keep the hot path fast.\n */\nexport function posthogAnalytics(config: PostHogAdapterConfig): AnalyticsAdapter {\n const hostRaw = config.host ?? 'https://us.i.posthog.com'\n const base = (/^https?:\\/\\//.test(hostRaw) ? hostRaw : `https://${hostRaw}`).replace(/\\/$/, '')\n const path = (config.path ?? '/i/v0/e/').replace(/^(?!\\/)/, '/')\n const endpoint = `${base}${path}`\n const fetchImpl = config.fetchImpl ?? fetch\n\n return {\n async capture(event: CaptureEvent): Promise<void> {\n // PostHog runs its own user-agent and GeoIP enrichment, but only off its\n // canonical property names. We already carry both values under our own\n // keys, so mirroring them costs nothing and unlocks a free second opinion:\n // getTrafficCategory(), getBotName() and friends all read\n // `properties.$raw_user_agent`, and GeoIP reads `properties.$ip`.\n //\n // Without this every event arrives with traffic category `no_user_agent`\n // and PostHog's whole bot taxonomy sits dormant — which is exactly what\n // happened on ours until someone queried it.\n //\n // `$ip` is mirrored only when the caller already opted into `captureIp`.\n // Adding it otherwise would put a raw address on the event that the\n // caller deliberately kept off.\n const ua = event.properties.user_agent\n const ip = event.properties.client_ip\n const payload = {\n api_key: config.apiKey,\n event: event.event,\n distinct_id: event.distinctId,\n timestamp: event.timestamp,\n properties: {\n ...event.properties,\n ...(typeof ua === 'string' && ua ? { $raw_user_agent: ua } : {}),\n ...(typeof ip === 'string' && ip ? { $ip: ip } : {})\n }\n }\n // A 401 from a mistyped key used to look identical to success. Surface\n // it: `trackVisit` routes it to `onError` and still never throws into\n // the response path.\n const res = await fetchImpl(endpoint, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(payload),\n keepalive: true,\n signal: AbortSignal.timeout(config.timeoutMs ?? 3000)\n })\n if (!res.ok) {\n throw new CaptureTransportError(\n `PostHog capture failed: ${res.status} ${res.statusText}`,\n res.status,\n await res.text().catch(() => undefined)\n )\n }\n }\n }\n}\n"]}
|
package/dist/adapters/posthog.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
var
|
|
2
|
-
export{
|
|
1
|
+
var o=class extends Error{status;body;constructor(s,i,p){super(s),this.name="CaptureTransportError",this.status=i,this.body=p;}};function y(t){let s=t.host??"https://us.i.posthog.com",i=(/^https?:\/\//.test(s)?s:`https://${s}`).replace(/\/$/,""),p=(t.path??"/i/v0/e/").replace(/^(?!\/)/,"/"),c=`${i}${p}`,u=t.fetchImpl??fetch;return {async capture(e){let a=e.properties.user_agent,n=e.properties.client_ip,d={api_key:t.apiKey,event:e.event,distinct_id:e.distinctId,timestamp:e.timestamp,properties:{...e.properties,...typeof a=="string"&&a?{$raw_user_agent:a}:{},...typeof n=="string"&&n?{$ip:n}:{}}},r=await u(c,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(d),keepalive:true,signal:AbortSignal.timeout(t.timeoutMs??3e3)});if(!r.ok)throw new o(`PostHog capture failed: ${r.status} ${r.statusText}`,r.status,await r.text().catch(()=>{}))}}}
|
|
2
|
+
export{y as posthogAnalytics};//# sourceMappingURL=posthog.js.map
|
|
3
3
|
//# sourceMappingURL=posthog.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/errors.ts","../../src/adapters/posthog.ts"],"names":["CaptureTransportError","message","status","body","posthogAnalytics","config","hostRaw","base","path","endpoint","fetchImpl","event","payload","res"],"mappings":"AACO,IAAMA,
|
|
1
|
+
{"version":3,"sources":["../../src/errors.ts","../../src/adapters/posthog.ts"],"names":["CaptureTransportError","message","status","body","posthogAnalytics","config","hostRaw","base","path","endpoint","fetchImpl","event","ua","ip","payload","res"],"mappings":"AACO,IAAMA,EAAN,cAAoC,KAAM,CACtC,MAAA,CACA,IAAA,CACT,YAAYC,CAAAA,CAAiBC,CAAAA,CAAiBC,CAAAA,CAAe,CAC3D,MAAMF,CAAO,CAAA,CACb,KAAK,IAAA,CAAO,uBAAA,CACZ,KAAK,MAAA,CAASC,CAAAA,CACd,IAAA,CAAK,IAAA,CAAOC,EACd,CACF,CAAA,CC2BO,SAASC,CAAAA,CAAiBC,CAAAA,CAAgD,CAC/E,IAAMC,CAAAA,CAAUD,EAAO,IAAA,EAAQ,0BAAA,CACzBE,GAAQ,cAAA,CAAe,IAAA,CAAKD,CAAO,CAAA,CAAIA,CAAAA,CAAU,WAAWA,CAAO,CAAA,CAAA,EAAI,OAAA,CAAQ,KAAA,CAAO,EAAE,CAAA,CACxFE,CAAAA,CAAAA,CAAQH,EAAO,IAAA,EAAQ,UAAA,EAAY,QAAQ,SAAA,CAAW,GAAG,CAAA,CACzDI,CAAAA,CAAW,GAAGF,CAAI,CAAA,EAAGC,CAAI,CAAA,CAAA,CACzBE,CAAAA,CAAYL,EAAO,SAAA,EAAa,KAAA,CAEtC,OAAO,CACL,MAAM,OAAA,CAAQM,CAAAA,CAAoC,CAchD,IAAMC,CAAAA,CAAKD,EAAM,UAAA,CAAW,UAAA,CACtBE,EAAKF,CAAAA,CAAM,UAAA,CAAW,UACtBG,CAAAA,CAAU,CACd,QAAST,CAAAA,CAAO,MAAA,CAChB,MAAOM,CAAAA,CAAM,KAAA,CACb,WAAA,CAAaA,CAAAA,CAAM,WACnB,SAAA,CAAWA,CAAAA,CAAM,UACjB,UAAA,CAAY,CACV,GAAGA,CAAAA,CAAM,UAAA,CACT,GAAI,OAAOC,GAAO,QAAA,EAAYA,CAAAA,CAAK,CAAE,eAAA,CAAiBA,CAAG,EAAI,EAAC,CAC9D,GAAI,OAAOC,GAAO,QAAA,EAAYA,CAAAA,CAAK,CAAE,GAAA,CAAKA,CAAG,EAAI,EACnD,CACF,CAAA,CAIME,CAAAA,CAAM,MAAML,CAAAA,CAAUD,CAAAA,CAAU,CACpC,MAAA,CAAQ,MAAA,CACR,QAAS,CAAE,cAAA,CAAgB,kBAAmB,CAAA,CAC9C,KAAM,IAAA,CAAK,SAAA,CAAUK,CAAO,CAAA,CAC5B,SAAA,CAAW,KACX,MAAA,CAAQ,WAAA,CAAY,QAAQT,CAAAA,CAAO,SAAA,EAAa,GAAI,CACtD,CAAC,EACD,GAAI,CAACU,EAAI,EAAA,CACP,MAAM,IAAIf,CAAAA,CACR,2BAA2Be,CAAAA,CAAI,MAAM,IAAIA,CAAAA,CAAI,UAAU,GACvDA,CAAAA,CAAI,MAAA,CACJ,MAAMA,CAAAA,CAAI,IAAA,GAAO,KAAA,CAAM,IAAG,EAAY,CACxC,CAEJ,CACF,CACF","file":"posthog.js","sourcesContent":["/** Thrown when the analytics backend rejects, errors, or times out a capture. */\nexport class CaptureTransportError extends Error {\n readonly status: number | undefined\n readonly body: string | undefined\n constructor(message: string, status?: number, body?: string) {\n super(message)\n this.name = 'CaptureTransportError'\n this.status = status\n this.body = body\n }\n}\n","import type { AnalyticsAdapter, CaptureEvent } from '../types.js'\nimport { CaptureTransportError } from '../errors.js'\n\n\nexport interface PostHogAdapterConfig {\n /** PostHog project API key (the public one used by the JS SDK). */\n apiKey: string\n /**\n * PostHog host, with or without scheme. Defaults to `https://us.i.posthog.com`.\n * Use `https://eu.i.posthog.com` for EU cloud, or your own reverse-proxy\n * domain (e.g. `https://svc.example.com`).\n */\n host?: string\n /**\n * Path on the host that accepts single-event captures. Defaults to\n * `/i/v0/e/` which is PostHog's current endpoint for this.\n */\n path?: string\n /**\n * Override the `fetch` implementation (useful for tests or custom runtimes\n * that need a pinned fetch).\n */\n fetchImpl?: typeof fetch\n /**\n * Abort the capture after this many milliseconds. Defaults to 3000. Without\n * a bound, a hung backend leaves a pending promise for the lifetime of an\n * edge invocation.\n */\n timeoutMs?: number\n}\n\n/**\n * Adapter that posts each event to the PostHog capture endpoint. Uses\n * `keepalive: true` so the request survives after a serverless response\n * returns — events aren't guaranteed (fire-and-forget), but that's the\n * trade we want to keep the hot path fast.\n */\nexport function posthogAnalytics(config: PostHogAdapterConfig): AnalyticsAdapter {\n const hostRaw = config.host ?? 'https://us.i.posthog.com'\n const base = (/^https?:\\/\\//.test(hostRaw) ? hostRaw : `https://${hostRaw}`).replace(/\\/$/, '')\n const path = (config.path ?? '/i/v0/e/').replace(/^(?!\\/)/, '/')\n const endpoint = `${base}${path}`\n const fetchImpl = config.fetchImpl ?? fetch\n\n return {\n async capture(event: CaptureEvent): Promise<void> {\n // PostHog runs its own user-agent and GeoIP enrichment, but only off its\n // canonical property names. We already carry both values under our own\n // keys, so mirroring them costs nothing and unlocks a free second opinion:\n // getTrafficCategory(), getBotName() and friends all read\n // `properties.$raw_user_agent`, and GeoIP reads `properties.$ip`.\n //\n // Without this every event arrives with traffic category `no_user_agent`\n // and PostHog's whole bot taxonomy sits dormant — which is exactly what\n // happened on ours until someone queried it.\n //\n // `$ip` is mirrored only when the caller already opted into `captureIp`.\n // Adding it otherwise would put a raw address on the event that the\n // caller deliberately kept off.\n const ua = event.properties.user_agent\n const ip = event.properties.client_ip\n const payload = {\n api_key: config.apiKey,\n event: event.event,\n distinct_id: event.distinctId,\n timestamp: event.timestamp,\n properties: {\n ...event.properties,\n ...(typeof ua === 'string' && ua ? { $raw_user_agent: ua } : {}),\n ...(typeof ip === 'string' && ip ? { $ip: ip } : {})\n }\n }\n // A 401 from a mistyped key used to look identical to success. Surface\n // it: `trackVisit` routes it to `onError` and still never throws into\n // the response path.\n const res = await fetchImpl(endpoint, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(payload),\n keepalive: true,\n signal: AbortSignal.timeout(config.timeoutMs ?? 3000)\n })\n if (!res.ok) {\n throw new CaptureTransportError(\n `PostHog capture failed: ${res.status} ${res.statusText}`,\n res.status,\n await res.text().catch(() => undefined)\n )\n }\n }\n }\n}\n"]}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
'use strict';function f(i){return `'${JSON.stringify(i).replace(/'/g,"'\\''")}'`}function v(i){let e=[`vercel firewall rules add ${JSON.stringify(i.name)}`];if(i.groups.forEach((n,o)=>{o>0&&e.push(" --or");for(let l of n)e.push(` --condition ${f(l)}`);}),e.push(` --action ${i.action}`),i.action==="rate_limit"&&i.rateLimit){e.push(` --rate-limit-window ${i.rateLimit.window}`),e.push(` --rate-limit-requests ${i.rateLimit.requests}`),e.push(` --rate-limit-action ${i.rateLimit.action}`);for(let n of i.rateLimit.keys)e.push(` --rate-limit-keys ${n}`);}return e.push(" --yes"),e.join(` \\
|
|
2
|
+
`)}function w(i){return {name:i.name,conditionGroup:i.groups.map(e=>({conditions:e})),action:{mitigate:{action:i.action}}}}function u(i){return {...i,cli:v(i),json:w(i)}}function y(i){if(!i.length)return 0;let e=[...i].sort((o,l)=>o-l),n=Math.floor(e.length/2);return e.length%2?e[n]:(e[n-1]+e[n])/2}function b(i,e={}){let n=[];if(!e.omitProtectiveBypass){let t=i.filter(a=>a.intent==="retrieval"||a.intent==="search"),r=t.reduce((a,g)=>a+g.requests,0),s=[...new Set(t.map(a=>a.botName))];n.push(u({name:"Allow retrieval and search agents",rationale:"Retrieval agents and search crawlers must never be caught by the rules below \u2014 they bring readers and rankings.",evidence:r?`${r.toLocaleString("en-US")} observed requests across ${s.length} vendors (${s.slice(0,6).join(", ")})`:"no retrieval or search traffic observed yet; installed pre-emptively",groups:[[{type:"user_agent",op:"inc",value:["ChatGPT-User","OAI-SearchBot","Claude-User","Claude-SearchBot","Perplexity-User","Googlebot","bingbot","DuckDuckBot","Applebot"]}]],action:"bypass",eventual:"bypass",risk:"low",caveat:"Place this rule first (`vercel firewall rules reorder ... --first`). A user-agent allowlist is spoofable, so pair with verification in middleware rather than relying on it for security \u2014 its job here is to stop your own rules misfiring."}));}let o=i.filter(t=>t.verification==="spoofed"),l=[...new Set(o.map(t=>t.ip).filter(t=>!!t))];if(l.length){let t=o.reduce((s,a)=>s+a.requests,0),r=[...new Set(o.map(s=>s.botName))];n.push(u({name:"Deny impersonated crawler identities",rationale:"These addresses claimed a crawler identity that failed verification against the vendor\u2019s published ranges or signature.",evidence:`${t.toLocaleString("en-US")} requests from ${l.length} address${l.length===1?"":"es"} impersonating ${r.join(", ")}`,groups:[[{type:"ip_address",op:"inc",value:l}]],action:"log",eventual:"deny",risk:"low",caveat:"Verification failure is strong evidence, but confirm your edge controls x-forwarded-for before enforcing \u2014 behind a proxy that forwards a client-supplied header the verdict is worthless."}));}let h=i.filter(t=>t.ip&&t.verification!=="verified"),p=e.abuseThreshold??Math.max(500,Math.round(y(h.map(t=>t.requests))*10)),c=h.filter(t=>t.requests>=p).sort((t,r)=>r.requests-t.requests).slice(0,50);if(c.length){let t=c.filter(r=>(r.distinctPaths??0)>100);n.push(u({name:"Rate limit high-volume unverified addresses",rationale:"A single address making orders of magnitude more requests than the median, with no verified identity.",evidence:`${c.length} address${c.length===1?"":"es"} above ${p.toLocaleString("en-US")} requests`+(t.length?`; ${t.length} swept >100 distinct paths, which reads as a scrape rather than a reader`:""),groups:[[{type:"ip_address",op:"inc",value:c.map(r=>r.ip)}]],action:"log",eventual:"rate_limit",rateLimit:{window:60,requests:60,action:"rate_limit",keys:["ip"]},risk:"medium",caveat:"Shared egress means one address can front many real users \u2014 a corporate NAT, a mobile carrier, or a VPN. Review the dashboard before enforcing."}));}let m=i.filter(t=>t.intent==="training");if(m.length){let t=m.reduce((a,g)=>a+g.requests,0),r=[...new Set(m.map(a=>a.botName))],s=e.trainingBudget??{window:3600,requests:600};n.push(u({name:"Rate limit training crawlers",rationale:"Bound what bulk corpus collection costs you without removing yourself from training sets.",evidence:`${t.toLocaleString("en-US")} training requests from ${r.length} vendors (${r.slice(0,6).join(", ")})`,groups:[[{type:"user_agent",op:"inc",value:["GPTBot","ClaudeBot","CCBot","Bytespider","Amazonbot","meta-externalagent"]}]],action:"log",eventual:"rate_limit",rateLimit:{window:s.window,requests:s.requests,action:"rate_limit",keys:["ip"]},risk:"medium",caveat:"Denying these removes you from future training sets, which may be exactly wrong for discoverability. Rate limit rather than deny unless you have decided otherwise. Note Vercel counters are per region, so N regions can collectively exceed the limit by ~Nx."}));}let d=[...new Set(i.filter(t=>t.asn!==void 0&&/Mozilla|Chrome|Safari/i.test(t.userAgent)).filter(t=>t.verification!=="verified").map(t=>t.asn))];if(d.length){let r=i.filter(s=>s.asn!==void 0&&d.includes(s.asn)).reduce((s,a)=>s+a.requests,0);n.push(u({name:"Challenge browser user agents from datacenter networks",rationale:"A browser user agent arriving from a hosting network is automation wearing a costume \u2014 real browsers come from consumer ISPs.",evidence:`${r.toLocaleString("en-US")} requests across ${d.length} datacenter AS numbers`,groups:[[{type:"geo_as_number",op:"inc",value:d},{type:"user_agent",op:"sub",value:"Mozilla"}]],action:"log",eventual:"challenge",risk:"high",caveat:"Highest false-positive risk here. Corporate VPNs, privacy relays and some mobile carriers egress from hosting ASNs, and a challenge page breaks API clients and link unfurlers outright. Keep this in log mode for a full week before considering enforcement."}));}return n}function $(i){let e=["#!/usr/bin/env bash","# Vercel WAF proposals generated from observed agent traffic.","#","# Every rule starts in LOG mode and blocks nothing. Vercel stages rule","# changes as drafts, so nothing is live until you run:","#","# vercel firewall diff # review","# vercel firewall publish --yes # go live","#","# Review each rule in the dashboard before promoting it to its eventual","# action. Rules evaluate top to bottom, so keep the bypass rule first.","set -euo pipefail",""];return i.forEach((n,o)=>{e.push(`# ${o+1}. ${n.name}`),e.push(`# why: ${n.rationale}`),e.push(`# evidence: ${n.evidence}`),e.push(`# risk: ${n.risk} \u2014 eventual action: ${n.eventual}`),n.caveat&&e.push(`# caveat: ${n.caveat}`),e.push(n.cli),e.push("");}),i.length&&(e.push("# Keep the protective allow rule at the top of the evaluation order."),e.push(`vercel firewall rules reorder ${JSON.stringify(i[0].name)} --first --yes`),e.push(""),e.push("vercel firewall diff"),e.push('echo "Review above, then: vercel firewall publish --yes"')),e.join(`
|
|
3
|
+
`)}exports.firewallScript=$;exports.recommendFirewallRules=b;//# sourceMappingURL=firewall.cjs.map
|
|
4
|
+
//# sourceMappingURL=firewall.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/firewall.ts"],"names":["shellQuote","json","toCli","r","parts","group","i","c","k","toJson","conditions","finish","median","ns","s","a","b","mid","recommendFirewallRules","observations","opts","out","wanted","o","requests","n","names","spoofed","spoofedIps","v","vendors","perIp","threshold","heavy","sweeping","training","budget","headlessAsns","firewallScript","recommendations","lines"],"mappings":"aA6GA,SAASA,CAAAA,CAAWC,CAAAA,CAAuB,CACzC,OAAO,CAAA,CAAA,EAAI,KAAK,SAAA,CAAUA,CAAI,CAAA,CAAE,OAAA,CAAQ,IAAA,CAAM,OAAO,CAAC,CAAA,CAAA,CACxD,CAEA,SAASC,CAAAA,CAAMC,CAAAA,CAAyD,CACtE,IAAMC,CAAAA,CAAQ,CAAC,CAAA,0BAAA,EAA6B,IAAA,CAAK,SAAA,CAAUD,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAE,CAAA,CAMpE,GALAA,CAAAA,CAAE,MAAA,CAAO,OAAA,CAAQ,CAACE,CAAAA,CAAOC,CAAAA,GAAM,CACzBA,CAAAA,CAAI,CAAA,EAAGF,CAAAA,CAAM,KAAK,QAAQ,CAAA,CAC9B,IAAA,IAAWG,CAAAA,IAAKF,CAAAA,CAAOD,CAAAA,CAAM,IAAA,CAAK,CAAA,cAAA,EAAiBJ,CAAAA,CAAWO,CAAC,CAAC,CAAA,CAAE,EACpE,CAAC,EACDH,CAAAA,CAAM,IAAA,CAAK,CAAA,WAAA,EAAcD,CAAAA,CAAE,MAAM,CAAA,CAAE,EAC/BA,CAAAA,CAAE,MAAA,GAAW,YAAA,EAAgBA,CAAAA,CAAE,SAAA,CAAW,CAC5CC,EAAM,IAAA,CAAK,CAAA,sBAAA,EAAyBD,CAAAA,CAAE,SAAA,CAAU,MAAM,CAAA,CAAE,CAAA,CACxDC,CAAAA,CAAM,IAAA,CAAK,CAAA,wBAAA,EAA2BD,CAAAA,CAAE,SAAA,CAAU,QAAQ,CAAA,CAAE,EAC5DC,CAAAA,CAAM,IAAA,CAAK,CAAA,sBAAA,EAAyBD,CAAAA,CAAE,SAAA,CAAU,MAAM,EAAE,CAAA,CACxD,IAAA,IAAWK,CAAAA,IAAKL,CAAAA,CAAE,SAAA,CAAU,IAAA,CAAMC,EAAM,IAAA,CAAK,CAAA,oBAAA,EAAuBI,CAAC,CAAA,CAAE,EACzE,CACA,OAAAJ,CAAAA,CAAM,IAAA,CAAK,SAAS,CAAA,CACbA,CAAAA,CAAM,IAAA,CAAK,CAAA;AAAA,CAAO,CAC3B,CAEA,SAASK,CAAAA,CAAON,EAA0D,CACxE,OAAO,CACL,IAAA,CAAMA,CAAAA,CAAE,KACR,cAAA,CAAgBA,CAAAA,CAAE,OAAO,GAAA,CAAKO,CAAAA,GAAgB,CAAE,UAAA,CAAAA,CAAW,CAAA,CAAE,CAAA,CAC7D,MAAA,CAAQ,CAAE,SAAU,CAAE,MAAA,CAAQP,EAAE,MAAO,CAAE,CAC3C,CACF,CAEA,SAASQ,CAAAA,CAAOR,CAAAA,CAAyE,CACvF,OAAO,CAAE,GAAGA,CAAAA,CAAG,GAAA,CAAKD,EAAMC,CAAC,CAAA,CAAG,IAAA,CAAMM,CAAAA,CAAON,CAAC,CAAE,CAChD,CAEA,SAASS,EAAOC,CAAAA,CAAsB,CACpC,GAAI,CAACA,CAAAA,CAAG,OAAQ,OAAO,CAAA,CACvB,IAAMC,CAAAA,CAAI,CAAC,GAAGD,CAAE,CAAA,CAAE,KAAK,CAACE,CAAAA,CAAGC,CAAAA,GAAMD,CAAAA,CAAIC,CAAC,CAAA,CAChCC,EAAM,IAAA,CAAK,KAAA,CAAMH,EAAE,MAAA,CAAS,CAAC,EACnC,OAAOA,CAAAA,CAAE,MAAA,CAAS,CAAA,CAAIA,CAAAA,CAAEG,CAAG,GAAMH,CAAAA,CAAEG,CAAAA,CAAM,CAAC,CAAA,CAAKH,CAAAA,CAAEG,CAAG,CAAA,EAAM,CAC5D,CAiBO,SAASC,CAAAA,CACdC,CAAAA,CACAC,EAAyB,EAAC,CACA,CAC1B,IAAMC,CAAAA,CAAgC,EAAC,CAKvC,GAAI,CAACD,CAAAA,CAAK,oBAAA,CAAsB,CAC9B,IAAME,CAAAA,CAASH,EAAa,MAAA,CAAQI,CAAAA,EAAMA,EAAE,MAAA,GAAW,WAAA,EAAeA,CAAAA,CAAE,MAAA,GAAW,QAAQ,CAAA,CACrFC,EAAWF,CAAAA,CAAO,MAAA,CAAO,CAACG,CAAAA,CAAGF,CAAAA,GAAME,EAAIF,CAAAA,CAAE,QAAA,CAAU,CAAC,CAAA,CACpDG,CAAAA,CAAQ,CAAC,GAAG,IAAI,GAAA,CAAIJ,EAAO,GAAA,CAAKC,CAAAA,EAAMA,EAAE,OAAO,CAAC,CAAC,CAAA,CACvDF,CAAAA,CAAI,IAAA,CACFV,EAAO,CACL,IAAA,CAAM,oCACN,SAAA,CACE,sHAAA,CACF,SAAUa,CAAAA,CACN,CAAA,EAAGA,EAAS,cAAA,CAAe,OAAO,CAAC,CAAA,0BAAA,EAA6BE,CAAAA,CAAM,MAAM,CAAA,UAAA,EAAaA,CAAAA,CAAM,MAAM,CAAA,CAAG,CAAC,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,IACrH,sEAAA,CACJ,MAAA,CAAQ,CACN,CACE,CACE,KAAM,YAAA,CACN,EAAA,CAAI,KAAA,CACJ,KAAA,CAAO,CACL,cAAA,CACA,gBACA,aAAA,CACA,kBAAA,CACA,kBACA,WAAA,CACA,SAAA,CACA,cACA,UACF,CACF,CACF,CACF,CAAA,CACA,MAAA,CAAQ,SACR,QAAA,CAAU,QAAA,CACV,KAAM,KAAA,CACN,MAAA,CACE,mPACJ,CAAC,CACH,EACF,CAGA,IAAMC,EAAUR,CAAAA,CAAa,MAAA,CAAQI,GAAMA,CAAAA,CAAE,YAAA,GAAiB,SAAS,CAAA,CACjEK,CAAAA,CAAa,CAAC,GAAG,IAAI,GAAA,CAAID,EAAQ,GAAA,CAAKJ,CAAAA,EAAMA,EAAE,EAAE,CAAA,CAAE,OAAQM,CAAAA,EAAmB,CAAC,CAACA,CAAC,CAAC,CAAC,CAAA,CACxF,GAAID,EAAW,MAAA,CAAQ,CACrB,IAAMJ,CAAAA,CAAWG,CAAAA,CAAQ,MAAA,CAAO,CAACF,CAAAA,CAAGF,CAAAA,GAAME,EAAIF,CAAAA,CAAE,QAAA,CAAU,CAAC,CAAA,CACrDO,CAAAA,CAAU,CAAC,GAAG,IAAI,IAAIH,CAAAA,CAAQ,GAAA,CAAKJ,GAAMA,CAAAA,CAAE,OAAO,CAAC,CAAC,CAAA,CAC1DF,EAAI,IAAA,CACFV,CAAAA,CAAO,CACL,IAAA,CAAM,sCAAA,CACN,SAAA,CACE,+HACF,QAAA,CAAU,CAAA,EAAGa,EAAS,cAAA,CAAe,OAAO,CAAC,CAAA,eAAA,EAAkBI,CAAAA,CAAW,MAAM,CAAA,QAAA,EAAWA,CAAAA,CAAW,MAAA,GAAW,EAAI,EAAA,CAAK,IAAI,kBAAkBE,CAAAA,CAAQ,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAClK,MAAA,CAAQ,CAAC,CAAC,CAAE,KAAM,YAAA,CAAc,EAAA,CAAI,MAAO,KAAA,CAAOF,CAAW,CAAC,CAAC,CAAA,CAC/D,OAAQ,KAAA,CACR,QAAA,CAAU,OACV,IAAA,CAAM,KAAA,CACN,OACE,iMACJ,CAAC,CACH,EACF,CAGA,IAAMG,CAAAA,CAAQZ,CAAAA,CAAa,MAAA,CAAQI,GAAMA,CAAAA,CAAE,EAAA,EAAMA,EAAE,YAAA,GAAiB,UAAU,EACxES,CAAAA,CACJZ,CAAAA,CAAK,cAAA,EAAkB,IAAA,CAAK,GAAA,CAAI,GAAA,CAAK,KAAK,KAAA,CAAMR,CAAAA,CAAOmB,EAAM,GAAA,CAAKR,CAAAA,EAAMA,EAAE,QAAQ,CAAC,CAAA,CAAI,EAAE,CAAC,CAAA,CACtFU,EAAQF,CAAAA,CACX,MAAA,CAAQR,GAAMA,CAAAA,CAAE,QAAA,EAAYS,CAAS,CAAA,CACrC,IAAA,CAAK,CAACjB,CAAAA,CAAGC,CAAAA,GAAMA,EAAE,QAAA,CAAWD,CAAAA,CAAE,QAAQ,CAAA,CACtC,KAAA,CAAM,EAAG,EAAE,CAAA,CACd,GAAIkB,CAAAA,CAAM,MAAA,CAAQ,CAChB,IAAMC,CAAAA,CAAWD,CAAAA,CAAM,OAAQV,CAAAA,EAAAA,CAAOA,CAAAA,CAAE,eAAiB,CAAA,EAAK,GAAG,CAAA,CACjEF,CAAAA,CAAI,IAAA,CACFV,CAAAA,CAAO,CACL,IAAA,CAAM,6CAAA,CACN,UACE,uGAAA,CACF,QAAA,CACE,GAAGsB,CAAAA,CAAM,MAAM,CAAA,QAAA,EAAWA,CAAAA,CAAM,MAAA,GAAW,CAAA,CAAI,GAAK,IAAI,CAAA,OAAA,EAAUD,EAAU,cAAA,CAAe,OAAO,CAAC,CAAA,SAAA,CAAA,EAClGE,CAAAA,CAAS,OACN,CAAA,EAAA,EAAKA,CAAAA,CAAS,MAAM,CAAA,wEAAA,CAAA,CACpB,EAAA,CAAA,CACN,OAAQ,CAAC,CAAC,CAAE,IAAA,CAAM,YAAA,CAAc,EAAA,CAAI,KAAA,CAAO,KAAA,CAAOD,CAAAA,CAAM,IAAKV,CAAAA,EAAMA,CAAAA,CAAE,EAAG,CAAE,CAAC,CAAC,CAAA,CAC5E,MAAA,CAAQ,MACR,QAAA,CAAU,YAAA,CACV,UAAW,CAAE,MAAA,CAAQ,GAAI,QAAA,CAAU,EAAA,CAAI,OAAQ,YAAA,CAAc,IAAA,CAAM,CAAC,IAAI,CAAE,CAAA,CAC1E,KAAM,QAAA,CACN,MAAA,CACE,sJACJ,CAAC,CACH,EACF,CAGA,IAAMY,EAAWhB,CAAAA,CAAa,MAAA,CAAQI,GAAMA,CAAAA,CAAE,MAAA,GAAW,UAAU,CAAA,CACnE,GAAIY,EAAS,MAAA,CAAQ,CACnB,IAAMX,CAAAA,CAAWW,CAAAA,CAAS,MAAA,CAAO,CAACV,CAAAA,CAAGF,CAAAA,GAAME,EAAIF,CAAAA,CAAE,QAAA,CAAU,CAAC,CAAA,CACtDO,CAAAA,CAAU,CAAC,GAAG,IAAI,GAAA,CAAIK,EAAS,GAAA,CAAKZ,CAAAA,EAAMA,EAAE,OAAO,CAAC,CAAC,CAAA,CACrDa,CAAAA,CAAShB,CAAAA,CAAK,cAAA,EAAkB,CAAE,MAAA,CAAQ,KAAM,QAAA,CAAU,GAAI,EACpEC,CAAAA,CAAI,IAAA,CACFV,EAAO,CACL,IAAA,CAAM,+BACN,SAAA,CACE,2FAAA,CACF,SAAU,CAAA,EAAGa,CAAAA,CAAS,eAAe,OAAO,CAAC,2BAA2BM,CAAAA,CAAQ,MAAM,CAAA,UAAA,EAAaA,CAAAA,CAAQ,KAAA,CAAM,CAAA,CAAG,CAAC,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,CACjI,OAAQ,CACN,CACE,CACE,IAAA,CAAM,YAAA,CACN,EAAA,CAAI,MACJ,KAAA,CAAO,CAAC,SAAU,WAAA,CAAa,OAAA,CAAS,aAAc,WAAA,CAAa,oBAAoB,CACzF,CACF,CACF,CAAA,CACA,OAAQ,KAAA,CACR,QAAA,CAAU,aACV,SAAA,CAAW,CAAE,OAAQM,CAAAA,CAAO,MAAA,CAAQ,SAAUA,CAAAA,CAAO,QAAA,CAAU,OAAQ,YAAA,CAAc,IAAA,CAAM,CAAC,IAAI,CAAE,EAClG,IAAA,CAAM,QAAA,CACN,MAAA,CACE,iQACJ,CAAC,CACH,EACF,CAGA,IAAMC,EAAe,CACnB,GAAG,IAAI,GAAA,CACLlB,CAAAA,CACG,MAAA,CAAQI,CAAAA,EAAMA,CAAAA,CAAE,GAAA,GAAQ,QAAa,wBAAA,CAAyB,IAAA,CAAKA,EAAE,SAAS,CAAC,EAC/E,MAAA,CAAQA,CAAAA,EAAMA,CAAAA,CAAE,YAAA,GAAiB,UAAU,CAAA,CAC3C,IAAKA,CAAAA,EAAMA,CAAAA,CAAE,GAAI,CACtB,CACF,EACA,GAAIc,CAAAA,CAAa,OAAQ,CAEvB,IAAMb,EADSL,CAAAA,CAAa,MAAA,CAAQI,GAAMA,CAAAA,CAAE,GAAA,GAAQ,QAAac,CAAAA,CAAa,QAAA,CAASd,CAAAA,CAAE,GAAG,CAAC,CAAA,CACrE,OAAO,CAACE,CAAAA,CAAGF,IAAME,CAAAA,CAAIF,CAAAA,CAAE,SAAU,CAAC,CAAA,CAC1DF,EAAI,IAAA,CACFV,CAAAA,CAAO,CACL,IAAA,CAAM,wDAAA,CACN,UACE,oIAAA,CACF,QAAA,CAAU,GAAGa,CAAAA,CAAS,cAAA,CAAe,OAAO,CAAC,CAAA,iBAAA,EAAoBa,CAAAA,CAAa,MAAM,CAAA,sBAAA,CAAA,CACpF,MAAA,CAAQ,CACN,CACE,CAAE,KAAM,eAAA,CAAiB,EAAA,CAAI,MAAO,KAAA,CAAOA,CAAa,EACxD,CAAE,IAAA,CAAM,aAAc,EAAA,CAAI,KAAA,CAAO,MAAO,SAAU,CACpD,CACF,CAAA,CACA,MAAA,CAAQ,KAAA,CACR,SAAU,WAAA,CACV,IAAA,CAAM,OACN,MAAA,CACE,gQACJ,CAAC,CACH,EACF,CAEA,OAAOhB,CACT,CAGO,SAASiB,CAAAA,CAAeC,CAAAA,CAA4D,CACzF,IAAMC,CAAAA,CAAQ,CACZ,qBAAA,CACA,+DAAA,CACA,GAAA,CACA,wEAAA,CACA,wDAAA,CACA,GAAA,CACA,+CACA,+CAAA,CACA,GAAA,CACA,0EACA,wEAAA,CACA,mBAAA,CACA,EACF,CAAA,CACA,OAAAD,EAAgB,OAAA,CAAQ,CAACpC,EAAGG,CAAAA,GAAM,CAChCkC,EAAM,IAAA,CAAK,CAAA,EAAA,EAAKlC,EAAI,CAAC,CAAA,EAAA,EAAKH,CAAAA,CAAE,IAAI,CAAA,CAAE,CAAA,CAClCqC,EAAM,IAAA,CAAK,CAAA,eAAA,EAAkBrC,EAAE,SAAS,CAAA,CAAE,EAC1CqC,CAAAA,CAAM,IAAA,CAAK,CAAA,eAAA,EAAkBrC,CAAAA,CAAE,QAAQ,CAAA,CAAE,EACzCqC,CAAAA,CAAM,IAAA,CAAK,kBAAkBrC,CAAAA,CAAE,IAAI,4BAAuBA,CAAAA,CAAE,QAAQ,CAAA,CAAE,CAAA,CAClEA,CAAAA,CAAE,MAAA,EAAQqC,EAAM,IAAA,CAAK,CAAA,eAAA,EAAkBrC,EAAE,MAAM,CAAA,CAAE,EACrDqC,CAAAA,CAAM,IAAA,CAAKrC,EAAE,GAAG,CAAA,CAChBqC,EAAM,IAAA,CAAK,EAAE,EACf,CAAC,CAAA,CACGD,EAAgB,MAAA,GAClBC,CAAAA,CAAM,IAAA,CAAK,sEAAsE,CAAA,CACjFA,CAAAA,CAAM,KACJ,CAAA,8BAAA,EAAiC,IAAA,CAAK,UAAUD,CAAAA,CAAgB,CAAC,EAAG,IAAI,CAAC,CAAA,cAAA,CAC3E,CAAA,CACAC,CAAAA,CAAM,IAAA,CAAK,EAAE,CAAA,CACbA,CAAAA,CAAM,KAAK,sBAAsB,CAAA,CACjCA,EAAM,IAAA,CAAK,0DAA0D,CAAA,CAAA,CAEhEA,CAAAA,CAAM,IAAA,CAAK;AAAA,CAAI,CACxB","file":"firewall.cjs","sourcesContent":["/**\n * Recommend Vercel WAF rules from observed agent traffic.\n *\n * This generates *proposals*, never live changes. Every recommendation comes out\n * with `action: 'log'`, because a firewall rule's blast radius is unpredictable\n * until real traffic hits it and a bad `deny` takes out real users or your SEO.\n * Vercel's own guidance is log → review → preview → production; the `eventual`\n * field records where a rule is meant to end up, and `cli` emits the command for\n * the *current* stage only.\n *\n * Two hard rules, both from measurement rather than taste:\n *\n * 1. Retrieval agents and search crawlers are never proposed for blocking.\n * 60% of AI traffic on one production site is retrieval — a person asked a\n * question and an assistant went to read the page. Blocking that is\n * blocking your own distribution. The recommender emits a `bypass` rule to\n * protect them *first*, so later rules cannot catch them.\n *\n * 2. Training crawlers get rate limits, not denials, by default. The point is\n * to bound cost, not to disappear from corpora.\n *\n * Only abuse gets a denial: an identity that failed cryptographic or IP\n * verification, or a single address behaving like a scraper.\n */\n\nimport type { AgentIntent } from './policy.js'\n\n/** A Vercel WAF condition. Mirrors the CLI's `--condition` JSON. */\nexport interface FirewallCondition {\n type:\n | 'user_agent'\n | 'ip_address'\n | 'geo_as_number'\n | 'geo_country'\n | 'path'\n | 'method'\n | 'environment'\n | 'ja4_digest'\n op: 'eq' | 'neq' | 'sub' | 'pre' | 'suf' | 're' | 'inc' | 'ninc' | 'gt' | 'gte'\n value?: string | number | Array<string | number>\n key?: string\n neg?: boolean\n}\n\nexport type FirewallAction = 'log' | 'deny' | 'challenge' | 'bypass' | 'rate_limit'\n\nexport interface RateLimitSpec {\n /** Seconds, 10–3600. */\n window: number\n /** Max requests per window. */\n requests: number\n /** What happens on breach. */\n action: 'rate_limit' | 'deny' | 'challenge' | 'log'\n keys: Array<'ip' | 'ja4'>\n}\n\nexport interface FirewallRecommendation {\n name: string\n /** Why this rule is proposed, in one sentence. */\n rationale: string\n /** The measurement behind it. Never propose a rule without evidence. */\n evidence: string\n /** OR of ANDs: outer array is groups, inner is conditions within a group. */\n groups: FirewallCondition[][]\n /** Always `'log'` or `'bypass'` — see the module note. */\n action: FirewallAction\n /** Where this rule is intended to end up after review. */\n eventual: FirewallAction\n rateLimit?: RateLimitSpec\n /** How likely this is to catch traffic you wanted. */\n risk: 'low' | 'medium' | 'high'\n /** What could go wrong, when it is not obvious. */\n caveat?: string\n /** Ready-to-run CLI for the *current* stage. */\n cli: string\n /** Equivalent `--json` payload. */\n json: unknown\n}\n\n/** One aggregated slice of observed traffic. */\nexport interface TrafficObservation {\n userAgent: string\n botName: string\n intent: AgentIntent\n requests: number\n ip?: string\n /** Autonomous system number, if you resolved one. */\n asn?: number\n /** Distinct paths this slice touched — a scraper sweeps, a reader does not. */\n distinctPaths?: number\n /** Verification verdict, if you ran one. */\n verification?: 'verified' | 'spoofed' | 'unverifiable' | 'not-claimed'\n country?: string\n}\n\nexport interface RecommendOptions {\n /**\n * Requests-per-slice above which a single IP is considered abusive. Defaults\n * to 10x the median across observations, floored at 500.\n */\n abuseThreshold?: number\n /** Rate-limit budget proposed for training crawlers. Defaults to 600/hour. */\n trainingBudget?: { window: number; requests: number }\n /** Skip the protective bypass rule. Rarely a good idea. */\n omitProtectiveBypass?: boolean\n}\n\n/* -------------------------------------------------------------------------- */\n\nfunction shellQuote(json: unknown): string {\n return `'${JSON.stringify(json).replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction toCli(r: Omit<FirewallRecommendation, 'cli' | 'json'>): string {\n const parts = [`vercel firewall rules add ${JSON.stringify(r.name)}`]\n r.groups.forEach((group, i) => {\n if (i > 0) parts.push(' --or')\n for (const c of group) parts.push(` --condition ${shellQuote(c)}`)\n })\n parts.push(` --action ${r.action}`)\n if (r.action === 'rate_limit' && r.rateLimit) {\n parts.push(` --rate-limit-window ${r.rateLimit.window}`)\n parts.push(` --rate-limit-requests ${r.rateLimit.requests}`)\n parts.push(` --rate-limit-action ${r.rateLimit.action}`)\n for (const k of r.rateLimit.keys) parts.push(` --rate-limit-keys ${k}`)\n }\n parts.push(' --yes')\n return parts.join(' \\\\\\n')\n}\n\nfunction toJson(r: Omit<FirewallRecommendation, 'cli' | 'json'>): unknown {\n return {\n name: r.name,\n conditionGroup: r.groups.map((conditions) => ({ conditions })),\n action: { mitigate: { action: r.action } }\n }\n}\n\nfunction finish(r: Omit<FirewallRecommendation, 'cli' | 'json'>): FirewallRecommendation {\n return { ...r, cli: toCli(r), json: toJson(r) }\n}\n\nfunction median(ns: number[]): number {\n if (!ns.length) return 0\n const s = [...ns].sort((a, b) => a - b)\n const mid = Math.floor(s.length / 2)\n return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2\n}\n\n/* -------------------------------------------------------------------------- */\n\n/**\n * Turn observations into staged WAF proposals.\n *\n * @example\n * ```ts\n * const rules = recommendFirewallRules(observations)\n * for (const r of rules) {\n * console.log(`# ${r.name} — ${r.rationale}`)\n * console.log(`# evidence: ${r.evidence}`)\n * console.log(r.cli)\n * }\n * ```\n */\nexport function recommendFirewallRules(\n observations: readonly TrafficObservation[],\n opts: RecommendOptions = {}\n): FirewallRecommendation[] {\n const out: FirewallRecommendation[] = []\n\n /* 1. Protect the traffic you want, first and above everything else. --------\n Rules are evaluated top to bottom, so this has to be rule #1 or a later\n user-agent rule will swallow the agents that bring you readers. */\n if (!opts.omitProtectiveBypass) {\n const wanted = observations.filter((o) => o.intent === 'retrieval' || o.intent === 'search')\n const requests = wanted.reduce((n, o) => n + o.requests, 0)\n const names = [...new Set(wanted.map((o) => o.botName))]\n out.push(\n finish({\n name: 'Allow retrieval and search agents',\n rationale:\n 'Retrieval agents and search crawlers must never be caught by the rules below — they bring readers and rankings.',\n evidence: requests\n ? `${requests.toLocaleString('en-US')} observed requests across ${names.length} vendors (${names.slice(0, 6).join(', ')})`\n : 'no retrieval or search traffic observed yet; installed pre-emptively',\n groups: [\n [\n {\n type: 'user_agent',\n op: 'inc',\n value: [\n 'ChatGPT-User',\n 'OAI-SearchBot',\n 'Claude-User',\n 'Claude-SearchBot',\n 'Perplexity-User',\n 'Googlebot',\n 'bingbot',\n 'DuckDuckBot',\n 'Applebot'\n ]\n }\n ]\n ],\n action: 'bypass',\n eventual: 'bypass',\n risk: 'low',\n caveat:\n 'Place this rule first (`vercel firewall rules reorder ... --first`). A user-agent allowlist is spoofable, so pair with verification in middleware rather than relying on it for security — its job here is to stop your own rules misfiring.'\n })\n )\n }\n\n /* 2. Failed verification — the only class that earns a denial. ------------- */\n const spoofed = observations.filter((o) => o.verification === 'spoofed')\n const spoofedIps = [...new Set(spoofed.map((o) => o.ip).filter((v): v is string => !!v))]\n if (spoofedIps.length) {\n const requests = spoofed.reduce((n, o) => n + o.requests, 0)\n const vendors = [...new Set(spoofed.map((o) => o.botName))]\n out.push(\n finish({\n name: 'Deny impersonated crawler identities',\n rationale:\n 'These addresses claimed a crawler identity that failed verification against the vendor’s published ranges or signature.',\n evidence: `${requests.toLocaleString('en-US')} requests from ${spoofedIps.length} address${spoofedIps.length === 1 ? '' : 'es'} impersonating ${vendors.join(', ')}`,\n groups: [[{ type: 'ip_address', op: 'inc', value: spoofedIps }]],\n action: 'log',\n eventual: 'deny',\n risk: 'low',\n caveat:\n 'Verification failure is strong evidence, but confirm your edge controls x-forwarded-for before enforcing — behind a proxy that forwards a client-supplied header the verdict is worthless.'\n })\n )\n }\n\n /* 3. Single addresses behaving like scrapers. ------------------------------ */\n const perIp = observations.filter((o) => o.ip && o.verification !== 'verified')\n const threshold =\n opts.abuseThreshold ?? Math.max(500, Math.round(median(perIp.map((o) => o.requests)) * 10))\n const heavy = perIp\n .filter((o) => o.requests >= threshold)\n .sort((a, b) => b.requests - a.requests)\n .slice(0, 50)\n if (heavy.length) {\n const sweeping = heavy.filter((o) => (o.distinctPaths ?? 0) > 100)\n out.push(\n finish({\n name: 'Rate limit high-volume unverified addresses',\n rationale:\n 'A single address making orders of magnitude more requests than the median, with no verified identity.',\n evidence:\n `${heavy.length} address${heavy.length === 1 ? '' : 'es'} above ${threshold.toLocaleString('en-US')} requests` +\n (sweeping.length\n ? `; ${sweeping.length} swept >100 distinct paths, which reads as a scrape rather than a reader`\n : ''),\n groups: [[{ type: 'ip_address', op: 'inc', value: heavy.map((o) => o.ip!) }]],\n action: 'log',\n eventual: 'rate_limit',\n rateLimit: { window: 60, requests: 60, action: 'rate_limit', keys: ['ip'] },\n risk: 'medium',\n caveat:\n 'Shared egress means one address can front many real users — a corporate NAT, a mobile carrier, or a VPN. Review the dashboard before enforcing.'\n })\n )\n }\n\n /* 4. Training crawlers: bound the cost, do not disappear from corpora. ----- */\n const training = observations.filter((o) => o.intent === 'training')\n if (training.length) {\n const requests = training.reduce((n, o) => n + o.requests, 0)\n const vendors = [...new Set(training.map((o) => o.botName))]\n const budget = opts.trainingBudget ?? { window: 3600, requests: 600 }\n out.push(\n finish({\n name: 'Rate limit training crawlers',\n rationale:\n 'Bound what bulk corpus collection costs you without removing yourself from training sets.',\n evidence: `${requests.toLocaleString('en-US')} training requests from ${vendors.length} vendors (${vendors.slice(0, 6).join(', ')})`,\n groups: [\n [\n {\n type: 'user_agent',\n op: 'inc',\n value: ['GPTBot', 'ClaudeBot', 'CCBot', 'Bytespider', 'Amazonbot', 'meta-externalagent']\n }\n ]\n ],\n action: 'log',\n eventual: 'rate_limit',\n rateLimit: { window: budget.window, requests: budget.requests, action: 'rate_limit', keys: ['ip'] },\n risk: 'medium',\n caveat:\n 'Denying these removes you from future training sets, which may be exactly wrong for discoverability. Rate limit rather than deny unless you have decided otherwise. Note Vercel counters are per region, so N regions can collectively exceed the limit by ~Nx.'\n })\n )\n }\n\n /* 5. Datacenter ASNs presenting browser user agents. ---------------------- */\n const headlessAsns = [\n ...new Set(\n observations\n .filter((o) => o.asn !== undefined && /Mozilla|Chrome|Safari/i.test(o.userAgent))\n .filter((o) => o.verification !== 'verified')\n .map((o) => o.asn!)\n )\n ]\n if (headlessAsns.length) {\n const slices = observations.filter((o) => o.asn !== undefined && headlessAsns.includes(o.asn))\n const requests = slices.reduce((n, o) => n + o.requests, 0)\n out.push(\n finish({\n name: 'Challenge browser user agents from datacenter networks',\n rationale:\n 'A browser user agent arriving from a hosting network is automation wearing a costume — real browsers come from consumer ISPs.',\n evidence: `${requests.toLocaleString('en-US')} requests across ${headlessAsns.length} datacenter AS numbers`,\n groups: [\n [\n { type: 'geo_as_number', op: 'inc', value: headlessAsns },\n { type: 'user_agent', op: 'sub', value: 'Mozilla' }\n ]\n ],\n action: 'log',\n eventual: 'challenge',\n risk: 'high',\n caveat:\n 'Highest false-positive risk here. Corporate VPNs, privacy relays and some mobile carriers egress from hosting ASNs, and a challenge page breaks API clients and link unfurlers outright. Keep this in log mode for a full week before considering enforcement.'\n })\n )\n }\n\n return out\n}\n\n/** Render recommendations as a runnable, commented shell script. */\nexport function firewallScript(recommendations: readonly FirewallRecommendation[]): string {\n const lines = [\n '#!/usr/bin/env bash',\n '# Vercel WAF proposals generated from observed agent traffic.',\n '#',\n '# Every rule starts in LOG mode and blocks nothing. Vercel stages rule',\n '# changes as drafts, so nothing is live until you run:',\n '#',\n '# vercel firewall diff # review',\n '# vercel firewall publish --yes # go live',\n '#',\n '# Review each rule in the dashboard before promoting it to its eventual',\n '# action. Rules evaluate top to bottom, so keep the bypass rule first.',\n 'set -euo pipefail',\n ''\n ]\n recommendations.forEach((r, i) => {\n lines.push(`# ${i + 1}. ${r.name}`)\n lines.push(`# why: ${r.rationale}`)\n lines.push(`# evidence: ${r.evidence}`)\n lines.push(`# risk: ${r.risk} — eventual action: ${r.eventual}`)\n if (r.caveat) lines.push(`# caveat: ${r.caveat}`)\n lines.push(r.cli)\n lines.push('')\n })\n if (recommendations.length) {\n lines.push('# Keep the protective allow rule at the top of the evaluation order.')\n lines.push(\n `vercel firewall rules reorder ${JSON.stringify(recommendations[0]!.name)} --first --yes`\n )\n lines.push('')\n lines.push('vercel firewall diff')\n lines.push('echo \"Review above, then: vercel firewall publish --yes\"')\n }\n return lines.join('\\n')\n}\n"]}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { A as AgentIntent } from './policy-B3AakjOJ.cjs';
|
|
2
|
+
import './types-Dw43eu7D.cjs';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Recommend Vercel WAF rules from observed agent traffic.
|
|
6
|
+
*
|
|
7
|
+
* This generates *proposals*, never live changes. Every recommendation comes out
|
|
8
|
+
* with `action: 'log'`, because a firewall rule's blast radius is unpredictable
|
|
9
|
+
* until real traffic hits it and a bad `deny` takes out real users or your SEO.
|
|
10
|
+
* Vercel's own guidance is log → review → preview → production; the `eventual`
|
|
11
|
+
* field records where a rule is meant to end up, and `cli` emits the command for
|
|
12
|
+
* the *current* stage only.
|
|
13
|
+
*
|
|
14
|
+
* Two hard rules, both from measurement rather than taste:
|
|
15
|
+
*
|
|
16
|
+
* 1. Retrieval agents and search crawlers are never proposed for blocking.
|
|
17
|
+
* 60% of AI traffic on one production site is retrieval — a person asked a
|
|
18
|
+
* question and an assistant went to read the page. Blocking that is
|
|
19
|
+
* blocking your own distribution. The recommender emits a `bypass` rule to
|
|
20
|
+
* protect them *first*, so later rules cannot catch them.
|
|
21
|
+
*
|
|
22
|
+
* 2. Training crawlers get rate limits, not denials, by default. The point is
|
|
23
|
+
* to bound cost, not to disappear from corpora.
|
|
24
|
+
*
|
|
25
|
+
* Only abuse gets a denial: an identity that failed cryptographic or IP
|
|
26
|
+
* verification, or a single address behaving like a scraper.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** A Vercel WAF condition. Mirrors the CLI's `--condition` JSON. */
|
|
30
|
+
interface FirewallCondition {
|
|
31
|
+
type: 'user_agent' | 'ip_address' | 'geo_as_number' | 'geo_country' | 'path' | 'method' | 'environment' | 'ja4_digest';
|
|
32
|
+
op: 'eq' | 'neq' | 'sub' | 'pre' | 'suf' | 're' | 'inc' | 'ninc' | 'gt' | 'gte';
|
|
33
|
+
value?: string | number | Array<string | number>;
|
|
34
|
+
key?: string;
|
|
35
|
+
neg?: boolean;
|
|
36
|
+
}
|
|
37
|
+
type FirewallAction = 'log' | 'deny' | 'challenge' | 'bypass' | 'rate_limit';
|
|
38
|
+
interface RateLimitSpec {
|
|
39
|
+
/** Seconds, 10–3600. */
|
|
40
|
+
window: number;
|
|
41
|
+
/** Max requests per window. */
|
|
42
|
+
requests: number;
|
|
43
|
+
/** What happens on breach. */
|
|
44
|
+
action: 'rate_limit' | 'deny' | 'challenge' | 'log';
|
|
45
|
+
keys: Array<'ip' | 'ja4'>;
|
|
46
|
+
}
|
|
47
|
+
interface FirewallRecommendation {
|
|
48
|
+
name: string;
|
|
49
|
+
/** Why this rule is proposed, in one sentence. */
|
|
50
|
+
rationale: string;
|
|
51
|
+
/** The measurement behind it. Never propose a rule without evidence. */
|
|
52
|
+
evidence: string;
|
|
53
|
+
/** OR of ANDs: outer array is groups, inner is conditions within a group. */
|
|
54
|
+
groups: FirewallCondition[][];
|
|
55
|
+
/** Always `'log'` or `'bypass'` — see the module note. */
|
|
56
|
+
action: FirewallAction;
|
|
57
|
+
/** Where this rule is intended to end up after review. */
|
|
58
|
+
eventual: FirewallAction;
|
|
59
|
+
rateLimit?: RateLimitSpec;
|
|
60
|
+
/** How likely this is to catch traffic you wanted. */
|
|
61
|
+
risk: 'low' | 'medium' | 'high';
|
|
62
|
+
/** What could go wrong, when it is not obvious. */
|
|
63
|
+
caveat?: string;
|
|
64
|
+
/** Ready-to-run CLI for the *current* stage. */
|
|
65
|
+
cli: string;
|
|
66
|
+
/** Equivalent `--json` payload. */
|
|
67
|
+
json: unknown;
|
|
68
|
+
}
|
|
69
|
+
/** One aggregated slice of observed traffic. */
|
|
70
|
+
interface TrafficObservation {
|
|
71
|
+
userAgent: string;
|
|
72
|
+
botName: string;
|
|
73
|
+
intent: AgentIntent;
|
|
74
|
+
requests: number;
|
|
75
|
+
ip?: string;
|
|
76
|
+
/** Autonomous system number, if you resolved one. */
|
|
77
|
+
asn?: number;
|
|
78
|
+
/** Distinct paths this slice touched — a scraper sweeps, a reader does not. */
|
|
79
|
+
distinctPaths?: number;
|
|
80
|
+
/** Verification verdict, if you ran one. */
|
|
81
|
+
verification?: 'verified' | 'spoofed' | 'unverifiable' | 'not-claimed';
|
|
82
|
+
country?: string;
|
|
83
|
+
}
|
|
84
|
+
interface RecommendOptions {
|
|
85
|
+
/**
|
|
86
|
+
* Requests-per-slice above which a single IP is considered abusive. Defaults
|
|
87
|
+
* to 10x the median across observations, floored at 500.
|
|
88
|
+
*/
|
|
89
|
+
abuseThreshold?: number;
|
|
90
|
+
/** Rate-limit budget proposed for training crawlers. Defaults to 600/hour. */
|
|
91
|
+
trainingBudget?: {
|
|
92
|
+
window: number;
|
|
93
|
+
requests: number;
|
|
94
|
+
};
|
|
95
|
+
/** Skip the protective bypass rule. Rarely a good idea. */
|
|
96
|
+
omitProtectiveBypass?: boolean;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Turn observations into staged WAF proposals.
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```ts
|
|
103
|
+
* const rules = recommendFirewallRules(observations)
|
|
104
|
+
* for (const r of rules) {
|
|
105
|
+
* console.log(`# ${r.name} — ${r.rationale}`)
|
|
106
|
+
* console.log(`# evidence: ${r.evidence}`)
|
|
107
|
+
* console.log(r.cli)
|
|
108
|
+
* }
|
|
109
|
+
* ```
|
|
110
|
+
*/
|
|
111
|
+
declare function recommendFirewallRules(observations: readonly TrafficObservation[], opts?: RecommendOptions): FirewallRecommendation[];
|
|
112
|
+
/** Render recommendations as a runnable, commented shell script. */
|
|
113
|
+
declare function firewallScript(recommendations: readonly FirewallRecommendation[]): string;
|
|
114
|
+
|
|
115
|
+
export { type FirewallAction, type FirewallCondition, type FirewallRecommendation, type RateLimitSpec, type RecommendOptions, type TrafficObservation, firewallScript, recommendFirewallRules };
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { A as AgentIntent } from './policy-DMTBUe4F.js';
|
|
2
|
+
import './types-Dw43eu7D.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Recommend Vercel WAF rules from observed agent traffic.
|
|
6
|
+
*
|
|
7
|
+
* This generates *proposals*, never live changes. Every recommendation comes out
|
|
8
|
+
* with `action: 'log'`, because a firewall rule's blast radius is unpredictable
|
|
9
|
+
* until real traffic hits it and a bad `deny` takes out real users or your SEO.
|
|
10
|
+
* Vercel's own guidance is log → review → preview → production; the `eventual`
|
|
11
|
+
* field records where a rule is meant to end up, and `cli` emits the command for
|
|
12
|
+
* the *current* stage only.
|
|
13
|
+
*
|
|
14
|
+
* Two hard rules, both from measurement rather than taste:
|
|
15
|
+
*
|
|
16
|
+
* 1. Retrieval agents and search crawlers are never proposed for blocking.
|
|
17
|
+
* 60% of AI traffic on one production site is retrieval — a person asked a
|
|
18
|
+
* question and an assistant went to read the page. Blocking that is
|
|
19
|
+
* blocking your own distribution. The recommender emits a `bypass` rule to
|
|
20
|
+
* protect them *first*, so later rules cannot catch them.
|
|
21
|
+
*
|
|
22
|
+
* 2. Training crawlers get rate limits, not denials, by default. The point is
|
|
23
|
+
* to bound cost, not to disappear from corpora.
|
|
24
|
+
*
|
|
25
|
+
* Only abuse gets a denial: an identity that failed cryptographic or IP
|
|
26
|
+
* verification, or a single address behaving like a scraper.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** A Vercel WAF condition. Mirrors the CLI's `--condition` JSON. */
|
|
30
|
+
interface FirewallCondition {
|
|
31
|
+
type: 'user_agent' | 'ip_address' | 'geo_as_number' | 'geo_country' | 'path' | 'method' | 'environment' | 'ja4_digest';
|
|
32
|
+
op: 'eq' | 'neq' | 'sub' | 'pre' | 'suf' | 're' | 'inc' | 'ninc' | 'gt' | 'gte';
|
|
33
|
+
value?: string | number | Array<string | number>;
|
|
34
|
+
key?: string;
|
|
35
|
+
neg?: boolean;
|
|
36
|
+
}
|
|
37
|
+
type FirewallAction = 'log' | 'deny' | 'challenge' | 'bypass' | 'rate_limit';
|
|
38
|
+
interface RateLimitSpec {
|
|
39
|
+
/** Seconds, 10–3600. */
|
|
40
|
+
window: number;
|
|
41
|
+
/** Max requests per window. */
|
|
42
|
+
requests: number;
|
|
43
|
+
/** What happens on breach. */
|
|
44
|
+
action: 'rate_limit' | 'deny' | 'challenge' | 'log';
|
|
45
|
+
keys: Array<'ip' | 'ja4'>;
|
|
46
|
+
}
|
|
47
|
+
interface FirewallRecommendation {
|
|
48
|
+
name: string;
|
|
49
|
+
/** Why this rule is proposed, in one sentence. */
|
|
50
|
+
rationale: string;
|
|
51
|
+
/** The measurement behind it. Never propose a rule without evidence. */
|
|
52
|
+
evidence: string;
|
|
53
|
+
/** OR of ANDs: outer array is groups, inner is conditions within a group. */
|
|
54
|
+
groups: FirewallCondition[][];
|
|
55
|
+
/** Always `'log'` or `'bypass'` — see the module note. */
|
|
56
|
+
action: FirewallAction;
|
|
57
|
+
/** Where this rule is intended to end up after review. */
|
|
58
|
+
eventual: FirewallAction;
|
|
59
|
+
rateLimit?: RateLimitSpec;
|
|
60
|
+
/** How likely this is to catch traffic you wanted. */
|
|
61
|
+
risk: 'low' | 'medium' | 'high';
|
|
62
|
+
/** What could go wrong, when it is not obvious. */
|
|
63
|
+
caveat?: string;
|
|
64
|
+
/** Ready-to-run CLI for the *current* stage. */
|
|
65
|
+
cli: string;
|
|
66
|
+
/** Equivalent `--json` payload. */
|
|
67
|
+
json: unknown;
|
|
68
|
+
}
|
|
69
|
+
/** One aggregated slice of observed traffic. */
|
|
70
|
+
interface TrafficObservation {
|
|
71
|
+
userAgent: string;
|
|
72
|
+
botName: string;
|
|
73
|
+
intent: AgentIntent;
|
|
74
|
+
requests: number;
|
|
75
|
+
ip?: string;
|
|
76
|
+
/** Autonomous system number, if you resolved one. */
|
|
77
|
+
asn?: number;
|
|
78
|
+
/** Distinct paths this slice touched — a scraper sweeps, a reader does not. */
|
|
79
|
+
distinctPaths?: number;
|
|
80
|
+
/** Verification verdict, if you ran one. */
|
|
81
|
+
verification?: 'verified' | 'spoofed' | 'unverifiable' | 'not-claimed';
|
|
82
|
+
country?: string;
|
|
83
|
+
}
|
|
84
|
+
interface RecommendOptions {
|
|
85
|
+
/**
|
|
86
|
+
* Requests-per-slice above which a single IP is considered abusive. Defaults
|
|
87
|
+
* to 10x the median across observations, floored at 500.
|
|
88
|
+
*/
|
|
89
|
+
abuseThreshold?: number;
|
|
90
|
+
/** Rate-limit budget proposed for training crawlers. Defaults to 600/hour. */
|
|
91
|
+
trainingBudget?: {
|
|
92
|
+
window: number;
|
|
93
|
+
requests: number;
|
|
94
|
+
};
|
|
95
|
+
/** Skip the protective bypass rule. Rarely a good idea. */
|
|
96
|
+
omitProtectiveBypass?: boolean;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Turn observations into staged WAF proposals.
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```ts
|
|
103
|
+
* const rules = recommendFirewallRules(observations)
|
|
104
|
+
* for (const r of rules) {
|
|
105
|
+
* console.log(`# ${r.name} — ${r.rationale}`)
|
|
106
|
+
* console.log(`# evidence: ${r.evidence}`)
|
|
107
|
+
* console.log(r.cli)
|
|
108
|
+
* }
|
|
109
|
+
* ```
|
|
110
|
+
*/
|
|
111
|
+
declare function recommendFirewallRules(observations: readonly TrafficObservation[], opts?: RecommendOptions): FirewallRecommendation[];
|
|
112
|
+
/** Render recommendations as a runnable, commented shell script. */
|
|
113
|
+
declare function firewallScript(recommendations: readonly FirewallRecommendation[]): string;
|
|
114
|
+
|
|
115
|
+
export { type FirewallAction, type FirewallCondition, type FirewallRecommendation, type RateLimitSpec, type RecommendOptions, type TrafficObservation, firewallScript, recommendFirewallRules };
|
package/dist/firewall.js
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
function f(i){return `'${JSON.stringify(i).replace(/'/g,"'\\''")}'`}function v(i){let e=[`vercel firewall rules add ${JSON.stringify(i.name)}`];if(i.groups.forEach((n,o)=>{o>0&&e.push(" --or");for(let l of n)e.push(` --condition ${f(l)}`);}),e.push(` --action ${i.action}`),i.action==="rate_limit"&&i.rateLimit){e.push(` --rate-limit-window ${i.rateLimit.window}`),e.push(` --rate-limit-requests ${i.rateLimit.requests}`),e.push(` --rate-limit-action ${i.rateLimit.action}`);for(let n of i.rateLimit.keys)e.push(` --rate-limit-keys ${n}`);}return e.push(" --yes"),e.join(` \\
|
|
2
|
+
`)}function w(i){return {name:i.name,conditionGroup:i.groups.map(e=>({conditions:e})),action:{mitigate:{action:i.action}}}}function u(i){return {...i,cli:v(i),json:w(i)}}function y(i){if(!i.length)return 0;let e=[...i].sort((o,l)=>o-l),n=Math.floor(e.length/2);return e.length%2?e[n]:(e[n-1]+e[n])/2}function b(i,e={}){let n=[];if(!e.omitProtectiveBypass){let t=i.filter(a=>a.intent==="retrieval"||a.intent==="search"),r=t.reduce((a,g)=>a+g.requests,0),s=[...new Set(t.map(a=>a.botName))];n.push(u({name:"Allow retrieval and search agents",rationale:"Retrieval agents and search crawlers must never be caught by the rules below \u2014 they bring readers and rankings.",evidence:r?`${r.toLocaleString("en-US")} observed requests across ${s.length} vendors (${s.slice(0,6).join(", ")})`:"no retrieval or search traffic observed yet; installed pre-emptively",groups:[[{type:"user_agent",op:"inc",value:["ChatGPT-User","OAI-SearchBot","Claude-User","Claude-SearchBot","Perplexity-User","Googlebot","bingbot","DuckDuckBot","Applebot"]}]],action:"bypass",eventual:"bypass",risk:"low",caveat:"Place this rule first (`vercel firewall rules reorder ... --first`). A user-agent allowlist is spoofable, so pair with verification in middleware rather than relying on it for security \u2014 its job here is to stop your own rules misfiring."}));}let o=i.filter(t=>t.verification==="spoofed"),l=[...new Set(o.map(t=>t.ip).filter(t=>!!t))];if(l.length){let t=o.reduce((s,a)=>s+a.requests,0),r=[...new Set(o.map(s=>s.botName))];n.push(u({name:"Deny impersonated crawler identities",rationale:"These addresses claimed a crawler identity that failed verification against the vendor\u2019s published ranges or signature.",evidence:`${t.toLocaleString("en-US")} requests from ${l.length} address${l.length===1?"":"es"} impersonating ${r.join(", ")}`,groups:[[{type:"ip_address",op:"inc",value:l}]],action:"log",eventual:"deny",risk:"low",caveat:"Verification failure is strong evidence, but confirm your edge controls x-forwarded-for before enforcing \u2014 behind a proxy that forwards a client-supplied header the verdict is worthless."}));}let h=i.filter(t=>t.ip&&t.verification!=="verified"),p=e.abuseThreshold??Math.max(500,Math.round(y(h.map(t=>t.requests))*10)),c=h.filter(t=>t.requests>=p).sort((t,r)=>r.requests-t.requests).slice(0,50);if(c.length){let t=c.filter(r=>(r.distinctPaths??0)>100);n.push(u({name:"Rate limit high-volume unverified addresses",rationale:"A single address making orders of magnitude more requests than the median, with no verified identity.",evidence:`${c.length} address${c.length===1?"":"es"} above ${p.toLocaleString("en-US")} requests`+(t.length?`; ${t.length} swept >100 distinct paths, which reads as a scrape rather than a reader`:""),groups:[[{type:"ip_address",op:"inc",value:c.map(r=>r.ip)}]],action:"log",eventual:"rate_limit",rateLimit:{window:60,requests:60,action:"rate_limit",keys:["ip"]},risk:"medium",caveat:"Shared egress means one address can front many real users \u2014 a corporate NAT, a mobile carrier, or a VPN. Review the dashboard before enforcing."}));}let m=i.filter(t=>t.intent==="training");if(m.length){let t=m.reduce((a,g)=>a+g.requests,0),r=[...new Set(m.map(a=>a.botName))],s=e.trainingBudget??{window:3600,requests:600};n.push(u({name:"Rate limit training crawlers",rationale:"Bound what bulk corpus collection costs you without removing yourself from training sets.",evidence:`${t.toLocaleString("en-US")} training requests from ${r.length} vendors (${r.slice(0,6).join(", ")})`,groups:[[{type:"user_agent",op:"inc",value:["GPTBot","ClaudeBot","CCBot","Bytespider","Amazonbot","meta-externalagent"]}]],action:"log",eventual:"rate_limit",rateLimit:{window:s.window,requests:s.requests,action:"rate_limit",keys:["ip"]},risk:"medium",caveat:"Denying these removes you from future training sets, which may be exactly wrong for discoverability. Rate limit rather than deny unless you have decided otherwise. Note Vercel counters are per region, so N regions can collectively exceed the limit by ~Nx."}));}let d=[...new Set(i.filter(t=>t.asn!==void 0&&/Mozilla|Chrome|Safari/i.test(t.userAgent)).filter(t=>t.verification!=="verified").map(t=>t.asn))];if(d.length){let r=i.filter(s=>s.asn!==void 0&&d.includes(s.asn)).reduce((s,a)=>s+a.requests,0);n.push(u({name:"Challenge browser user agents from datacenter networks",rationale:"A browser user agent arriving from a hosting network is automation wearing a costume \u2014 real browsers come from consumer ISPs.",evidence:`${r.toLocaleString("en-US")} requests across ${d.length} datacenter AS numbers`,groups:[[{type:"geo_as_number",op:"inc",value:d},{type:"user_agent",op:"sub",value:"Mozilla"}]],action:"log",eventual:"challenge",risk:"high",caveat:"Highest false-positive risk here. Corporate VPNs, privacy relays and some mobile carriers egress from hosting ASNs, and a challenge page breaks API clients and link unfurlers outright. Keep this in log mode for a full week before considering enforcement."}));}return n}function $(i){let e=["#!/usr/bin/env bash","# Vercel WAF proposals generated from observed agent traffic.","#","# Every rule starts in LOG mode and blocks nothing. Vercel stages rule","# changes as drafts, so nothing is live until you run:","#","# vercel firewall diff # review","# vercel firewall publish --yes # go live","#","# Review each rule in the dashboard before promoting it to its eventual","# action. Rules evaluate top to bottom, so keep the bypass rule first.","set -euo pipefail",""];return i.forEach((n,o)=>{e.push(`# ${o+1}. ${n.name}`),e.push(`# why: ${n.rationale}`),e.push(`# evidence: ${n.evidence}`),e.push(`# risk: ${n.risk} \u2014 eventual action: ${n.eventual}`),n.caveat&&e.push(`# caveat: ${n.caveat}`),e.push(n.cli),e.push("");}),i.length&&(e.push("# Keep the protective allow rule at the top of the evaluation order."),e.push(`vercel firewall rules reorder ${JSON.stringify(i[0].name)} --first --yes`),e.push(""),e.push("vercel firewall diff"),e.push('echo "Review above, then: vercel firewall publish --yes"')),e.join(`
|
|
3
|
+
`)}export{$ as firewallScript,b as recommendFirewallRules};//# sourceMappingURL=firewall.js.map
|
|
4
|
+
//# sourceMappingURL=firewall.js.map
|