@walkeros/server-transformer-bot 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +283 -53
- package/dist/dev.d.mts +108 -9
- package/dist/dev.d.ts +108 -9
- package/dist/dev.js +1 -1
- package/dist/dev.js.map +1 -1
- package/dist/dev.mjs +1 -1
- package/dist/dev.mjs.map +1 -1
- package/dist/index.d.mts +147 -7
- package/dist/index.d.ts +147 -7
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/walkerOS.json +759 -38
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @walkeros/server-transformer-bot
|
|
2
2
|
|
|
3
|
+
## 4.5.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 71d83a3: `settings.context` now accepts any `Mapping.Value`, not just the enum
|
|
8
|
+
literals, and resolves it per request against `{ event, ingest }`, so one
|
|
9
|
+
instance serves every transport:
|
|
10
|
+
`[{ "key": "ingest.transport" }, { "value": "beacon" }]` reads the sender's
|
|
11
|
+
annotation and pins `beacon` without it. The widening introduces no
|
|
12
|
+
silent-degradation path: a result outside the vocabulary falls back to `auto`
|
|
13
|
+
and reports `context_undetermined`, and a bare string that is neither a
|
|
14
|
+
literal nor a dot path is rejected by the schema.
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- 7dc6cb1: Bot detection now scores all sixteen request signals instead of the
|
|
19
|
+
user agent alone, and emits one `botScore` with a `botCategory`, `botProduct`
|
|
20
|
+
and reason codes. This replaces the old output: `agentScore` is removed, score
|
|
21
|
+
values changed, and `settings.output` takes `string | false`. New settings:
|
|
22
|
+
`context` and `suspiciousAt`.
|
|
23
|
+
- Updated dependencies [63845bb]
|
|
24
|
+
- Updated dependencies [79cdcb0]
|
|
25
|
+
- Updated dependencies [756b571]
|
|
26
|
+
- @walkeros/core@4.5.0
|
|
27
|
+
|
|
3
28
|
## 4.4.0
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
Server-side bot and AI-agent detection transformer for walkerOS.
|
|
4
4
|
|
|
5
|
-
Annotates events with
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
destination mappings decide policy.
|
|
5
|
+
Annotates events with an automation likelihood, a category saying what kind of
|
|
6
|
+
client it is, the identified product when there is one, and stable reason codes.
|
|
7
|
+
Never drops events: downstream destination mappings decide policy.
|
|
9
8
|
|
|
10
9
|
## Install
|
|
11
10
|
|
|
@@ -27,13 +26,47 @@ await startFlow({
|
|
|
27
26
|
ingest: {
|
|
28
27
|
map: {
|
|
29
28
|
userAgent: { key: 'headers.user-agent' },
|
|
29
|
+
acceptLanguage: { key: 'headers.accept-language' },
|
|
30
|
+
acceptEncoding: { key: 'headers.accept-encoding' },
|
|
31
|
+
secFetchSite: { key: 'headers.sec-fetch-site' },
|
|
32
|
+
secFetchMode: { key: 'headers.sec-fetch-mode' },
|
|
33
|
+
secFetchDest: { key: 'headers.sec-fetch-dest' },
|
|
34
|
+
secFetchUser: { key: 'headers.sec-fetch-user' },
|
|
35
|
+
secChUa: { key: 'headers.sec-ch-ua' },
|
|
36
|
+
secChUaMobile: { key: 'headers.sec-ch-ua-mobile' },
|
|
37
|
+
secChUaPlatform: { key: 'headers.sec-ch-ua-platform' },
|
|
38
|
+
accept: { key: 'headers.accept' },
|
|
39
|
+
contentType: { key: 'headers.content-type' },
|
|
40
|
+
referer: { key: 'headers.referer' },
|
|
41
|
+
signatureAgent: { key: 'headers.signature-agent' },
|
|
42
|
+
method: { key: 'method' },
|
|
30
43
|
},
|
|
31
44
|
},
|
|
32
45
|
},
|
|
33
46
|
},
|
|
34
47
|
},
|
|
35
48
|
transformers: {
|
|
36
|
-
bot: {
|
|
49
|
+
bot: {
|
|
50
|
+
code: transformerBot,
|
|
51
|
+
config: {
|
|
52
|
+
settings: {
|
|
53
|
+
// Pinning the context unlocks the context-dependent checks. A
|
|
54
|
+
// literal pins every request; any Mapping.Value resolves per
|
|
55
|
+
// request. See "Request context".
|
|
56
|
+
context: 'beacon',
|
|
57
|
+
// Declaring a name asserts the signal is wired, which unlocks the
|
|
58
|
+
// absence-based checks for its family. See "Declared signals".
|
|
59
|
+
input: {
|
|
60
|
+
acceptLanguage: 'ingest.acceptLanguage',
|
|
61
|
+
acceptEncoding: 'ingest.acceptEncoding',
|
|
62
|
+
secFetchSite: 'ingest.secFetchSite',
|
|
63
|
+
secFetchMode: 'ingest.secFetchMode',
|
|
64
|
+
secFetchDest: 'ingest.secFetchDest',
|
|
65
|
+
secChUa: 'ingest.secChUa',
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
},
|
|
37
70
|
},
|
|
38
71
|
destinations: {
|
|
39
72
|
ga4: {
|
|
@@ -51,82 +84,279 @@ After the transformer runs:
|
|
|
51
84
|
{
|
|
52
85
|
"user": {
|
|
53
86
|
"botScore": 0,
|
|
54
|
-
"
|
|
87
|
+
"botCategory": "human"
|
|
55
88
|
}
|
|
56
89
|
}
|
|
57
90
|
```
|
|
58
91
|
|
|
59
|
-
##
|
|
92
|
+
## Output fields
|
|
93
|
+
|
|
94
|
+
Every field takes `string | false` in `settings.output`. `false` disables it,
|
|
95
|
+
any dot path renames and reroutes it, and an `ingest.` prefix keeps it off the
|
|
96
|
+
analytics event and in pipeline scratch.
|
|
97
|
+
|
|
98
|
+
| Field | Type | Default path | Meaning |
|
|
99
|
+
| ------------- | --------------------- | -------------------- | ----------------------------------------------------------------------------------------- |
|
|
100
|
+
| `botScore` | `number` 0-99, `null` | `user.botScore` | Automation likelihood. Higher = more automated. `null` means not measured, never "human". |
|
|
101
|
+
| `botCategory` | enum | `user.botCategory` | What kind of client. Carries the confidence class. |
|
|
102
|
+
| `botProduct` | `string` | `user.botProduct` | Identified product, e.g. `ChatGPT-User`, `Googlebot`. Written only on a named match. |
|
|
103
|
+
| `botReasons` | `string[]` | `ingest.bot.reasons` | Stable reason codes. Doubles as the pipeline's configuration diagnostic. |
|
|
104
|
+
|
|
105
|
+
`ingest` is shared by every event in a scope and the write is an assignment, so
|
|
106
|
+
if a `before`-chain step fans one input into several events,
|
|
107
|
+
`ingest.bot.reasons` reflects only the most recently scored one. That is
|
|
108
|
+
harmless for the diagnostic codes, which are identical for every event on a
|
|
109
|
+
given pipeline. A deployment that fans out and needs per-event scoring codes
|
|
110
|
+
should point `settings.output.botReasons` at an event path instead.
|
|
111
|
+
|
|
112
|
+
## Categories
|
|
113
|
+
|
|
114
|
+
| Value | Assigned when | Score |
|
|
115
|
+
| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ------ |
|
|
116
|
+
| `human` | no evidence, or graded sum below the cut | 0-24 |
|
|
117
|
+
| `suspicious` | graded sum at or above the cut, nothing decisive | 25-60 |
|
|
118
|
+
| `automation` | isbot match, missing UA, or a value impossible for the pinned context | 70-80 |
|
|
119
|
+
| `search-crawler` | search engine crawler (Googlebot, bingbot, Applebot, YandexBot, Baiduspider, PetalBot, …) | 90 |
|
|
120
|
+
| `seo-tool` | commercial SEO crawler (AhrefsBot, SemrushBot, DotBot, MJ12bot, Screaming Frog) | 90 |
|
|
121
|
+
| `monitor` | uptime and synthetic monitoring, usually your own infrastructure (UptimeRobot, Pingdom, StatusCake) | 90 |
|
|
122
|
+
| `link-preview` | link unfurler, meaning a person just shared this URL (facebookexternalhit, Twitterbot, LinkedInBot, Slackbot, Discordbot, TelegramBot, WhatsApp) | 90 |
|
|
123
|
+
| `ai-agent` | AI agent acting for a person (ChatGPT-User, Claude-User, Perplexity-User, Google-Agent) | 90 |
|
|
124
|
+
| `ai-crawler` | AI training or search-index crawler (GPTBot, ClaudeBot, CCBot, OAI-SearchBot) | 90 |
|
|
125
|
+
| `unknown` | nothing resolvable at all | `null` |
|
|
126
|
+
|
|
127
|
+
Every UA-map match scores 90 regardless of kind: a match is a match, and ranking
|
|
128
|
+
them in the number would assert a confidence difference that does not exist. The
|
|
129
|
+
category carries the kind. `isbot` stays a rung lower at 80 because it is a
|
|
130
|
+
weaker statement, saying "this looks automated" without naming a product.
|
|
131
|
+
|
|
132
|
+
`botScore` is `null`, not omitted, for `unknown`. Events arrive as a request
|
|
133
|
+
body on an endpoint that may be open, so an inbound `user.botScore` is
|
|
134
|
+
client-controllable; writing `null` overwrites it and states "not measured",
|
|
135
|
+
while still evaluating false under a `botScore > 50` filter.
|
|
136
|
+
|
|
137
|
+
## Detection layers
|
|
138
|
+
|
|
139
|
+
Deterministic, most specific first, first match wins. The graded layer does not
|
|
140
|
+
run when one of these fires.
|
|
141
|
+
|
|
142
|
+
| Score | Category | Trigger |
|
|
143
|
+
| ----- | ------------ | ----------------------------------------- |
|
|
144
|
+
| 70 | `automation` | missing User-Agent |
|
|
145
|
+
| 90 | named | AI agent or AI crawler UA map |
|
|
146
|
+
| 90 | named | non-AI crawler UA map |
|
|
147
|
+
| 80 | `automation` | `isbot` |
|
|
148
|
+
| 75 | `automation` | a value impossible for the pinned context |
|
|
149
|
+
|
|
150
|
+
A UA-map match that co-occurs with a contradiction keeps its score, because the
|
|
151
|
+
client is still software, but loses its claimed identity: the category drops to
|
|
152
|
+
`automation`, no `botProduct` is written, and `identity_claim_contradicted`
|
|
153
|
+
appears in the reasons.
|
|
154
|
+
|
|
155
|
+
Graded, capped at 60, so the two layers never overlap and every score is
|
|
156
|
+
attributable to exactly one of them:
|
|
157
|
+
|
|
158
|
+
| Reason code | Weight |
|
|
159
|
+
| --------------------------------- | ------ |
|
|
160
|
+
| `ch_version_mismatch` | 30 |
|
|
161
|
+
| `ch_missing_on_chromium` | 25 |
|
|
162
|
+
| `accept_generic_on_typed_context` | 25 |
|
|
163
|
+
| `fetchmeta_missing_on_modern_ua` | 15 |
|
|
164
|
+
| `fetchmeta_profile_mismatch` | 15 |
|
|
165
|
+
| `accept_language_missing` | 10 |
|
|
166
|
+
| `accept_encoding_missing` | 5 |
|
|
167
|
+
|
|
168
|
+
These weights and the `suspiciousAt` default of 25 are provisional starting
|
|
169
|
+
values from vendor specs and published prior art, not calibrated against a
|
|
170
|
+
labelled corpus. The package's captured-headers fixture suite is what keeps them
|
|
171
|
+
safe to ship: a weight that misfires on a real browser capture fails a test.
|
|
172
|
+
|
|
173
|
+
Reason codes are a semver-stable public API. Deterministic codes are
|
|
174
|
+
`ua_missing`, `ua_isbot`, `ua_named_bot`, `identity_claim_contradicted`,
|
|
175
|
+
`fetchmeta_impossible_for_context`, `content_type_impossible_for_context` and
|
|
176
|
+
`ch_platform_contradiction`. `signature_agent_present` is informational and
|
|
177
|
+
makes no verification claim. A clean real browser on a fully wired pipeline
|
|
178
|
+
emits zero reasons.
|
|
179
|
+
|
|
180
|
+
## Declared signals: absent versus never wired
|
|
181
|
+
|
|
182
|
+
The package cannot tell "the client sent no `Sec-CH-UA`" from "the operator
|
|
183
|
+
never mapped `Sec-CH-UA` into `ingest`". Both arrive as `undefined`. Every
|
|
184
|
+
absence-based heuristic would therefore fire on a misconfigured pipeline and
|
|
185
|
+
mark real people as suspicious, which is the single most likely source of false
|
|
186
|
+
values in the whole design.
|
|
187
|
+
|
|
188
|
+
**So an absence-based heuristic runs only if its input name appears explicitly
|
|
189
|
+
in `settings.input`.** Reading still falls back to the defaults, so
|
|
190
|
+
presence-based detection works out of the box; but listing a name is you
|
|
191
|
+
asserting "this signal is wired in my pipeline", which is the only reliable
|
|
192
|
+
source of that fact. Listing a name with its own default path therefore costs
|
|
193
|
+
nothing and unlocks the check.
|
|
60
194
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
| curl / wget / python-requests / well-known crawlers | 80 | 0 | — |
|
|
66
|
-
| AI training crawlers (GPTBot, ClaudeBot, CCBot, Bytespider, etc.) | 95 | 95 | e.g. "GPTBot" |
|
|
67
|
-
| AI search-index crawlers (OAI-SearchBot, Claude-SearchBot, PerplexityBot) | 95 | 95 | e.g. "PerplexityBot" |
|
|
68
|
-
| AI user-action agents (ChatGPT-User, Claude-User, Perplexity-User, etc.) | 90 | 95 | e.g. "ChatGPT-User" |
|
|
195
|
+
Until a family is declared, `ingest.bot.reasons` reports `ch_not_declared`,
|
|
196
|
+
`fetchmeta_not_declared` or `accept_not_declared`, naming exactly which mapping
|
|
197
|
+
to add to unlock which checks. The default experience is deliberately
|
|
198
|
+
conservative, and that is not a bug.
|
|
69
199
|
|
|
70
|
-
##
|
|
200
|
+
## Request context
|
|
71
201
|
|
|
72
|
-
|
|
202
|
+
The same header value means opposite things in different contexts. `Accept: */*`
|
|
203
|
+
from a browser UA is what every real browser sends on a beacon, and a strong bot
|
|
204
|
+
signal on an image pixel. So the context-dependent checks need to know how the
|
|
205
|
+
request was made.
|
|
73
206
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
|
78
|
-
|
|
207
|
+
`settings.context` is `BotContext | Mapping.Value`, which is three usable forms:
|
|
208
|
+
|
|
209
|
+
- **An enum literal**, one of
|
|
210
|
+
`'auto' | 'navigation' | 'pixel' | 'beacon' | 'fetch' | 'server'`, pins one
|
|
211
|
+
context for every request. Unset behaves like `'auto'`.
|
|
212
|
+
- **A per-request lookup**, resolved against `{ event, ingest }`: a dot-path
|
|
213
|
+
string such as `'ingest.transport'`, or a `{ key }` / `{ value }` / `{ fn }`
|
|
214
|
+
object.
|
|
215
|
+
- **A fallback array**, tried in order, e.g.
|
|
216
|
+
`[{ key: 'ingest.transport' }, { value: 'beacon' }]`.
|
|
217
|
+
|
|
218
|
+
A bare string is always read as a lookup path, so a string that is not one of
|
|
219
|
+
the six literals must contain a dot: `'beacn'` fails schema validation instead
|
|
220
|
+
of silently becoming a lookup that resolves to nothing. The guard holds inside
|
|
221
|
+
the array too, so a literal there is written `{ value: 'beacon' }` and never the
|
|
222
|
+
bare `'beacon'`.
|
|
223
|
+
|
|
224
|
+
The lookup form is what lets a single instance serve a deployment that receives
|
|
225
|
+
beacons, pixels and fetches on the same endpoint. The sender annotates its URL
|
|
226
|
+
(`?transport=beacon` on the collect URL, `?transport=pixel` on the pixel embed),
|
|
227
|
+
the source lifts it with `transport: { key: 'query.transport' }`, and the
|
|
228
|
+
transformer resolves it per request. A query parameter is a claim by whoever
|
|
229
|
+
controls the sender: right for telling your own transports apart, worthless
|
|
230
|
+
against a client that wants to be scored as a beacon. For server truth, give
|
|
231
|
+
pixels their own route and derive the context from `ingest.path` instead. The
|
|
232
|
+
end-to-end recipe is on
|
|
233
|
+
[the website page](https://www.walkeros.io/docs/transformers/bot).
|
|
234
|
+
|
|
235
|
+
Pinning is the strong mode: you know your own routes, and pinning unlocks the
|
|
236
|
+
`Accept` shape check, the Fetch Metadata profile comparison and the beacon
|
|
237
|
+
`Content-Type` check. **`auto` never enables them.** Absence of `Sec-Fetch-*` is
|
|
238
|
+
both a signal worth scoring and the reason auto-derivation fails, and the
|
|
239
|
+
request method does not rescue it, so `auto` runs the context-independent checks
|
|
240
|
+
only and reports `context_undetermined`. Anything that resolves outside the
|
|
241
|
+
vocabulary falls back to `auto` as well, so a broken annotation degrades to
|
|
242
|
+
fewer checks rather than to the wrong profile. The failure mode is "we scored
|
|
243
|
+
less", never "we scored wrong". `server` means server-to-server ingestion and
|
|
244
|
+
disables every browser-shaped check.
|
|
245
|
+
|
|
246
|
+
The client-hint coherence family, the most discriminating header signal
|
|
247
|
+
available, is context-independent and works in every deployment.
|
|
248
|
+
|
|
249
|
+
## Supported sources
|
|
250
|
+
|
|
251
|
+
Header-driven scoring depends on what each source puts in its raw scope, because
|
|
252
|
+
`config.ingest` resolves dot paths by plain property access.
|
|
253
|
+
|
|
254
|
+
| Source | Header access | Works with `{ key: 'headers.x' }` |
|
|
255
|
+
| ------------------ | ------------------------------------------------------------------------ | --------------------------------- |
|
|
256
|
+
| express | `req.headers`, a plain object, lowercased | yes |
|
|
257
|
+
| GCP Cloud Function | same, the Functions Framework request is Express-shaped | yes |
|
|
258
|
+
| AWS Lambda | `event.headers`; API Gateway v1 preserves original casing, v2 lowercases | casing-dependent |
|
|
259
|
+
| fetch | `request.headers` is a `Headers` instance, not own properties | **no** |
|
|
260
|
+
|
|
261
|
+
The fetch row is a hard limit rather than a nuance: a `Headers` instance holds
|
|
262
|
+
its values internally, so `{ key: 'headers.user-agent' }` yields `undefined` and
|
|
263
|
+
header-driven scoring is unavailable there until that source normalizes its
|
|
264
|
+
headers into a plain object. If every event scores `unknown`, check this table
|
|
265
|
+
first.
|
|
266
|
+
|
|
267
|
+
## Category reachability
|
|
268
|
+
|
|
269
|
+
Which categories can appear at all depends on how events reach the pipeline,
|
|
270
|
+
because most named bots do not execute JavaScript. An empty category is usually
|
|
271
|
+
expected, not broken.
|
|
272
|
+
|
|
273
|
+
| Category | JS tag (beacon / fetch) | Image pixel | Server-side / navigation |
|
|
274
|
+
| ------------------------ | ---------------------------------------- | -------------- | ------------------------ |
|
|
275
|
+
| `automation` | yes | yes | yes |
|
|
276
|
+
| `search-crawler` | **yes**, Googlebot renders with Chromium | yes | yes |
|
|
277
|
+
| `monitor` | some, those that drive a real browser | yes | yes |
|
|
278
|
+
| `seo-tool` | rarely, a few render JS optionally | yes | yes |
|
|
279
|
+
| `ai-agent`, `ai-crawler` | no | maybe | yes |
|
|
280
|
+
| `link-preview` | **no** | essentially no | **yes** |
|
|
281
|
+
|
|
282
|
+
Link unfurlers fetch the URL server-side, parse `<head>` for Open Graph tags and
|
|
283
|
+
stop. They do not run JavaScript and do not fetch body images, so `link-preview`
|
|
284
|
+
is structurally invisible to a JS-tagged pipeline and genuinely valuable only
|
|
285
|
+
for server-side collection, where "a person shared this on LinkedIn" otherwise
|
|
286
|
+
looks identical to a scraper.
|
|
79
287
|
|
|
80
288
|
## Destination filtering recipes
|
|
81
289
|
|
|
82
|
-
Drop
|
|
290
|
+
Drop everything automated:
|
|
83
291
|
|
|
84
292
|
```sql
|
|
85
293
|
event.user.botScore > 50
|
|
86
294
|
```
|
|
87
295
|
|
|
88
|
-
Drop crawlers but keep
|
|
296
|
+
Drop crawlers but keep the AI traffic a person triggered:
|
|
297
|
+
|
|
298
|
+
```sql
|
|
299
|
+
event.user.botScore > 50 AND event.user.botCategory != 'ai-agent'
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Keep link unfurls, which mean somebody just shared the URL:
|
|
89
303
|
|
|
90
304
|
```sql
|
|
91
|
-
event.user.
|
|
305
|
+
event.user.botCategory != 'link-preview'
|
|
92
306
|
```
|
|
93
307
|
|
|
94
|
-
AI
|
|
308
|
+
AI visibility report:
|
|
95
309
|
|
|
96
310
|
```sql
|
|
97
|
-
event.user.
|
|
311
|
+
event.user.botCategory IN ('ai-agent', 'ai-crawler'), grouped by event.user.botProduct
|
|
98
312
|
```
|
|
99
313
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
- **
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
- **
|
|
114
|
-
|
|
115
|
-
|
|
314
|
+
Cloudflare users: their `bot_score` runs the opposite way (1 = bot) because it
|
|
315
|
+
is a trust score. Ours matches its field name, so higher = more bot.
|
|
316
|
+
|
|
317
|
+
## Not yet implemented
|
|
318
|
+
|
|
319
|
+
- **Identity verification.** `settings.verify` (product to CIDR list), an
|
|
320
|
+
in-package CIDR matcher, and a `botVerified` output. Nothing here can
|
|
321
|
+
currently prove an identity, and a constant-`false` field would assert a check
|
|
322
|
+
that never ran. `Signature-Agent` presence is recorded as an observation only.
|
|
323
|
+
- **Web Bot Auth signature verification.** Needs a JWKS fetch, so it needs an
|
|
324
|
+
answer to async I/O in a transformer first.
|
|
325
|
+
- **Reverse DNS verification.** A DNS round trip would land on client-visible
|
|
326
|
+
latency, since the express source acknowledges synchronously.
|
|
327
|
+
- **ASN / datacenter-IP detection.** Impossible without an embedded database
|
|
328
|
+
(MaxMind GeoLite ASN is CC-BY-SA, incompatible with an MIT package) or a
|
|
329
|
+
network call. Bring-your-own CIDR lists replace it.
|
|
330
|
+
- **TLS / JA4 fingerprinting and header ordering.** The input names `ja4` and
|
|
331
|
+
`headerNames` are reserved and resolved, but nothing consumes them. `method`
|
|
332
|
+
is resolved and currently unconsumed for the same reason: under the `auto`
|
|
333
|
+
rule above there is nothing for a derived context to unlock.
|
|
334
|
+
- **Web-side runtime checks** (`navigator.webdriver`, `userAgentData`). Needs a
|
|
116
335
|
browser source.
|
|
117
|
-
- **Behavioral signals** (rate, session shape)
|
|
118
|
-
- **TLS / JA4 fingerprinting** — not application-layer reachable; would consume
|
|
119
|
-
an upstream-injected `ja4` header if provided.
|
|
120
|
-
- **agentScore graduation** — v1 emits 0 or 95. v1.1 will use intermediate
|
|
121
|
-
values (e.g. 70 for unverified UA claim, 99 for IP-reverse-DNS verified).
|
|
336
|
+
- **Behavioral signals** (rate, session shape). Needs a store.
|
|
122
337
|
|
|
123
338
|
## Limits
|
|
124
339
|
|
|
125
|
-
Will not catch: residential-proxy
|
|
126
|
-
paid CAPTCHA-solver farms
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
340
|
+
Will not catch: residential-proxy plus stealth-patched Chrome with realistic
|
|
341
|
+
behavior; paid CAPTCHA-solver farms; real-browser-as-a-service providers (Bright
|
|
342
|
+
Data, ScrapingBee, Browserbase, Browserless). For that threat model use
|
|
343
|
+
Cloudflare Bot Management, DataDome, or HUMAN.
|
|
344
|
+
|
|
345
|
+
Also undetectable at this layer, by construction: in-browser agents such as
|
|
346
|
+
Claude for Chrome and Microsoft Copilot Actions. They drive a real browser
|
|
347
|
+
session and produce headers identical to the underlying browser, so they
|
|
348
|
+
score 0.
|
|
349
|
+
|
|
350
|
+
A UA-map match is a claim, not proof. Screaming Frog ships Googlebot and Bingbot
|
|
351
|
+
presets, and any client can send any UA. Until identity verification ships,
|
|
352
|
+
treat `botProduct` as "what this client says it is".
|
|
353
|
+
|
|
354
|
+
The graded layer will occasionally mark real people `suspicious`: Android
|
|
355
|
+
WebViews send frozen UA strings with no client hints, enterprise Chrome policies
|
|
356
|
+
freeze the UA major, and UA-spoofing privacy extensions are genuinely
|
|
357
|
+
indistinguishable from UA-spoofing bots at the header layer. That is why those
|
|
358
|
+
signals are graded rather than deterministic, and why the category matters more
|
|
359
|
+
than the number.
|
|
130
360
|
|
|
131
361
|
## License
|
|
132
362
|
|
package/dist/dev.d.mts
CHANGED
|
@@ -103,12 +103,93 @@ declare const SettingsSchema: z.ZodObject<{
|
|
|
103
103
|
value: z.ZodOptional<z.ZodUnknown>;
|
|
104
104
|
fn: z.ZodOptional<z.ZodString>;
|
|
105
105
|
}, z.core.$strip>]>>]>>;
|
|
106
|
+
accept: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
107
|
+
key: z.ZodOptional<z.ZodString>;
|
|
108
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
109
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
110
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
111
|
+
key: z.ZodOptional<z.ZodString>;
|
|
112
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
113
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
114
|
+
}, z.core.$strip>]>>]>>;
|
|
115
|
+
contentType: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
116
|
+
key: z.ZodOptional<z.ZodString>;
|
|
117
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
118
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
119
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
120
|
+
key: z.ZodOptional<z.ZodString>;
|
|
121
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
122
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
123
|
+
}, z.core.$strip>]>>]>>;
|
|
124
|
+
referer: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
125
|
+
key: z.ZodOptional<z.ZodString>;
|
|
126
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
127
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
128
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
129
|
+
key: z.ZodOptional<z.ZodString>;
|
|
130
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
131
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
132
|
+
}, z.core.$strip>]>>]>>;
|
|
133
|
+
signatureAgent: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
134
|
+
key: z.ZodOptional<z.ZodString>;
|
|
135
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
136
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
137
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
138
|
+
key: z.ZodOptional<z.ZodString>;
|
|
139
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
140
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
141
|
+
}, z.core.$strip>]>>]>>;
|
|
142
|
+
method: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
143
|
+
key: z.ZodOptional<z.ZodString>;
|
|
144
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
145
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
146
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
147
|
+
key: z.ZodOptional<z.ZodString>;
|
|
148
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
149
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
150
|
+
}, z.core.$strip>]>>]>>;
|
|
151
|
+
ja4: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
152
|
+
key: z.ZodOptional<z.ZodString>;
|
|
153
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
154
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
155
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
156
|
+
key: z.ZodOptional<z.ZodString>;
|
|
157
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
158
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
159
|
+
}, z.core.$strip>]>>]>>;
|
|
160
|
+
headerNames: z.ZodOptional<z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
161
|
+
key: z.ZodOptional<z.ZodString>;
|
|
162
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
163
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
164
|
+
}, z.core.$strip>]>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
165
|
+
key: z.ZodOptional<z.ZodString>;
|
|
166
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
167
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
168
|
+
}, z.core.$strip>]>>]>>;
|
|
106
169
|
}, z.core.$strip>>;
|
|
107
170
|
output: z.ZodOptional<z.ZodObject<{
|
|
108
|
-
botScore: z.ZodOptional<z.ZodString
|
|
109
|
-
|
|
110
|
-
|
|
171
|
+
botScore: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodLiteral<false>]>>;
|
|
172
|
+
botCategory: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodLiteral<false>]>>;
|
|
173
|
+
botProduct: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodLiteral<false>]>>;
|
|
174
|
+
botReasons: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodLiteral<false>]>>;
|
|
111
175
|
}, z.core.$strip>>;
|
|
176
|
+
context: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
|
|
177
|
+
auto: "auto";
|
|
178
|
+
navigation: "navigation";
|
|
179
|
+
pixel: "pixel";
|
|
180
|
+
beacon: "beacon";
|
|
181
|
+
fetch: "fetch";
|
|
182
|
+
server: "server";
|
|
183
|
+
}>, z.ZodString, z.ZodObject<{
|
|
184
|
+
key: z.ZodOptional<z.ZodString>;
|
|
185
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
186
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
187
|
+
}, z.core.$strip>, z.ZodArray<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
188
|
+
key: z.ZodOptional<z.ZodString>;
|
|
189
|
+
value: z.ZodOptional<z.ZodUnknown>;
|
|
190
|
+
fn: z.ZodOptional<z.ZodString>;
|
|
191
|
+
}, z.core.$strip>]>>]>>;
|
|
192
|
+
suspiciousAt: z.ZodOptional<z.ZodNumber>;
|
|
112
193
|
}, z.core.$strip>;
|
|
113
194
|
type Settings = z.infer<typeof SettingsSchema>;
|
|
114
195
|
|
|
@@ -121,24 +202,42 @@ declare namespace index$1 {
|
|
|
121
202
|
export { type index$1_Settings as Settings, index$1_SettingsSchema as SettingsSchema, index$1_settings as settings };
|
|
122
203
|
}
|
|
123
204
|
|
|
124
|
-
/** Real Chrome
|
|
205
|
+
/** Real Chrome, nothing to report. */
|
|
125
206
|
declare const humanChrome: Flow.StepExample;
|
|
126
|
-
/** GPTBot
|
|
207
|
+
/** GPTBot, an AI training crawler. */
|
|
127
208
|
declare const gptBotCrawler: Flow.StepExample;
|
|
128
|
-
/** ChatGPT-User
|
|
209
|
+
/** ChatGPT-User, an AI agent acting for a person. */
|
|
129
210
|
declare const chatgptUserAgent: Flow.StepExample;
|
|
130
|
-
/**
|
|
211
|
+
/** Googlebot, a search crawler: the one named bot that renders JavaScript. */
|
|
212
|
+
declare const searchCrawler: Flow.StepExample;
|
|
213
|
+
/**
|
|
214
|
+
* The same request, twice, through one instance configured
|
|
215
|
+
* `context: [{ key: "ingest.transport" }, { value: "beacon" }]`.
|
|
216
|
+
*/
|
|
217
|
+
declare const pixelWildcardAccept: Flow.StepExample;
|
|
218
|
+
/** Byte-identical headers, no annotation: the fallback entry decides. */
|
|
219
|
+
declare const unannotatedBeaconFallback: Flow.StepExample;
|
|
220
|
+
/** A Chromium UA whose client hints disagree with it. */
|
|
221
|
+
declare const headerMismatch: Flow.StepExample;
|
|
222
|
+
/** curl, caught by isbot but not named. */
|
|
131
223
|
declare const curlClient: Flow.StepExample;
|
|
132
|
-
/**
|
|
224
|
+
/** No User-Agent on an otherwise wired request. */
|
|
133
225
|
declare const missingUA: Flow.StepExample;
|
|
226
|
+
/** Nothing in ingest at all: not measured, which is not the same as human. */
|
|
227
|
+
declare const unwiredPipeline: Flow.StepExample;
|
|
134
228
|
|
|
135
229
|
declare const step_chatgptUserAgent: typeof chatgptUserAgent;
|
|
136
230
|
declare const step_curlClient: typeof curlClient;
|
|
137
231
|
declare const step_gptBotCrawler: typeof gptBotCrawler;
|
|
232
|
+
declare const step_headerMismatch: typeof headerMismatch;
|
|
138
233
|
declare const step_humanChrome: typeof humanChrome;
|
|
139
234
|
declare const step_missingUA: typeof missingUA;
|
|
235
|
+
declare const step_pixelWildcardAccept: typeof pixelWildcardAccept;
|
|
236
|
+
declare const step_searchCrawler: typeof searchCrawler;
|
|
237
|
+
declare const step_unannotatedBeaconFallback: typeof unannotatedBeaconFallback;
|
|
238
|
+
declare const step_unwiredPipeline: typeof unwiredPipeline;
|
|
140
239
|
declare namespace step {
|
|
141
|
-
export { step_chatgptUserAgent as chatgptUserAgent, step_curlClient as curlClient, step_gptBotCrawler as gptBotCrawler, step_humanChrome as humanChrome, step_missingUA as missingUA };
|
|
240
|
+
export { step_chatgptUserAgent as chatgptUserAgent, step_curlClient as curlClient, step_gptBotCrawler as gptBotCrawler, step_headerMismatch as headerMismatch, step_humanChrome as humanChrome, step_missingUA as missingUA, step_pixelWildcardAccept as pixelWildcardAccept, step_searchCrawler as searchCrawler, step_unannotatedBeaconFallback as unannotatedBeaconFallback, step_unwiredPipeline as unwiredPipeline };
|
|
142
241
|
}
|
|
143
242
|
|
|
144
243
|
declare const index_step: typeof step;
|