@osqd/bothandlerjs 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -1
- package/README.md +18 -10
- package/dist/adapters/fastify.d.ts +10 -0
- package/dist/adapters/index.cjs +38 -13
- package/dist/adapters/index.cjs.map +1 -1
- package/dist/adapters/index.js +38 -13
- package/dist/adapters/index.js.map +1 -1
- package/dist/challenge/index.d.ts +40 -0
- package/dist/cli.cjs +1783 -83
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1783 -83
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +55 -0
- package/dist/core.d.ts +33 -21
- package/dist/corpus/index.cjs +282 -8
- package/dist/corpus/index.cjs.map +1 -1
- package/dist/corpus/index.js +282 -8
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/schema.d.ts +25 -0
- package/dist/dashboard/client/actions.d.ts +1 -1
- package/dist/dashboard/client/app.d.ts +9 -2
- package/dist/dashboard/client/boot.d.ts +32 -3
- package/dist/dashboard/client/query.d.ts +72 -12
- package/dist/dashboard/client/registry.d.ts +25 -0
- package/dist/dashboard/client/saved.d.ts +29 -0
- package/dist/dashboard/client/store.d.ts +16 -2
- package/dist/dashboard/client/types.d.ts +2 -0
- package/dist/dashboard/client.generated.d.ts +1 -1
- package/dist/dashboard/types.d.ts +15 -0
- package/dist/detectors/blended-identity.d.ts +34 -0
- package/dist/detectors/challenge-integrity.d.ts +26 -0
- package/dist/detectors/challenge-reaction.d.ts +39 -0
- package/dist/detectors/clearance.d.ts +1 -23
- package/dist/detectors/index.d.ts +16 -1
- package/dist/detectors/marker.d.ts +106 -0
- package/dist/detectors/probe-signature.d.ts +27 -0
- package/dist/detectors/site-baseline.d.ts +135 -0
- package/dist/detectors/target-integrity.d.ts +16 -0
- package/dist/detectors/trap.d.ts +10 -3
- package/dist/detectors/types.d.ts +17 -0
- package/dist/element/index.cjs +726 -79
- package/dist/element/index.cjs.map +1 -1
- package/dist/element/index.js +726 -79
- package/dist/element/index.js.map +1 -1
- package/dist/index.cjs +1657 -125
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1643 -125
- package/dist/index.js.map +1 -1
- package/dist/internal/async.d.ts +0 -3
- package/dist/internal/ip.d.ts +18 -0
- package/dist/internal/text.d.ts +28 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/probe/index.d.ts +153 -0
- package/dist/probe/marker.d.ts +119 -0
- package/dist/site/index.d.ts +122 -0
- package/dist/state.d.ts +130 -0
- package/dist/stores/redis.d.ts +24 -1
- package/dist/types.d.ts +70 -0
- package/docs/course/05-detectors.md +6 -5
- package/docs/course/06-identity.md +1 -1
- package/docs/course/16-proving-it.md +15 -9
- package/docs/course/index.md +1 -1
- package/docs/design/decisions.md +1 -1
- package/docs/detection/correlation.md +284 -0
- package/docs/detection/detectors.md +139 -3
- package/docs/detection/index.md +2 -1
- package/docs/detection/shadow-mode.md +147 -0
- package/docs/detection/signatures.md +1 -1
- package/docs/index.md +3 -2
- package/docs/integration/client-ip.md +16 -0
- package/docs/operations/dashboard.md +40 -1
- package/docs/operations/filters.md +143 -0
- package/docs/operations/index.md +1 -0
- package/docs/operations/metrics.md +18 -0
- package/docs/policy/presets.md +1 -1
- package/docs/start/choosing-a-policy.md +1 -1
- package/docs/start/first-integration.md +1 -1
- package/docs/start/installation.md +2 -2
- package/docs/testing/cli.md +7 -1
- package/docs/testing/corpus.md +12 -8
- package/docs/testing/index.md +1 -1
- package/docs/testing/try-it.md +1 -1
- package/package.json +4 -1
package/dist/stores/redis.d.ts
CHANGED
|
@@ -9,7 +9,6 @@ import type { Clock } from "../internal/clock.js";
|
|
|
9
9
|
*/
|
|
10
10
|
export interface RedisLike {
|
|
11
11
|
incr(key: string): Promise<number>;
|
|
12
|
-
pexpire(key: string, ms: number): Promise<unknown>;
|
|
13
12
|
/**
|
|
14
13
|
* `SET key value PX ttl [NX]`.
|
|
15
14
|
*
|
|
@@ -51,6 +50,30 @@ export declare class RedisStore implements BotHandlerStore {
|
|
|
51
50
|
private readonly prefix;
|
|
52
51
|
private readonly clock;
|
|
53
52
|
constructor(client: RedisLike, options?: RedisStoreOptions);
|
|
53
|
+
/**
|
|
54
|
+
* Counts one request into the fixed window `key` is bucketed by.
|
|
55
|
+
*
|
|
56
|
+
* This used to be `INCR`, then `PEXPIRE` when the count came back as 1 — which is
|
|
57
|
+
* correct only if the process survives long enough to send the second command. A
|
|
58
|
+
* deploy, an OOM kill or a dropped connection in between left a counter key behind
|
|
59
|
+
* with no expiry at all, and nothing would ever clean it up: the next request falls
|
|
60
|
+
* into the next bucket, under a different key, so the orphan is never touched again.
|
|
61
|
+
* One per unlucky restart is nothing; the point is that it accumulates forever, in a
|
|
62
|
+
* Redis the operator may well be running with `noeviction`.
|
|
63
|
+
*
|
|
64
|
+
* So the expiry is armed by the command that *creates* the key rather than by a
|
|
65
|
+
* follow-up. `SET … PX … NX` writes the seed only if nothing is there, always with a
|
|
66
|
+
* lifetime, and does nothing at all once the bucket exists — so it neither costs a
|
|
67
|
+
* count nor re-arms a window under load. The `INCR` is issued without waiting for its
|
|
68
|
+
* reply, so both commands are on the wire together and this stays one round trip.
|
|
69
|
+
* Ordering holds because a Redis client writes commands to its connection in call
|
|
70
|
+
* order and Redis executes them in arrival order, which means the key has a lifetime
|
|
71
|
+
* from the instant it exists.
|
|
72
|
+
*
|
|
73
|
+
* Not a Lua script, which would make it a single command: `eval` is the one thing
|
|
74
|
+
* `ioredis` and `node-redis` spell differently enough that this interface could not
|
|
75
|
+
* describe both, and staying client-agnostic is worth more than the last round trip.
|
|
76
|
+
*/
|
|
54
77
|
increment(key: string, windowMs: number): Promise<number>;
|
|
55
78
|
consumeOnce(key: string, ttlMs: number): Promise<boolean>;
|
|
56
79
|
get(key: string): Promise<string | undefined>;
|
package/dist/types.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* detection is a solved problem: probabilistic signals exist, they are useful, and
|
|
8
8
|
* they are structurally barred from reaching a terminal action.
|
|
9
9
|
*/
|
|
10
|
+
import type { MarkerObservation } from "./probe/index.js";
|
|
10
11
|
/** What kind of client we believe we are talking to. */
|
|
11
12
|
export type BotClass =
|
|
12
13
|
/** Positive evidence of a person driving a real browser. */
|
|
@@ -92,6 +93,15 @@ export interface Evidence {
|
|
|
92
93
|
deterministicBasis?: string | undefined;
|
|
93
94
|
/** Structured detail for logs and dashboards. Must be JSON-serialisable. */
|
|
94
95
|
metadata?: Record<string, unknown> | undefined;
|
|
96
|
+
/**
|
|
97
|
+
* Set when this came from a detector named in {@link BotHandlerConfig.shadowDetectors}.
|
|
98
|
+
*
|
|
99
|
+
* It is a label rather than a mechanism: shadowed evidence never reaches scoring at
|
|
100
|
+
* all, because it is kept in {@link Assessment.shadowEvidence} rather than filtered out
|
|
101
|
+
* of {@link Assessment.evidence} later. The flag is here so that anything rendering the
|
|
102
|
+
* two lists together can say which is which.
|
|
103
|
+
*/
|
|
104
|
+
shadow?: true | undefined;
|
|
95
105
|
}
|
|
96
106
|
/**
|
|
97
107
|
* Every verdict and class, in a fixed order.
|
|
@@ -128,6 +138,21 @@ export interface RequestFacts {
|
|
|
128
138
|
method: string;
|
|
129
139
|
/** Path only, no query string. Always begins with `/`. */
|
|
130
140
|
path: string;
|
|
141
|
+
/**
|
|
142
|
+
* The target as the client actually spelled it, present only when that is not how
|
|
143
|
+
* `path` reads.
|
|
144
|
+
*
|
|
145
|
+
* `path` is normalised — decoded once, backslashes and doubled slashes collapsed, dot
|
|
146
|
+
* segments resolved — because a rule scoped to `/admin` has to hold against `/%61dmin`
|
|
147
|
+
* and `/./admin` too. That normalisation is also the only thing that makes an evasive
|
|
148
|
+
* target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml` becomes `/app/config.yml`.
|
|
149
|
+
* This is where the difference is kept, so `target-integrity` can read it.
|
|
150
|
+
*
|
|
151
|
+
* Absent on the overwhelming majority of requests, which is the whole reason it is
|
|
152
|
+
* cheap: its presence already means the target was spelled unusually, though not
|
|
153
|
+
* necessarily suspiciously — a trailing slash is enough.
|
|
154
|
+
*/
|
|
155
|
+
rawPath?: string | undefined;
|
|
131
156
|
/** Decoded query parameters. Null-prototype so `?__proto__=x` is visible, not swallowed. */
|
|
132
157
|
query: Record<string, string>;
|
|
133
158
|
/** Lowercased header names to values. Multi-value headers are joined with `, `. */
|
|
@@ -214,6 +239,13 @@ export interface ActorSnapshot {
|
|
|
214
239
|
count: number;
|
|
215
240
|
span: number;
|
|
216
241
|
} | undefined;
|
|
242
|
+
/**
|
|
243
|
+
* A name somebody gave this actor. Never read by detection.
|
|
244
|
+
*
|
|
245
|
+
* An address is not a memory: whoever worked out that one belongs to a partner's price
|
|
246
|
+
* feed should be able to write it where the next person will see it.
|
|
247
|
+
*/
|
|
248
|
+
label?: string | undefined;
|
|
217
249
|
/** First and last sighting, ms since epoch. */
|
|
218
250
|
firstSeen: number;
|
|
219
251
|
lastSeen: number;
|
|
@@ -255,6 +287,36 @@ export interface Assessment {
|
|
|
255
287
|
evidence: Evidence[];
|
|
256
288
|
/** Every human-pointing observation. These rebut and dampen the score. */
|
|
257
289
|
humanEvidence: Evidence[];
|
|
290
|
+
/**
|
|
291
|
+
* What the shadowed detectors said, in both directions, and what none of it did.
|
|
292
|
+
*
|
|
293
|
+
* A detector listed in {@link BotHandlerConfig.shadowDetectors} runs exactly as it
|
|
294
|
+
* otherwise would and its findings land here instead of in `evidence` — so they are
|
|
295
|
+
* counted, charted and readable, and they took no part in the verdict, the score, the
|
|
296
|
+
* class, the identity, or any rule. Not "weighted at zero": kept out of the arithmetic
|
|
297
|
+
* altogether, because a `certain` finding does not go through the arithmetic and a
|
|
298
|
+
* weight of zero would not have stopped it.
|
|
299
|
+
*
|
|
300
|
+
* This is how a new detector, or a threshold nobody is sure of yet, is answered with a
|
|
301
|
+
* week of your own traffic rather than with an argument.
|
|
302
|
+
*/
|
|
303
|
+
shadowEvidence: Evidence[];
|
|
304
|
+
/**
|
|
305
|
+
* What this assessment would have been had the shadowed detectors been counted.
|
|
306
|
+
*
|
|
307
|
+
* Present only when a shadowed detector actually found something, which is what keeps
|
|
308
|
+
* the second pass off the hot path for the requests it would say nothing about.
|
|
309
|
+
*
|
|
310
|
+
* "It fired 312 times" is not the question anybody has. The question is what turning it
|
|
311
|
+
* on would *do*, and the only honest form of that is the verdict this request would
|
|
312
|
+
* have received — including, and especially, when it is a person.
|
|
313
|
+
*/
|
|
314
|
+
shadowVerdict?: {
|
|
315
|
+
verdict: Verdict;
|
|
316
|
+
botClass: BotClass;
|
|
317
|
+
score: number;
|
|
318
|
+
certain: boolean;
|
|
319
|
+
} | undefined;
|
|
258
320
|
actor: ActorSnapshot;
|
|
259
321
|
/**
|
|
260
322
|
* Set when detection was skipped rather than performed. `undefined` means every
|
|
@@ -268,6 +330,14 @@ export interface Assessment {
|
|
|
268
330
|
/** Detectors that threw or timed out. Detection continues without them. */
|
|
269
331
|
failures: DetectorFailure[];
|
|
270
332
|
facts: RequestFacts;
|
|
333
|
+
/**
|
|
334
|
+
* What the marker cookie on this request turned out to be, when the probe is on.
|
|
335
|
+
*
|
|
336
|
+
* An input rather than a conclusion, kept here for the same reason `facts` is: the
|
|
337
|
+
* action path needs it to decide whether this response should carry a new marker, and
|
|
338
|
+
* recomputing it would mean verifying the same HMAC twice on every request.
|
|
339
|
+
*/
|
|
340
|
+
marker?: MarkerObservation | undefined;
|
|
271
341
|
}
|
|
272
342
|
/** Why an assessment skipped detection. */
|
|
273
343
|
export type BypassReason =
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Lesson 5 — The detectors
|
|
2
2
|
|
|
3
|
-
**Goal:** know what each of the
|
|
3
|
+
**Goal:** know what each of the detectors reads, what it costs, and what it is
|
|
4
4
|
allowed to conclude — then turn one off and watch the score move.
|
|
5
5
|
|
|
6
6
|
← [Course](index.md) · Prev: [The guard](04-the-guard.md) · Next: [Identity and verification](06-identity.md)
|
|
@@ -36,7 +36,7 @@ browsing-coherence cheap always
|
|
|
36
36
|
crawler-verification io confirming
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
**
|
|
39
|
+
**Twenty-two ship on by default.** The others —
|
|
40
40
|
`identity-rotation`, `tls-fingerprint`, `clearance`, `client-signals` — each need something
|
|
41
41
|
from you, and you will switch three of them on later in the course.
|
|
42
42
|
|
|
@@ -61,7 +61,7 @@ fails closed is an outage with extra steps.
|
|
|
61
61
|
|
|
62
62
|
| Detector | Ceiling | Reads |
|
|
63
63
|
| -------- | ------- | ----- |
|
|
64
|
-
| `self-identified` | `certain` |
|
|
64
|
+
| `self-identified` | `certain` | 205 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
|
|
65
65
|
| `crawler-verification` | `certain` | Forward-confirmed reverse DNS, or published address ranges. **Confirms and refutes** |
|
|
66
66
|
|
|
67
67
|
### Single-request consistency
|
|
@@ -84,8 +84,9 @@ fails closed is an outage with extra steps.
|
|
|
84
84
|
| `crawl-breadth` | `weak` | Distinct paths against total requests: reading a site against enumerating it |
|
|
85
85
|
| `parameter-sweep` | `weak` | Distinct query strings against the paths they sit on. Catches the collection that leaves the path unchanged — `?page=1..200` |
|
|
86
86
|
| `session-integrity` | `moderate` | A "browser" that never carries a cookie |
|
|
87
|
+
| `blended-identity` | `strong` | The *set* of identities one actor claimed: two scanners, or two crawlers that cannot both be true |
|
|
87
88
|
| `id-enumeration` | `moderate` | A contiguous run of numeric ids under one path shape — walking `/user/1..n` rather than following links |
|
|
88
|
-
| `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`;
|
|
89
|
+
| `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; every bundled adapter wires it up |
|
|
89
90
|
| `transport-coherence` | `moderate` | The HTTP version and the verbs across a visit: a "Chrome" on HTTP/1.0, a visit made only of HEAD |
|
|
90
91
|
| `identity-rotation` | `moderate` | One actor, several User-Agents. **Off by default** — under an IP actor key this fires on every corporate NAT |
|
|
91
92
|
| `browsing-coherence` | `moderate` | The only detector arguing *for* the client. Human-pointing, so it discounts |
|
|
@@ -167,7 +168,7 @@ IP-based actor key.
|
|
|
167
168
|
|
|
168
169
|
## What you learned
|
|
169
170
|
|
|
170
|
-
-
|
|
171
|
+
- Twenty-one detectors are on by default; the rest need something from you — a marker probe, a challenge, a site baseline, or data only you have
|
|
171
172
|
- `cost` and `stage` decide when a detector runs, and confirming work is skipped when
|
|
172
173
|
nothing claimed an identity
|
|
173
174
|
- A failing detector is dropped, never the request
|
|
@@ -9,7 +9,7 @@ a name alone is worth so little.
|
|
|
9
9
|
|
|
10
10
|
## A name is a claim, not a fact
|
|
11
11
|
|
|
12
|
-
`self-identified` matches
|
|
12
|
+
`self-identified` matches 205 signatures across search, AI, SEO, social, monitoring,
|
|
13
13
|
feeds, archives, security tooling, HTTP libraries and headless runtimes. When it fires you
|
|
14
14
|
get an `identity` — `"googlebot"`, `"gptbot"` — and a `category`.
|
|
15
15
|
|
|
@@ -13,29 +13,35 @@ everything into a Serif you can defend.
|
|
|
13
13
|
|
|
14
14
|
Three tools answer it, and they answer different halves.
|
|
15
15
|
|
|
16
|
-
## `check` — against
|
|
16
|
+
## `check` — against 548 shapes of real traffic
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
19
|
npx @osqd/bothandlerjs check --preset protect-content
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
```
|
|
23
|
-
protect-content against
|
|
23
|
+
protect-content against 539 shapes of real traffic
|
|
24
24
|
|
|
25
|
-
human
|
|
26
|
-
benign-bot 144 cases
|
|
27
|
-
declared-bot 32 cases 18 tag,
|
|
28
|
-
unwanted-bot
|
|
29
|
-
hostile
|
|
30
|
-
infrastructure 33 cases
|
|
25
|
+
human 184 cases 178 allow, 3 tag, 3 challenge
|
|
26
|
+
benign-bot 144 cases 86 tag, 48 block, 6 allow, 2 challenge, 2 rate-limit
|
|
27
|
+
declared-bot 32 cases 18 tag, 9 block, 3 allow, 2 rate-limit
|
|
28
|
+
unwanted-bot 109 cases 71 challenge, 16 allow, 11 rate-limit, 8 tag, 3 block
|
|
29
|
+
hostile 37 cases 20 block, 9 allow, 7 challenge, 1 tag
|
|
30
|
+
infrastructure 33 cases 14 challenge, 12 tag, 7 allow
|
|
31
31
|
|
|
32
32
|
No case marked as a person was denied service.
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
The corpus holds 548 cases and this run reports 539, which is not a discrepancy: a case
|
|
36
|
+
naming a capability the configuration under test does not provide is *skipped* rather than
|
|
37
|
+
failed. The nine here need a marker probe or a site baseline, and judging a marker case
|
|
38
|
+
against a configuration that issues no markers would be a verdict about nothing. Turn those
|
|
39
|
+
sources on and the same command reports all 548.
|
|
40
|
+
|
|
35
41
|
**It exits non-zero if any case marked `human` is denied**, which is what makes it a CI step
|
|
36
42
|
rather than a report.
|
|
37
43
|
|
|
38
|
-
The corpus is
|
|
44
|
+
The corpus is 548 cases with provenance: 186 of them people — 30 browser profiles, 40 in-app
|
|
39
45
|
WebViews, Tor, screen readers, IE11, a car's infotainment screen, corporate proxies, CGNAT,
|
|
40
46
|
an author signing in at `/wp-login.php`. Header order is reproduced rather than invented, and
|
|
41
47
|
DNS is controlled rather than mocked away, so *"the operator's DNS disproves this"* and
|
package/docs/course/index.md
CHANGED
|
@@ -58,7 +58,7 @@ lesson 11.
|
|
|
58
58
|
|
|
59
59
|
| | | |
|
|
60
60
|
|-|-|-|
|
|
61
|
-
| 5 | [The detectors](05-detectors.md) | All twenty: what each reads, what each costs, what each may conclude. |
|
|
61
|
+
| 5 | [The detectors](05-detectors.md) | All twenty-two: what each reads, what each costs, what each may conclude. |
|
|
62
62
|
| 6 | [Identity and verification](06-identity.md) | Recognising a crawler by name — and proving or refuting the claim. |
|
|
63
63
|
| 7 | [Actors and behaviour](07-actors.md) | Who "the same client" is, and what watching one over time tells you. |
|
|
64
64
|
| 8 | [Traps](08-traps.md) | The one detector that needs no statistics, and how to lay one properly. |
|
package/docs/design/decisions.md
CHANGED
|
@@ -187,7 +187,7 @@ type checker catches something.
|
|
|
187
187
|
|
|
188
188
|
## The corpus is a published entry point, not a test fixture
|
|
189
189
|
|
|
190
|
-
**The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all
|
|
190
|
+
**The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 548 cases.
|
|
191
191
|
|
|
192
192
|
**Why.** The claims in this documentation are only worth anything if you can check them
|
|
193
193
|
against *your* configuration. A corpus locked inside the test suite proves things about the
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# Correlating a client's own requests
|
|
2
|
+
|
|
3
|
+
Most of this library reads one request. A smaller and more valuable part reads a
|
|
4
|
+
*series* — what an actor has done across many requests — because the tells that matter
|
|
5
|
+
most are not visible in any single one. A wordlist scan is a hundred ordinary-looking
|
|
6
|
+
404s. A scrape is a thousand ordinary-looking page loads. Nothing in any one of those
|
|
7
|
+
requests is remarkable; the shape of all of them together is.
|
|
8
|
+
|
|
9
|
+
This page is about the join: what makes two requests "the same client", and what each
|
|
10
|
+
kind of join is worth.
|
|
11
|
+
|
|
12
|
+
## The problem with joining on an address
|
|
13
|
+
|
|
14
|
+
Every cross-request detector needs to decide which requests belong together, and until
|
|
15
|
+
recently there was only one way to decide it — the **actor key**, derived from the
|
|
16
|
+
client address. It is available on every request, it costs nothing, and it is wrong in
|
|
17
|
+
both directions:
|
|
18
|
+
|
|
19
|
+
- **It merges people who are unrelated.** An office, a school, a household and a mobile
|
|
20
|
+
carrier all put many people behind one address. Anything inferred about "the actor" is
|
|
21
|
+
really about a crowd.
|
|
22
|
+
- **It splits a client that is one thing.** A scraper on a rotating proxy pool is a new
|
|
23
|
+
actor every few requests, and a phone changing networks is a new actor several times a
|
|
24
|
+
day.
|
|
25
|
+
|
|
26
|
+
Both errors have teeth. The first is how a library ends up denying somebody for a
|
|
27
|
+
stranger's behaviour. The second is how a scraper walks straight past every per-actor
|
|
28
|
+
threshold by changing address more often than the threshold counts.
|
|
29
|
+
|
|
30
|
+
One detector was left unwritten for exactly this reason. `identityRotationDetector` —
|
|
31
|
+
the client that arrives as Chrome, then as curl, then as Googlebot — fires on any
|
|
32
|
+
address fronting several browsers, which describes every corporate network on the
|
|
33
|
+
internet. It ships, but it is not installed by default, and the note in
|
|
34
|
+
`defaultDetectors()` says why: it is only safe once your actor key is narrower than an
|
|
35
|
+
address.
|
|
36
|
+
|
|
37
|
+
## The marker
|
|
38
|
+
|
|
39
|
+
A marker is a signed cookie this server issues and reads back. It is off by default.
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
new BotHandler({
|
|
43
|
+
probe: { secrets: [process.env.MARKER_SECRET] },
|
|
44
|
+
});
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Two requests carrying the same marker came from the same client. Not the same address,
|
|
48
|
+
not the same network — the same browser profile, because the marker holds an HMAC only
|
|
49
|
+
this server can produce and only that client received. That single fact is what makes
|
|
50
|
+
the rest of this page possible, and it is what turns "a client at this address claimed
|
|
51
|
+
two identities" into "this client claimed two identities".
|
|
52
|
+
|
|
53
|
+
It contains a random id, a validity window, and three short hashes standing for the
|
|
54
|
+
identity claimed when it was issued. It carries no identifier of a person, is
|
|
55
|
+
first-party, is `HttpOnly`, and expires on its own. Like every token here it is **signed
|
|
56
|
+
and not encrypted**, so nothing secret may go in one.
|
|
57
|
+
|
|
58
|
+
### What it costs
|
|
59
|
+
|
|
60
|
+
Verifying a marker is an HMAC, and a session presents the same cookie on every request,
|
|
61
|
+
so successful verifications are cached — with expiry re-checked on each hit, and failures
|
|
62
|
+
never cached, since caching those would let anyone fill the cache with unique junk. The
|
|
63
|
+
probe costs roughly 5% of an assessment with a marker held, and nothing at all when the
|
|
64
|
+
client holds none.
|
|
65
|
+
|
|
66
|
+
Network fan-out is sketched into 128 bits per marker rather than remembered as a set of
|
|
67
|
+
addresses, which measured at 55.6 MB with both caps full. The estimate carries a few
|
|
68
|
+
percent of error either way — 16 real networks read as 17 — so `marker-fanout`'s threshold
|
|
69
|
+
is a soft boundary. With everything full the probe holds well under a megabyte.
|
|
70
|
+
|
|
71
|
+
A `Set-Cookie` makes a response uncacheable by most shared caches, so the probe issues a
|
|
72
|
+
marker **only when the client is not already holding a valid one** — for an ordinary
|
|
73
|
+
visitor, the first request of a session and no other. That is also why the marker is not
|
|
74
|
+
reissued to refresh it, and why verified crawlers are never issued one at all: Googlebot
|
|
75
|
+
keeps no cookies, so a marker sent to it is a header that never comes back.
|
|
76
|
+
|
|
77
|
+
### Secrets
|
|
78
|
+
|
|
79
|
+
`secrets` is required, and deliberately has no default. A secret generated at startup
|
|
80
|
+
would read every marker minted by another replica — or by this one before a restart — as
|
|
81
|
+
*forged*, turning the strongest signal here into a machine for accusing ordinary
|
|
82
|
+
visitors. The first secret signs and all of them verify, so rotation is a prepend
|
|
83
|
+
followed by a removal one marker lifetime later.
|
|
84
|
+
|
|
85
|
+
Rotation does not disturb anything. The identity hashes inside a marker are derived under
|
|
86
|
+
a fixed salt rather than under the signing secret, precisely so that prepending a key
|
|
87
|
+
does not silently re-describe every visitor as a different browser.
|
|
88
|
+
|
|
89
|
+
## What the marker makes visible
|
|
90
|
+
|
|
91
|
+
| Detector | Ceiling | Reads |
|
|
92
|
+
| --- | --- | --- |
|
|
93
|
+
| `identity-drift` | `strong` | The identity claimed now against the one claimed when the marker was issued |
|
|
94
|
+
| `marker-integrity` | `strong` | A marker presented with a signature this server could not have produced |
|
|
95
|
+
| `marker-fanout` | `moderate` | Distinct networks one marker has been presented from |
|
|
96
|
+
| `marker-persistence` | `moderate` | A client that sends cookies but never returns the one this server set |
|
|
97
|
+
| `challenge-reaction` | `strong` | What a client did in the seconds after it was challenged |
|
|
98
|
+
| `challenge-integrity` | `moderate` | Solutions replayed, or returned faster than the puzzle allows |
|
|
99
|
+
|
|
100
|
+
All of them are installed automatically when `probe` is configured and are absent
|
|
101
|
+
otherwise, because a marker nobody issued is a marker nobody can fail to return.
|
|
102
|
+
|
|
103
|
+
### Identity drift, and why the parts are weighed separately
|
|
104
|
+
|
|
105
|
+
A **browser family** that changes — Chrome to curl, Firefox to Googlebot — has no benign
|
|
106
|
+
reading. Software does not change what it is, so one of the two claims is false and the
|
|
107
|
+
evidence is `strong`.
|
|
108
|
+
|
|
109
|
+
A **platform** that changes does have a benign reading, and a common one: "Request
|
|
110
|
+
desktop site" on a phone rewrites the User-Agent to claim a desktop, and the person doing
|
|
111
|
+
it is a person. A language changes when somebody changes their language. Those are
|
|
112
|
+
reported at `moderate` and named as the soft case in the summary, or turned off:
|
|
113
|
+
|
|
114
|
+
```ts
|
|
115
|
+
identityDriftDetector({ reportSoftDrift: false });
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Not returning the marker, and who that describes
|
|
119
|
+
|
|
120
|
+
`marker-persistence` deliberately says nothing about a client that sends **no** cookies at
|
|
121
|
+
all. That client is `session-integrity`'s business, and it already reports it at a weight
|
|
122
|
+
chosen for the people who produce it — people who block cookies. Having both speak is one
|
|
123
|
+
observation counted twice, landing on exactly that population: measured on the corpus, the
|
|
124
|
+
overlapping version took the `cookies-blocked` case from 21 to 38 and put +24 on five
|
|
125
|
+
ordinary browsing sessions.
|
|
126
|
+
|
|
127
|
+
So this asks the narrower question only a marker can answer — the client is demonstrably
|
|
128
|
+
keeping cookies, and ours is not among them — and the two share an evidence `family`, so
|
|
129
|
+
even where both apply the stronger stands rather than the two summing.
|
|
130
|
+
|
|
131
|
+
### Reaction beats observation
|
|
132
|
+
|
|
133
|
+
`challenge-reaction` is the strongest idea here, and the reason is structural. Every
|
|
134
|
+
other detector reads traffic that would have happened anyway and argues backwards from
|
|
135
|
+
it. This one reads a response to a stimulus **we chose**: we decided when the challenge
|
|
136
|
+
went out, so a client that changes what it claims to be within seconds of receiving one
|
|
137
|
+
is reacting to it. There was no reason to look at that moment except that we created it.
|
|
138
|
+
|
|
139
|
+
It reports two things — a changed identity, and never answering at all across repeated
|
|
140
|
+
asks. The first is `strong` when a marker ties the two requests together and `moderate`
|
|
141
|
+
when only the address does, because that is genuinely how much less an address-based join
|
|
142
|
+
is worth. The second is capped at `moderate` forever: a person with JavaScript disabled
|
|
143
|
+
produces it every time, and they are a person.
|
|
144
|
+
|
|
145
|
+
## Comparing a client with everybody else
|
|
146
|
+
|
|
147
|
+
The marker answers "is this the same client". A different set of questions needs the
|
|
148
|
+
opposite comparison — not this client against itself, but this client against the rest of
|
|
149
|
+
your traffic. It is also off by default:
|
|
150
|
+
|
|
151
|
+
```ts
|
|
152
|
+
new BotHandler({
|
|
153
|
+
site: { warmupRequests: 5000 },
|
|
154
|
+
});
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
| Detector | Ceiling | Reads |
|
|
158
|
+
| --- | --- | --- |
|
|
159
|
+
| `distributed-walk` | `moderate` | A numeric range walked across many clients, none of which walks enough of it alone |
|
|
160
|
+
| `path-novelty` | `moderate` | A client whose requests are almost all for paths nobody else has asked for |
|
|
161
|
+
| `miss-baseline` | `moderate` | This client's miss rate against the site's own |
|
|
162
|
+
| `path-campaign` | `moderate` | A path the site never served that many unrelated clients suddenly want |
|
|
163
|
+
|
|
164
|
+
### Warmup is the whole safety story
|
|
165
|
+
|
|
166
|
+
Nothing is reported until `warmupRequests` have been observed, and that number is the most
|
|
167
|
+
important setting here. A baseline is a claim about what is normal, and a claim drawn from
|
|
168
|
+
four hundred requests is not one: on a quiet site at three in the morning *every* path is
|
|
169
|
+
one nobody else has asked for, because nothing has been asked for. A profile consulted
|
|
170
|
+
early does not merely fail — it fails confidently, about everybody. The failure mode of the
|
|
171
|
+
whole module is silence, which is the correct direction for something whose mistakes land
|
|
172
|
+
on all your visitors at once.
|
|
173
|
+
|
|
174
|
+
The same reasoning caps every detector here at `moderate`. A baseline is wrong exactly when
|
|
175
|
+
a site is most unusual: the day of a redesign, the hour a campaign lands, the migration
|
|
176
|
+
that leaves half the URLs missing.
|
|
177
|
+
|
|
178
|
+
### What each one is really for
|
|
179
|
+
|
|
180
|
+
`distributed-walk` addresses the one threat per-actor thresholds miss **by construction**.
|
|
181
|
+
Split an id range across five hundred addresses at one request a minute each and every
|
|
182
|
+
actor is unremarkable, `id-enumeration` never fires for anybody, and the range is still
|
|
183
|
+
walked end to end. It is only visible in the union. What separates it from a busy shop is
|
|
184
|
+
that enumeration *covers* a contiguous range and visits each id about once, while real
|
|
185
|
+
readers cluster on popular items and return to them — so coverage and the revisit ratio
|
|
186
|
+
must both agree, and either alone would report an ordinary catalogue.
|
|
187
|
+
|
|
188
|
+
`path-novelty` is a self-maintaining wordlist. A wordlist is a list of paths that exist on
|
|
189
|
+
*some* sites; on yours most of them do not exist and nobody has ever asked for them. It
|
|
190
|
+
catches the scanner whose list is newer than the one this library ships.
|
|
191
|
+
|
|
192
|
+
`path-campaign` is its inverse, and catches what it misses. A freshly disclosed
|
|
193
|
+
vulnerability looks like one URL nobody had ever requested being requested by hundreds of
|
|
194
|
+
unrelated clients within the hour — each of them making a single request, which is nothing
|
|
195
|
+
at all on its own. The miss rate is **required** rather than optional here, because many
|
|
196
|
+
clients arriving at once on a brand-new URL is also exactly what a successful launch looks
|
|
197
|
+
like. What separates them is whether the site had anything to serve.
|
|
198
|
+
|
|
199
|
+
**The known cost of `path-campaign`** is a broken link. Somebody shares a URL with a typo
|
|
200
|
+
and thousands of real people follow it within the hour, which from the server is a path
|
|
201
|
+
the site has never served, requested by many unrelated clients, answered `not found` every
|
|
202
|
+
time — the firing shape exactly. Those people are reported at `moderate` and never
|
|
203
|
+
refused; the corpus carries the case (`broken-link-shared-widely`) and holds it to the
|
|
204
|
+
never-deny guarantee under every shipped preset.
|
|
205
|
+
|
|
206
|
+
**`distributed-walk` needs both of its bounds.** The revisit ratio is checked from above
|
|
207
|
+
*and* below, and the lower bound is what makes it usable on a real site. Ids spread across
|
|
208
|
+
a six-figure catalogue coarsen the bitmap until one bucket stands for hundreds of ids;
|
|
209
|
+
ordinary browsing then touches nearly every bucket, so coverage reads 1.0 and the
|
|
210
|
+
estimated id count runs far ahead of the requests that were actually made. Requiring the
|
|
211
|
+
visits to account for the ids claimed is what rejects an estimate that has left the
|
|
212
|
+
evidence behind — measured on a simulated shop, sixty long-tail shoppers were reported
|
|
213
|
+
before that bound existed and none after.
|
|
214
|
+
|
|
215
|
+
`miss-baseline` is `probe-volume` done relative. A fixed 80% threshold reports everybody on
|
|
216
|
+
a site mid-migration and stays silent on a tidy one where a client missing a third of the
|
|
217
|
+
time is remarkable.
|
|
218
|
+
|
|
219
|
+
### What it costs
|
|
220
|
+
|
|
221
|
+
Everything is bounded, and the bounds are the interesting part, because a client picks its
|
|
222
|
+
own paths and therefore picks how much there is to remember. Walk ids are held as a
|
|
223
|
+
**1024-bit map over the range rather than as a set of numbers**: a set measured at 45 MB
|
|
224
|
+
for a table anybody could fill on purpose by requesting `/anything/1`, while the bitmap is
|
|
225
|
+
128 bytes however wide the range grows, coarsening rather than growing. With every table at
|
|
226
|
+
its cap the profile holds about 17 MB, nearly all of it the path table, and costs roughly
|
|
227
|
+
3% of an assessment.
|
|
228
|
+
|
|
229
|
+
The state is kept in process like the rest of the behavioural series, so across replicas
|
|
230
|
+
each sees its own share of the traffic. That understates every count here, and understating
|
|
231
|
+
costs a missed detection rather than an accusation.
|
|
232
|
+
|
|
233
|
+
### What the assessment carries
|
|
234
|
+
|
|
235
|
+
An `Assessment` exposes `marker`: the reading, the drift, and the identity shape. It is
|
|
236
|
+
there because the action path needs it to decide whether a response should carry a new
|
|
237
|
+
marker, and because it is genuinely useful to an operator looking at one request.
|
|
238
|
+
|
|
239
|
+
What leaves the machine is less than that. Notification sinks receive a **redacted** copy:
|
|
240
|
+
`redactEvent` runs on the way out and is on by default, and it drops `facts.cookies`,
|
|
241
|
+
strips the credential headers — `cookie`, `authorization`, `x-api-key` among them — masks
|
|
242
|
+
the address and the actor key to a `/24`, and masks query values. A marker is reduced the
|
|
243
|
+
same way: what a sink sees is whether one was presented, whether it verified, whether the
|
|
244
|
+
identity moved, and how many networks it has come from. The claims inside it, the marker id
|
|
245
|
+
included, do not go. They are the decoded contents of a cookie, and the rule about cookies
|
|
246
|
+
already covered them.
|
|
247
|
+
|
|
248
|
+
The unredacted assessment is what your own process holds — `handle()` returns it, and the
|
|
249
|
+
dashboard renders it for an operator who is already inside your perimeter. Turning
|
|
250
|
+
redaction off (`notifications: { redaction: false }`) is the one way to send more than the
|
|
251
|
+
above, and it is worth knowing what you are choosing when you do.
|
|
252
|
+
|
|
253
|
+
## What this deliberately does not do
|
|
254
|
+
|
|
255
|
+
**It does not link two clients by how alike they look.** Joining strangers on a shared
|
|
256
|
+
fingerprint and letting one's verdict raise the other's is guilt by association, and when
|
|
257
|
+
the link is wrong it denies a person for a stranger's behaviour. Every join here is the
|
|
258
|
+
same actor, or the same marker — something this server issued — and never a statistical
|
|
259
|
+
resemblance between two clients.
|
|
260
|
+
|
|
261
|
+
The site detectors are the one place evidence about *other* traffic reaches a verdict, and
|
|
262
|
+
they are shaped by that. Each one describes a pattern the client in front of you genuinely
|
|
263
|
+
took part in — it requested that path, it walked that range — rather than importing
|
|
264
|
+
somebody else's verdict, and each is capped where it cannot deny anybody alone.
|
|
265
|
+
|
|
266
|
+
**It does not build a profile.** The marker holds no identifier of a person, is not
|
|
267
|
+
readable across sites, and expires. Nothing here is retained to describe a visitor; it is
|
|
268
|
+
retained to describe a *series of requests*, and it ages out with the series.
|
|
269
|
+
|
|
270
|
+
**It does not reach `certain`.** Nothing on this page can. `certain` means no benign
|
|
271
|
+
explanation exists, and every signal here joins two requests — with the join itself being
|
|
272
|
+
the thing that could be wrong. A tampered marker comes closest and still stops at
|
|
273
|
+
`strong`, because a middlebox can mangle a cookie in transit and that is not the client's
|
|
274
|
+
fault.
|
|
275
|
+
|
|
276
|
+
## Related
|
|
277
|
+
|
|
278
|
+
- [Shadow mode](shadow-mode.md) — run any of these against your own traffic for a week
|
|
279
|
+
before it is allowed to decide anything. Several of them fire at `moderate` on real
|
|
280
|
+
people by design, and whether the thresholds are right *here* is not a thing this
|
|
281
|
+
library can know.
|
|
282
|
+
- [Detectors](detectors.md) — the full catalogue and what each one is worth.
|
|
283
|
+
- [The client IP](../integration/client-ip.md) — why the actor key is what it is.
|
|
284
|
+
- [Writing a detector](writing-a-detector.md) — including what a detector may see.
|