@osqd/bothandlerjs 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +227 -1
  2. package/README.md +18 -10
  3. package/dist/adapters/fastify.d.ts +10 -0
  4. package/dist/adapters/index.cjs +38 -13
  5. package/dist/adapters/index.cjs.map +1 -1
  6. package/dist/adapters/index.js +38 -13
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/challenge/index.d.ts +40 -0
  9. package/dist/cli.cjs +1783 -83
  10. package/dist/cli.cjs.map +1 -1
  11. package/dist/cli.js +1783 -83
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts +55 -0
  14. package/dist/core.d.ts +33 -21
  15. package/dist/corpus/index.cjs +282 -8
  16. package/dist/corpus/index.cjs.map +1 -1
  17. package/dist/corpus/index.js +282 -8
  18. package/dist/corpus/index.js.map +1 -1
  19. package/dist/corpus/schema.d.ts +25 -0
  20. package/dist/dashboard/client/actions.d.ts +1 -1
  21. package/dist/dashboard/client/app.d.ts +9 -2
  22. package/dist/dashboard/client/boot.d.ts +32 -3
  23. package/dist/dashboard/client/query.d.ts +72 -12
  24. package/dist/dashboard/client/registry.d.ts +25 -0
  25. package/dist/dashboard/client/saved.d.ts +29 -0
  26. package/dist/dashboard/client/store.d.ts +16 -2
  27. package/dist/dashboard/client/types.d.ts +2 -0
  28. package/dist/dashboard/client.generated.d.ts +1 -1
  29. package/dist/dashboard/types.d.ts +15 -0
  30. package/dist/detectors/blended-identity.d.ts +34 -0
  31. package/dist/detectors/challenge-integrity.d.ts +26 -0
  32. package/dist/detectors/challenge-reaction.d.ts +39 -0
  33. package/dist/detectors/clearance.d.ts +1 -23
  34. package/dist/detectors/index.d.ts +16 -1
  35. package/dist/detectors/marker.d.ts +106 -0
  36. package/dist/detectors/probe-signature.d.ts +27 -0
  37. package/dist/detectors/site-baseline.d.ts +135 -0
  38. package/dist/detectors/target-integrity.d.ts +16 -0
  39. package/dist/detectors/trap.d.ts +10 -3
  40. package/dist/detectors/types.d.ts +17 -0
  41. package/dist/element/index.cjs +726 -79
  42. package/dist/element/index.cjs.map +1 -1
  43. package/dist/element/index.js +726 -79
  44. package/dist/element/index.js.map +1 -1
  45. package/dist/index.cjs +1657 -125
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.ts +4 -0
  48. package/dist/index.js +1643 -125
  49. package/dist/index.js.map +1 -1
  50. package/dist/internal/async.d.ts +0 -3
  51. package/dist/internal/ip.d.ts +18 -0
  52. package/dist/internal/text.d.ts +28 -0
  53. package/dist/metrics.d.ts +18 -0
  54. package/dist/probe/index.d.ts +153 -0
  55. package/dist/probe/marker.d.ts +119 -0
  56. package/dist/site/index.d.ts +122 -0
  57. package/dist/state.d.ts +130 -0
  58. package/dist/stores/redis.d.ts +24 -1
  59. package/dist/types.d.ts +70 -0
  60. package/docs/course/05-detectors.md +6 -5
  61. package/docs/course/06-identity.md +1 -1
  62. package/docs/course/16-proving-it.md +15 -9
  63. package/docs/course/index.md +1 -1
  64. package/docs/design/decisions.md +1 -1
  65. package/docs/detection/correlation.md +284 -0
  66. package/docs/detection/detectors.md +139 -3
  67. package/docs/detection/index.md +2 -1
  68. package/docs/detection/shadow-mode.md +147 -0
  69. package/docs/detection/signatures.md +1 -1
  70. package/docs/index.md +3 -2
  71. package/docs/integration/client-ip.md +16 -0
  72. package/docs/operations/dashboard.md +40 -1
  73. package/docs/operations/filters.md +143 -0
  74. package/docs/operations/index.md +1 -0
  75. package/docs/operations/metrics.md +18 -0
  76. package/docs/policy/presets.md +1 -1
  77. package/docs/start/choosing-a-policy.md +1 -1
  78. package/docs/start/first-integration.md +1 -1
  79. package/docs/start/installation.md +2 -2
  80. package/docs/testing/cli.md +7 -1
  81. package/docs/testing/corpus.md +12 -8
  82. package/docs/testing/index.md +1 -1
  83. package/docs/testing/try-it.md +1 -1
  84. package/package.json +4 -1
@@ -9,7 +9,6 @@ import type { Clock } from "../internal/clock.js";
9
9
  */
10
10
  export interface RedisLike {
11
11
  incr(key: string): Promise<number>;
12
- pexpire(key: string, ms: number): Promise<unknown>;
13
12
  /**
14
13
  * `SET key value PX ttl [NX]`.
15
14
  *
@@ -51,6 +50,30 @@ export declare class RedisStore implements BotHandlerStore {
51
50
  private readonly prefix;
52
51
  private readonly clock;
53
52
  constructor(client: RedisLike, options?: RedisStoreOptions);
53
+ /**
54
+ * Counts one request into the fixed window `key` is bucketed by.
55
+ *
56
+ * This used to be `INCR`, then `PEXPIRE` when the count came back as 1 — which is
57
+ * correct only if the process survives long enough to send the second command. A
58
+ * deploy, an OOM kill or a dropped connection in between left a counter key behind
59
+ * with no expiry at all, and nothing would ever clean it up: the next request falls
60
+ * into the next bucket, under a different key, so the orphan is never touched again.
61
+ * One per unlucky restart is nothing; the point is that it accumulates forever, in a
62
+ * Redis the operator may well be running with `noeviction`.
63
+ *
64
+ * So the expiry is armed by the command that *creates* the key rather than by a
65
+ * follow-up. `SET … PX … NX` writes the seed only if nothing is there, always with a
66
+ * lifetime, and does nothing at all once the bucket exists — so it neither costs a
67
+ * count nor re-arms a window under load. The `INCR` is issued without waiting for its
68
+ * reply, so both commands are on the wire together and this stays one round trip.
69
+ * Ordering holds because a Redis client writes commands to its connection in call
70
+ * order and Redis executes them in arrival order, which means the key has a lifetime
71
+ * from the instant it exists.
72
+ *
73
+ * Not a Lua script, which would make it a single command: `eval` is the one thing
74
+ * `ioredis` and `node-redis` spell differently enough that this interface could not
75
+ * describe both, and staying client-agnostic is worth more than the last round trip.
76
+ */
54
77
  increment(key: string, windowMs: number): Promise<number>;
55
78
  consumeOnce(key: string, ttlMs: number): Promise<boolean>;
56
79
  get(key: string): Promise<string | undefined>;
package/dist/types.d.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  * detection is a solved problem: probabilistic signals exist, they are useful, and
8
8
  * they are structurally barred from reaching a terminal action.
9
9
  */
10
+ import type { MarkerObservation } from "./probe/index.js";
10
11
  /** What kind of client we believe we are talking to. */
11
12
  export type BotClass =
12
13
  /** Positive evidence of a person driving a real browser. */
@@ -92,6 +93,15 @@ export interface Evidence {
92
93
  deterministicBasis?: string | undefined;
93
94
  /** Structured detail for logs and dashboards. Must be JSON-serialisable. */
94
95
  metadata?: Record<string, unknown> | undefined;
96
+ /**
97
+ * Set when this came from a detector named in {@link BotHandlerConfig.shadowDetectors}.
98
+ *
99
+ * It is a label rather than a mechanism: shadowed evidence never reaches scoring at
100
+ * all, because it is kept in {@link Assessment.shadowEvidence} rather than filtered out
101
+ * of {@link Assessment.evidence} later. The flag is here so that anything rendering the
102
+ * two lists together can say which is which.
103
+ */
104
+ shadow?: true | undefined;
95
105
  }
96
106
  /**
97
107
  * Every verdict and class, in a fixed order.
@@ -128,6 +138,21 @@ export interface RequestFacts {
128
138
  method: string;
129
139
  /** Path only, no query string. Always begins with `/`. */
130
140
  path: string;
141
+ /**
142
+ * The target as the client actually spelled it, present only when that is not how
143
+ * `path` reads.
144
+ *
145
+ * `path` is normalised — decoded once, backslashes and doubled slashes collapsed, dot
146
+ * segments resolved — because a rule scoped to `/admin` has to hold against `/%61dmin`
147
+ * and `/./admin` too. That normalisation is also the only thing that makes an evasive
148
+ * target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml` becomes `/app/config.yml`.
149
+ * This is where the difference is kept, so `target-integrity` can read it.
150
+ *
151
+ * Absent on the overwhelming majority of requests, which is the whole reason it is
152
+ * cheap: its presence already means the target was spelled unusually, though not
153
+ * necessarily suspiciously — a trailing slash is enough.
154
+ */
155
+ rawPath?: string | undefined;
131
156
  /** Decoded query parameters. Null-prototype so `?__proto__=x` is visible, not swallowed. */
132
157
  query: Record<string, string>;
133
158
  /** Lowercased header names to values. Multi-value headers are joined with `, `. */
@@ -214,6 +239,13 @@ export interface ActorSnapshot {
214
239
  count: number;
215
240
  span: number;
216
241
  } | undefined;
242
+ /**
243
+ * A name somebody gave this actor. Never read by detection.
244
+ *
245
+ * An address is not a memory: whoever worked out that one belongs to a partner's price
246
+ * feed should be able to write it where the next person will see it.
247
+ */
248
+ label?: string | undefined;
217
249
  /** First and last sighting, ms since epoch. */
218
250
  firstSeen: number;
219
251
  lastSeen: number;
@@ -255,6 +287,36 @@ export interface Assessment {
255
287
  evidence: Evidence[];
256
288
  /** Every human-pointing observation. These rebut and dampen the score. */
257
289
  humanEvidence: Evidence[];
290
+ /**
291
+ * What the shadowed detectors said, in both directions, and what none of it did.
292
+ *
293
+ * A detector listed in {@link BotHandlerConfig.shadowDetectors} runs exactly as it
294
+ * otherwise would and its findings land here instead of in `evidence` — so they are
295
+ * counted, charted and readable, and they took no part in the verdict, the score, the
296
+ * class, the identity, or any rule. Not "weighted at zero": kept out of the arithmetic
297
+ * altogether, because a `certain` finding does not go through the arithmetic and a
298
+ * weight of zero would not have stopped it.
299
+ *
300
+ * This is how a new detector, or a threshold nobody is sure of yet, is answered with a
301
+ * week of your own traffic rather than with an argument.
302
+ */
303
+ shadowEvidence: Evidence[];
304
+ /**
305
+ * What this assessment would have been had the shadowed detectors been counted.
306
+ *
307
+ * Present only when a shadowed detector actually found something, which is what keeps
308
+ * the second pass off the hot path for the requests it would say nothing about.
309
+ *
310
+ * "It fired 312 times" is not the question anybody has. The question is what turning it
311
+ * on would *do*, and the only honest form of that is the verdict this request would
312
+ * have received — including, and especially, when it is a person.
313
+ */
314
+ shadowVerdict?: {
315
+ verdict: Verdict;
316
+ botClass: BotClass;
317
+ score: number;
318
+ certain: boolean;
319
+ } | undefined;
258
320
  actor: ActorSnapshot;
259
321
  /**
260
322
  * Set when detection was skipped rather than performed. `undefined` means every
@@ -268,6 +330,14 @@ export interface Assessment {
268
330
  /** Detectors that threw or timed out. Detection continues without them. */
269
331
  failures: DetectorFailure[];
270
332
  facts: RequestFacts;
333
+ /**
334
+ * What the marker cookie on this request turned out to be, when the probe is on.
335
+ *
336
+ * An input rather than a conclusion, kept here for the same reason `facts` is: the
337
+ * action path needs it to decide whether this response should carry a new marker, and
338
+ * recomputing it would mean verifying the same HMAC twice on every request.
339
+ */
340
+ marker?: MarkerObservation | undefined;
271
341
  }
272
342
  /** Why an assessment skipped detection. */
273
343
  export type BypassReason =
@@ -1,6 +1,6 @@
1
1
  # Lesson 5 — The detectors
2
2
 
3
- **Goal:** know what each of the twenty detectors reads, what it costs, and what it is
3
+ **Goal:** know what each of the detectors reads, what it costs, and what it is
4
4
  allowed to conclude — then turn one off and watch the score move.
5
5
 
6
6
  ← [Course](index.md) · Prev: [The guard](04-the-guard.md) · Next: [Identity and verification](06-identity.md)
@@ -36,7 +36,7 @@ browsing-coherence cheap always
36
36
  crawler-verification io confirming
37
37
  ```
38
38
 
39
- **Sixteen of twenty ship on by default.** The other four
39
+ **Twenty-two ship on by default.** The others
40
40
  `identity-rotation`, `tls-fingerprint`, `clearance`, `client-signals` — each need something
41
41
  from you, and you will switch three of them on later in the course.
42
42
 
@@ -61,7 +61,7 @@ fails closed is an outage with extra steps.
61
61
 
62
62
  | Detector | Ceiling | Reads |
63
63
  | -------- | ------- | ----- |
64
- | `self-identified` | `certain` | 161 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
64
+ | `self-identified` | `certain` | 205 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
65
65
  | `crawler-verification` | `certain` | Forward-confirmed reverse DNS, or published address ranges. **Confirms and refutes** |
66
66
 
67
67
  ### Single-request consistency
@@ -84,8 +84,9 @@ fails closed is an outage with extra steps.
84
84
  | `crawl-breadth` | `weak` | Distinct paths against total requests: reading a site against enumerating it |
85
85
  | `parameter-sweep` | `weak` | Distinct query strings against the paths they sit on. Catches the collection that leaves the path unchanged — `?page=1..200` |
86
86
  | `session-integrity` | `moderate` | A "browser" that never carries a cookie |
87
+ | `blended-identity` | `strong` | The *set* of identities one actor claimed: two scanners, or two crawlers that cannot both be true |
87
88
  | `id-enumeration` | `moderate` | A contiguous run of numeric ids under one path shape — walking `/user/1..n` rather than following links |
88
- | `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; the Node adapter wires it up |
89
+ | `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; every bundled adapter wires it up |
89
90
  | `transport-coherence` | `moderate` | The HTTP version and the verbs across a visit: a "Chrome" on HTTP/1.0, a visit made only of HEAD |
90
91
  | `identity-rotation` | `moderate` | One actor, several User-Agents. **Off by default** — under an IP actor key this fires on every corporate NAT |
91
92
  | `browsing-coherence` | `moderate` | The only detector arguing *for* the client. Human-pointing, so it discounts |
@@ -167,7 +168,7 @@ IP-based actor key.
167
168
 
168
169
  ## What you learned
169
170
 
170
- - Sixteen of twenty detectors are on by default; the rest need something from you
171
+ - Twenty-one detectors are on by default; the rest need something from you — a marker probe, a challenge, a site baseline, or data only you have
171
172
  - `cost` and `stage` decide when a detector runs, and confirming work is skipped when
172
173
  nothing claimed an identity
173
174
  - A failing detector is dropped, never the request
@@ -9,7 +9,7 @@ a name alone is worth so little.
9
9
 
10
10
  ## A name is a claim, not a fact
11
11
 
12
- `self-identified` matches 161 signatures across search, AI, SEO, social, monitoring,
12
+ `self-identified` matches 205 signatures across search, AI, SEO, social, monitoring,
13
13
  feeds, archives, security tooling, HTTP libraries and headless runtimes. When it fires you
14
14
  get an `identity` — `"googlebot"`, `"gptbot"` — and a `category`.
15
15
 
@@ -13,29 +13,35 @@ everything into a Serif you can defend.
13
13
 
14
14
  Three tools answer it, and they answer different halves.
15
15
 
16
- ## `check` — against 526 shapes of real traffic
16
+ ## `check` — against 548 shapes of real traffic
17
17
 
18
18
  ```bash
19
19
  npx @osqd/bothandlerjs check --preset protect-content
20
20
  ```
21
21
 
22
22
  ```
23
- protect-content against 522 shapes of real traffic
23
+ protect-content against 539 shapes of real traffic
24
24
 
25
- human 181 cases 3 tag, 175 allow, 3 challenge
26
- benign-bot 144 cases 6 allow, 86 tag, 48 block, 2 challenge, 2 rate-limit
27
- declared-bot 32 cases 18 tag, 2 rate-limit, 3 allow, 9 block
28
- unwanted-bot 105 cases 8 tag, 11 rate-limit, 3 block, 70 challenge, 13 allow
29
- hostile 27 cases 6 challenge, 17 block, 4 allow
30
- infrastructure 33 cases 7 allow, 12 tag, 14 challenge
25
+ human 184 cases 178 allow, 3 tag, 3 challenge
26
+ benign-bot 144 cases 86 tag, 48 block, 6 allow, 2 challenge, 2 rate-limit
27
+ declared-bot 32 cases 18 tag, 9 block, 3 allow, 2 rate-limit
28
+ unwanted-bot 109 cases 71 challenge, 16 allow, 11 rate-limit, 8 tag, 3 block
29
+ hostile 37 cases 20 block, 9 allow, 7 challenge, 1 tag
30
+ infrastructure 33 cases 14 challenge, 12 tag, 7 allow
31
31
 
32
32
  No case marked as a person was denied service.
33
33
  ```
34
34
 
35
+ The corpus holds 548 cases and this run reports 539, which is not a discrepancy: a case
36
+ naming a capability the configuration under test does not provide is *skipped* rather than
37
+ failed. The nine here need a marker probe or a site baseline, and judging a marker case
38
+ against a configuration that issues no markers would be a verdict about nothing. Turn those
39
+ sources on and the same command reports all 548.
40
+
35
41
  **It exits non-zero if any case marked `human` is denied**, which is what makes it a CI step
36
42
  rather than a report.
37
43
 
38
- The corpus is 526 cases with provenance: 182 of them people — 30 browser profiles, 40 in-app
44
+ The corpus is 548 cases with provenance: 186 of them people — 30 browser profiles, 40 in-app
39
45
  WebViews, Tor, screen readers, IE11, a car's infotainment screen, corporate proxies, CGNAT,
40
46
  an author signing in at `/wp-login.php`. Header order is reproduced rather than invented, and
41
47
  DNS is controlled rather than mocked away, so *"the operator's DNS disproves this"* and
@@ -58,7 +58,7 @@ lesson 11.
58
58
 
59
59
  | | | |
60
60
  |-|-|-|
61
- | 5 | [The detectors](05-detectors.md) | All twenty: what each reads, what each costs, what each may conclude. |
61
+ | 5 | [The detectors](05-detectors.md) | All twenty-two: what each reads, what each costs, what each may conclude. |
62
62
  | 6 | [Identity and verification](06-identity.md) | Recognising a crawler by name — and proving or refuting the claim. |
63
63
  | 7 | [Actors and behaviour](07-actors.md) | Who "the same client" is, and what watching one over time tells you. |
64
64
  | 8 | [Traps](08-traps.md) | The one detector that needs no statistics, and how to lay one properly. |
@@ -187,7 +187,7 @@ type checker catches something.
187
187
 
188
188
  ## The corpus is a published entry point, not a test fixture
189
189
 
190
- **The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 526 cases.
190
+ **The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 548 cases.
191
191
 
192
192
  **Why.** The claims in this documentation are only worth anything if you can check them
193
193
  against *your* configuration. A corpus locked inside the test suite proves things about the
@@ -0,0 +1,284 @@
1
+ # Correlating a client's own requests
2
+
3
+ Most of this library reads one request. A smaller and more valuable part reads a
4
+ *series* — what an actor has done across many requests — because the tells that matter
5
+ most are not visible in any single one. A wordlist scan is a hundred ordinary-looking
6
+ 404s. A scrape is a thousand ordinary-looking page loads. Nothing in any one of those
7
+ requests is remarkable; the shape of all of them together is.
8
+
9
+ This page is about the join: what makes two requests "the same client", and what each
10
+ kind of join is worth.
11
+
12
+ ## The problem with joining on an address
13
+
14
+ Every cross-request detector needs to decide which requests belong together, and until
15
+ recently there was only one way to decide it — the **actor key**, derived from the
16
+ client address. It is available on every request, it costs nothing, and it is wrong in
17
+ both directions:
18
+
19
+ - **It merges people who are unrelated.** An office, a school, a household and a mobile
20
+ carrier all put many people behind one address. Anything inferred about "the actor" is
21
+ really about a crowd.
22
+ - **It splits a client that is one thing.** A scraper on a rotating proxy pool is a new
23
+ actor every few requests, and a phone changing networks is a new actor several times a
24
+ day.
25
+
26
+ Both errors have teeth. The first is how a library ends up denying somebody for a
27
+ stranger's behaviour. The second is how a scraper walks straight past every per-actor
28
+ threshold by changing address more often than the threshold counts.
29
+
30
+ One detector was left unwritten for exactly this reason. `identityRotationDetector` —
31
+ the client that arrives as Chrome, then as curl, then as Googlebot — fires on any
32
+ address fronting several browsers, which describes every corporate network on the
33
+ internet. It ships, but it is not installed by default, and the note in
34
+ `defaultDetectors()` says why: it is only safe once your actor key is narrower than an
35
+ address.
36
+
37
+ ## The marker
38
+
39
+ A marker is a signed cookie this server issues and reads back. It is off by default.
40
+
41
+ ```ts
42
+ new BotHandler({
43
+ probe: { secrets: [process.env.MARKER_SECRET] },
44
+ });
45
+ ```
46
+
47
+ Two requests carrying the same marker came from the same client. Not the same address,
48
+ not the same network — the same browser profile, because the marker holds an HMAC only
49
+ this server can produce and only that client received. That single fact is what makes
50
+ the rest of this page possible, and it is what turns "a client at this address claimed
51
+ two identities" into "this client claimed two identities".
52
+
53
+ It contains a random id, a validity window, and three short hashes standing for the
54
+ identity claimed when it was issued. It carries no identifier of a person, is
55
+ first-party, is `HttpOnly`, and expires on its own. Like every token here it is **signed
56
+ and not encrypted**, so nothing secret may go in one.
57
+
58
+ ### What it costs
59
+
60
+ Verifying a marker is an HMAC, and a session presents the same cookie on every request,
61
+ so successful verifications are cached — with expiry re-checked on each hit, and failures
62
+ never cached, since caching those would let anyone fill the cache with unique junk. The
63
+ probe costs roughly 5% of an assessment with a marker held, and nothing at all when the
64
+ client holds none.
65
+
66
+ Network fan-out is sketched into 128 bits per marker rather than remembered as a set of
67
+ addresses, which measured at 55.6 MB with both caps full. The estimate carries a few
68
+ percent of error either way — 16 real networks read as 17 — so `marker-fanout`'s threshold
69
+ is a soft boundary. With everything full the probe holds well under a megabyte.
70
+
71
+ A `Set-Cookie` makes a response uncacheable by most shared caches, so the probe issues a
72
+ marker **only when the client is not already holding a valid one** — for an ordinary
73
+ visitor, the first request of a session and no other. That is also why the marker is not
74
+ reissued to refresh it, and why verified crawlers are never issued one at all: Googlebot
75
+ keeps no cookies, so a marker sent to it is a header that never comes back.
76
+
77
+ ### Secrets
78
+
79
+ `secrets` is required, and deliberately has no default. A secret generated at startup
80
+ would read every marker minted by another replica — or by this one before a restart — as
81
+ *forged*, turning the strongest signal here into a machine for accusing ordinary
82
+ visitors. The first secret signs and all of them verify, so rotation is a prepend
83
+ followed by a removal one marker lifetime later.
84
+
85
+ Rotation does not disturb anything. The identity hashes inside a marker are derived under
86
+ a fixed salt rather than under the signing secret, precisely so that prepending a key
87
+ does not silently re-describe every visitor as a different browser.
88
+
89
+ ## What the marker makes visible
90
+
91
+ | Detector | Ceiling | Reads |
92
+ | --- | --- | --- |
93
+ | `identity-drift` | `strong` | The identity claimed now against the one claimed when the marker was issued |
94
+ | `marker-integrity` | `strong` | A marker presented with a signature this server could not have produced |
95
+ | `marker-fanout` | `moderate` | Distinct networks one marker has been presented from |
96
+ | `marker-persistence` | `moderate` | A client that sends cookies but never returns the one this server set |
97
+ | `challenge-reaction` | `strong` | What a client did in the seconds after it was challenged |
98
+ | `challenge-integrity` | `moderate` | Solutions replayed, or returned faster than the puzzle allows |
99
+
100
+ All of them are installed automatically when `probe` is configured and are absent
101
+ otherwise, because a marker nobody issued is a marker nobody can fail to return.
102
+
103
+ ### Identity drift, and why the parts are weighed separately
104
+
105
+ A **browser family** that changes — Chrome to curl, Firefox to Googlebot — has no benign
106
+ reading. Software does not change what it is, so one of the two claims is false and the
107
+ evidence is `strong`.
108
+
109
+ A **platform** that changes does have a benign reading, and a common one: "Request
110
+ desktop site" on a phone rewrites the User-Agent to claim a desktop, and the person doing
111
+ it is a person. A language changes when somebody changes their language. Those are
112
+ reported at `moderate` and named as the soft case in the summary, or turned off:
113
+
114
+ ```ts
115
+ identityDriftDetector({ reportSoftDrift: false });
116
+ ```
117
+
118
+ ### Not returning the marker, and who that describes
119
+
120
+ `marker-persistence` deliberately says nothing about a client that sends **no** cookies at
121
+ all. That client is `session-integrity`'s business, and it already reports it at a weight
122
+ chosen for the people who produce it — people who block cookies. Having both speak is one
123
+ observation counted twice, landing on exactly that population: measured on the corpus, the
124
+ overlapping version took the `cookies-blocked` case from 21 to 38 and put +24 on five
125
+ ordinary browsing sessions.
126
+
127
+ So this asks the narrower question only a marker can answer — the client is demonstrably
128
+ keeping cookies, and ours is not among them — and the two share an evidence `family`, so
129
+ even where both apply the stronger stands rather than the two summing.
130
+
131
+ ### Reaction beats observation
132
+
133
+ `challenge-reaction` is the strongest idea here, and the reason is structural. Every
134
+ other detector reads traffic that would have happened anyway and argues backwards from
135
+ it. This one reads a response to a stimulus **we chose**: we decided when the challenge
136
+ went out, so a client that changes what it claims to be within seconds of receiving one
137
+ is reacting to it. There was no reason to look at that moment except that we created it.
138
+
139
+ It reports two things — a changed identity, and never answering at all across repeated
140
+ asks. The first is `strong` when a marker ties the two requests together and `moderate`
141
+ when only the address does, because that is genuinely how much less an address-based join
142
+ is worth. The second is capped at `moderate` forever: a person with JavaScript disabled
143
+ produces it every time, and they are a person.
144
+
145
+ ## Comparing a client with everybody else
146
+
147
+ The marker answers "is this the same client". A different set of questions needs the
148
+ opposite comparison — not this client against itself, but this client against the rest of
149
+ your traffic. It is also off by default:
150
+
151
+ ```ts
152
+ new BotHandler({
153
+ site: { warmupRequests: 5000 },
154
+ });
155
+ ```
156
+
157
+ | Detector | Ceiling | Reads |
158
+ | --- | --- | --- |
159
+ | `distributed-walk` | `moderate` | A numeric range walked across many clients, none of which walks enough of it alone |
160
+ | `path-novelty` | `moderate` | A client whose requests are almost all for paths nobody else has asked for |
161
+ | `miss-baseline` | `moderate` | This client's miss rate against the site's own |
162
+ | `path-campaign` | `moderate` | A path the site never served that many unrelated clients suddenly want |
163
+
164
+ ### Warmup is the whole safety story
165
+
166
+ Nothing is reported until `warmupRequests` have been observed, and that number is the most
167
+ important setting here. A baseline is a claim about what is normal, and a claim drawn from
168
+ four hundred requests is not one: on a quiet site at three in the morning *every* path is
169
+ one nobody else has asked for, because nothing has been asked for. A profile consulted
170
+ early does not merely fail — it fails confidently, about everybody. The failure mode of the
171
+ whole module is silence, which is the correct direction for something whose mistakes land
172
+ on all your visitors at once.
173
+
174
+ The same reasoning caps every detector here at `moderate`. A baseline is wrong exactly when
175
+ a site is most unusual: the day of a redesign, the hour a campaign lands, the migration
176
+ that leaves half the URLs missing.
177
+
178
+ ### What each one is really for
179
+
180
+ `distributed-walk` addresses the one threat per-actor thresholds miss **by construction**.
181
+ Split an id range across five hundred addresses at one request a minute each and every
182
+ actor is unremarkable, `id-enumeration` never fires for anybody, and the range is still
183
+ walked end to end. It is only visible in the union. What separates it from a busy shop is
184
+ that enumeration *covers* a contiguous range and visits each id about once, while real
185
+ readers cluster on popular items and return to them — so coverage and the revisit ratio
186
+ must both agree, and either alone would report an ordinary catalogue.
187
+
188
+ `path-novelty` is a self-maintaining wordlist. A wordlist is a list of paths that exist on
189
+ *some* sites; on yours most of them do not exist and nobody has ever asked for them. It
190
+ catches the scanner whose list is newer than the one this library ships.
191
+
192
+ `path-campaign` is its inverse, and catches what it misses. A freshly disclosed
193
+ vulnerability looks like one URL nobody had ever requested being requested by hundreds of
194
+ unrelated clients within the hour — each of them making a single request, which is nothing
195
+ at all on its own. The miss rate is **required** rather than optional here, because many
196
+ clients arriving at once on a brand-new URL is also exactly what a successful launch looks
197
+ like. What separates them is whether the site had anything to serve.
198
+
199
+ **The known cost of `path-campaign`** is a broken link. Somebody shares a URL with a typo
200
+ and thousands of real people follow it within the hour, which from the server is a path
201
+ the site has never served, requested by many unrelated clients, answered `not found` every
202
+ time — the firing shape exactly. Those people are reported at `moderate` and never
203
+ refused; the corpus carries the case (`broken-link-shared-widely`) and holds it to the
204
+ never-deny guarantee under every shipped preset.
205
+
206
+ **`distributed-walk` needs both of its bounds.** The revisit ratio is checked from above
207
+ *and* below, and the lower bound is what makes it usable on a real site. Ids spread across
208
+ a six-figure catalogue coarsen the bitmap until one bucket stands for hundreds of ids;
209
+ ordinary browsing then touches nearly every bucket, so coverage reads 1.0 and the
210
+ estimated id count runs far ahead of the requests that were actually made. Requiring the
211
+ visits to account for the ids claimed is what rejects an estimate that has left the
212
+ evidence behind — measured on a simulated shop, sixty long-tail shoppers were reported
213
+ before that bound existed and none after.
214
+
215
+ `miss-baseline` is `probe-volume` done relative. A fixed 80% threshold reports everybody on
216
+ a site mid-migration and stays silent on a tidy one where a client missing a third of the
217
+ time is remarkable.
218
+
219
+ ### What it costs
220
+
221
+ Everything is bounded, and the bounds are the interesting part, because a client picks its
222
+ own paths and therefore picks how much there is to remember. Walk ids are held as a
223
+ **1024-bit map over the range rather than as a set of numbers**: a set measured at 45 MB
224
+ for a table anybody could fill on purpose by requesting `/anything/1`, while the bitmap is
225
+ 128 bytes however wide the range grows, coarsening rather than growing. With every table at
226
+ its cap the profile holds about 17 MB, nearly all of it the path table, and costs roughly
227
+ 3% of an assessment.
228
+
229
+ The state is kept in process like the rest of the behavioural series, so across replicas
230
+ each sees its own share of the traffic. That understates every count here, and understating
231
+ costs a missed detection rather than an accusation.
232
+
233
+ ### What the assessment carries
234
+
235
+ An `Assessment` exposes `marker`: the reading, the drift, and the identity shape. It is
236
+ there because the action path needs it to decide whether a response should carry a new
237
+ marker, and because it is genuinely useful to an operator looking at one request.
238
+
239
+ What leaves the machine is less than that. Notification sinks receive a **redacted** copy:
240
+ `redactEvent` runs on the way out and is on by default, and it drops `facts.cookies`,
241
+ strips the credential headers — `cookie`, `authorization`, `x-api-key` among them — masks
242
+ the address and the actor key to a `/24`, and masks query values. A marker is reduced the
243
+ same way: what a sink sees is whether one was presented, whether it verified, whether the
244
+ identity moved, and how many networks it has come from. The claims inside it, the marker id
245
+ included, do not go. They are the decoded contents of a cookie, and the rule about cookies
246
+ already covered them.
247
+
248
+ The unredacted assessment is what your own process holds — `handle()` returns it, and the
249
+ dashboard renders it for an operator who is already inside your perimeter. Turning
250
+ redaction off (`notifications: { redaction: false }`) is the one way to send more than the
251
+ above, and it is worth knowing what you are choosing when you do.
252
+
253
+ ## What this deliberately does not do
254
+
255
+ **It does not link two clients by how alike they look.** Joining strangers on a shared
256
+ fingerprint and letting one's verdict raise the other's is guilt by association, and when
257
+ the link is wrong it denies a person for a stranger's behaviour. Every join here is the
258
+ same actor, or the same marker — something this server issued — and never a statistical
259
+ resemblance between two clients.
260
+
261
+ The site detectors are the one place evidence about *other* traffic reaches a verdict, and
262
+ they are shaped by that. Each one describes a pattern the client in front of you genuinely
263
+ took part in — it requested that path, it walked that range — rather than importing
264
+ somebody else's verdict, and each is capped where it cannot deny anybody alone.
265
+
266
+ **It does not build a profile.** The marker holds no identifier of a person, is not
267
+ readable across sites, and expires. Nothing here is retained to describe a visitor; it is
268
+ retained to describe a *series of requests*, and it ages out with the series.
269
+
270
+ **It does not reach `certain`.** Nothing on this page can. `certain` means no benign
271
+ explanation exists, and every signal here joins two requests — with the join itself being
272
+ the thing that could be wrong. A tampered marker comes closest and still stops at
273
+ `strong`, because a middlebox can mangle a cookie in transit and that is not the client's
274
+ fault.
275
+
276
+ ## Related
277
+
278
+ - [Shadow mode](shadow-mode.md) — run any of these against your own traffic for a week
279
+ before it is allowed to decide anything. Several of them fire at `moderate` on real
280
+ people by design, and whether the thresholds are right *here* is not a thing this
281
+ library can know.
282
+ - [Detectors](detectors.md) — the full catalogue and what each one is worth.
283
+ - [The client IP](../integration/client-ip.md) — why the actor key is what it is.
284
+ - [Writing a detector](writing-a-detector.md) — including what a detector may see.