@osqd/bothandlerjs 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +227 -1
  2. package/README.md +18 -10
  3. package/dist/adapters/fastify.d.ts +10 -0
  4. package/dist/adapters/index.cjs +38 -10
  5. package/dist/adapters/index.cjs.map +1 -1
  6. package/dist/adapters/index.js +38 -10
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/challenge/index.d.ts +40 -0
  9. package/dist/cli.cjs +2256 -103
  10. package/dist/cli.cjs.map +1 -1
  11. package/dist/cli.js +2256 -103
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts +55 -0
  14. package/dist/core.d.ts +48 -21
  15. package/dist/corpus/index.cjs +365 -7
  16. package/dist/corpus/index.cjs.map +1 -1
  17. package/dist/corpus/index.js +365 -7
  18. package/dist/corpus/index.js.map +1 -1
  19. package/dist/corpus/schema.d.ts +33 -0
  20. package/dist/crawler-ranges.d.ts +31 -0
  21. package/dist/dashboard/client/actions.d.ts +1 -1
  22. package/dist/dashboard/client/app.d.ts +9 -2
  23. package/dist/dashboard/client/boot.d.ts +32 -3
  24. package/dist/dashboard/client/query.d.ts +72 -12
  25. package/dist/dashboard/client/registry.d.ts +25 -0
  26. package/dist/dashboard/client/saved.d.ts +29 -0
  27. package/dist/dashboard/client/store.d.ts +16 -2
  28. package/dist/dashboard/client/types.d.ts +2 -0
  29. package/dist/dashboard/client.generated.d.ts +1 -1
  30. package/dist/dashboard/types.d.ts +15 -0
  31. package/dist/detectors/blended-identity.d.ts +34 -0
  32. package/dist/detectors/challenge-integrity.d.ts +26 -0
  33. package/dist/detectors/challenge-reaction.d.ts +39 -0
  34. package/dist/detectors/clearance.d.ts +1 -23
  35. package/dist/detectors/id-enumeration.d.ts +31 -0
  36. package/dist/detectors/index.d.ts +24 -1
  37. package/dist/detectors/known-bots.d.ts +11 -0
  38. package/dist/detectors/marker.d.ts +106 -0
  39. package/dist/detectors/parameter-sweep.d.ts +39 -0
  40. package/dist/detectors/probe-signature.d.ts +27 -0
  41. package/dist/detectors/probe-volume.d.ts +26 -0
  42. package/dist/detectors/site-baseline.d.ts +135 -0
  43. package/dist/detectors/target-integrity.d.ts +16 -0
  44. package/dist/detectors/transport-coherence.d.ts +31 -0
  45. package/dist/detectors/trap.d.ts +10 -3
  46. package/dist/detectors/types.d.ts +17 -0
  47. package/dist/element/index.cjs +730 -80
  48. package/dist/element/index.cjs.map +1 -1
  49. package/dist/element/index.js +730 -80
  50. package/dist/element/index.js.map +1 -1
  51. package/dist/index.cjs +2044 -123
  52. package/dist/index.cjs.map +1 -1
  53. package/dist/index.d.ts +6 -2
  54. package/dist/index.js +2025 -123
  55. package/dist/index.js.map +1 -1
  56. package/dist/internal/async.d.ts +0 -3
  57. package/dist/internal/ip.d.ts +18 -0
  58. package/dist/internal/text.d.ts +28 -0
  59. package/dist/metrics.d.ts +18 -0
  60. package/dist/probe/index.d.ts +153 -0
  61. package/dist/probe/marker.d.ts +119 -0
  62. package/dist/site/index.d.ts +122 -0
  63. package/dist/state.d.ts +205 -0
  64. package/dist/stores/redis.d.ts +24 -1
  65. package/dist/types.d.ts +106 -0
  66. package/docs/course/05-detectors.md +9 -4
  67. package/docs/course/06-identity.md +1 -1
  68. package/docs/course/16-proving-it.md +15 -9
  69. package/docs/course/index.md +1 -1
  70. package/docs/design/decisions.md +1 -1
  71. package/docs/detection/correlation.md +284 -0
  72. package/docs/detection/detectors.md +259 -1
  73. package/docs/detection/index.md +2 -1
  74. package/docs/detection/shadow-mode.md +147 -0
  75. package/docs/detection/signatures.md +10 -2
  76. package/docs/index.md +3 -2
  77. package/docs/integration/client-ip.md +16 -0
  78. package/docs/operations/dashboard.md +40 -1
  79. package/docs/operations/filters.md +143 -0
  80. package/docs/operations/index.md +1 -0
  81. package/docs/operations/metrics.md +18 -0
  82. package/docs/policy/presets.md +1 -1
  83. package/docs/start/choosing-a-policy.md +1 -1
  84. package/docs/start/first-integration.md +1 -1
  85. package/docs/start/installation.md +2 -2
  86. package/docs/testing/cli.md +7 -1
  87. package/docs/testing/corpus.md +12 -8
  88. package/docs/testing/index.md +1 -1
  89. package/docs/testing/try-it.md +1 -1
  90. package/package.json +4 -1
package/dist/state.d.ts CHANGED
@@ -12,6 +12,7 @@ import type { ActorSnapshot, RequestFacts } from "./types.js";
12
12
  */
13
13
  export declare const MAX_TRACKED_ARRIVALS = 32;
14
14
  export declare const MAX_TRACKED_PATHS = 64;
15
+ export declare const MAX_TRACKED_QUERIES = 64;
15
16
  export declare const MAX_TRACKED_USER_AGENTS = 4;
16
17
  export declare class ActorState {
17
18
  readonly key: string;
@@ -53,6 +54,96 @@ export declare class ActorState {
53
54
  private readonly paths;
54
55
  private pathsOverflowed;
55
56
  private pathsSaturatedAtTotal;
57
+ /**
58
+ * Distinct *parameterised* requests: the path together with its query.
59
+ *
60
+ * Counted apart from `paths` because the two answer different questions and a scraper
61
+ * lives in the gap between them. `/products?page=1` through `?page=200` is one path and
62
+ * two hundred requests, so breadth reads it as somebody rereading a single page — which
63
+ * is exactly what enumerating a catalogue looks like from the path alone.
64
+ */
65
+ private readonly queries;
66
+ private queriesOverflowed;
67
+ /**
68
+ * Which HTTP methods this actor has used.
69
+ *
70
+ * A browser navigating issues GET. Something that has issued nothing but HEAD across a
71
+ * long visit is checking what exists rather than reading it, and that is a fact about
72
+ * the actor rather than about any one of its requests — which is why it is kept here.
73
+ */
74
+ /**
75
+ * What has happened with this actor's marker cookie.
76
+ *
77
+ * Counted rather than listed: the useful questions are all "how often", and a list of
78
+ * marker ids would grow with a client's cookie jar for no benefit. The three drift
79
+ * flags are sticky — once a client has been seen claiming two different browsers under
80
+ * one marker it has done so, and a later request that looks tidy again does not undo
81
+ * it. That is the point of correlating a series rather than judging a request.
82
+ */
83
+ /**
84
+ * When this actor was last challenged, and how it described itself at that moment.
85
+ *
86
+ * Kept so that what a client does *in response* to being challenged can be read. That
87
+ * reaction is better evidence than anything observed passively, because the stimulus
88
+ * was ours: we chose the moment, so a change of identity that follows it within
89
+ * seconds is a reaction to it rather than a coincidence we went looking for.
90
+ */
91
+ /**
92
+ * Answers to challenges that were valid in form but wrong in a way only the series
93
+ * shows: a solution already spent, or one returned faster than the puzzle allows.
94
+ */
95
+ /** Requests for a path no other client had ever asked this site for. */
96
+ private novelPaths;
97
+ private replayedSolutions;
98
+ private implausibleSolves;
99
+ private challengedAt;
100
+ private challengeShape;
101
+ private markerIssues;
102
+ private markerReturns;
103
+ private markerForgeries;
104
+ private driftSeen;
105
+ private driftEvents;
106
+ private readonly methods;
107
+ /**
108
+ * Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
109
+ *
110
+ * Three numbers per shape, deliberately — a count, a lowest and a highest — rather than
111
+ * the ids themselves. What separates enumeration from reading is not which ids were
112
+ * asked for but whether they *cover a range*: thirty requests spanning thirty
113
+ * consecutive ids is a walk, and thirty scattered across a hundred thousand is somebody
114
+ * following links. Both are answerable from a count and a span, and only the count and
115
+ * the span survive an actor asking for ten thousand of them.
116
+ */
117
+ private readonly walks;
118
+ /**
119
+ * Every named identity this actor has claimed, and what kind each was.
120
+ *
121
+ * Kept because the interesting question is not what one request said but what the *set*
122
+ * of them says. One address claiming sqlmap and nikto is a scan; one claiming Googlebot
123
+ * and Bingbot is a forgery, since at most one of those can be true of an address. Neither
124
+ * observation exists inside a single request.
125
+ */
126
+ private readonly identities;
127
+ /**
128
+ * A name somebody gave this actor.
129
+ *
130
+ * Nothing in detection reads it. It exists because an address is not a memory: the
131
+ * person who worked out that `198.51.100.4` is the partner's price feed should be able
132
+ * to write that down where the next person will see it, rather than in a ticket.
133
+ */
134
+ private actorLabel;
135
+ /** Requests from this actor that carried a scanner payload or target. */
136
+ private probePayloads;
137
+ /**
138
+ * What the application answered, for the requests anybody bothered to tell us about.
139
+ *
140
+ * The engine decides *before* the response exists, so this arrives afterwards and only
141
+ * when the adapter reports it. Kept as two counters rather than a list because the one
142
+ * question worth asking is a ratio: an actor whose requests are almost all misses is
143
+ * looking for something rather than reading anything.
144
+ */
145
+ private responsesSeen;
146
+ private missesSeen;
56
147
  private readonly userAgents;
57
148
  constructor(key: string, now: number);
58
149
  /** Records an arrival. Called exactly once per request, by the engine. */
@@ -60,6 +151,102 @@ export declare class ActorState {
60
151
  /** Distinct paths seen, by hash. Saturates at {@link PATH_CAP}; `pathsSaturated` says whether it did. */
61
152
  get distinctPaths(): number;
62
153
  get pathsSaturated(): boolean;
154
+ /** Distinct path-and-query combinations seen. Saturates at {@link QUERY_CAP}. */
155
+ get distinctQueries(): number;
156
+ get queriesSaturated(): boolean;
157
+ /**
158
+ * Records what the application answered. Called after the response, if at all.
159
+ *
160
+ * 404 and 410 only. A 403 is usually this library's own doing and counting it would
161
+ * make the detector that reads this argue with itself; a 500 is the site's problem and
162
+ * says nothing about the client.
163
+ */
164
+ recordOutcome(status: number): void;
165
+ /** Responses reported for this actor. Zero unless something is reporting them. */
166
+ get responses(): number;
167
+ /** Of those, how many were 404 or 410. */
168
+ get misses(): number;
169
+ /** Names this actor, or clears the name when given nothing. Trimmed and bounded. */
170
+ setLabel(label: string | undefined): void;
171
+ get label(): string | undefined;
172
+ /** Records a named identity this actor claimed. Called once per matching signature. */
173
+ noteIdentity(id: string, category: string, verifiable: boolean): void;
174
+ /** Records that this request was for a path the site had never served to anybody. */
175
+ noteNovelPath(): void;
176
+ /** How many of this actor's requests were for a path nobody else had ever asked for. */
177
+ get novelPathCount(): number;
178
+ /** Records something wrong with a submitted solution that only its history reveals. */
179
+ noteChallengeAnomaly(kind: "replay" | "implausible-speed"): void;
180
+ /** Solutions this actor submitted that had already been spent, and ones returned too fast. */
181
+ get challengeAnomalies(): {
182
+ replays: number;
183
+ implausible: number;
184
+ };
185
+ /** Records that a challenge went out, and the identity claimed as it did. */
186
+ noteChallengeIssued(at: number, shape: {
187
+ b: string;
188
+ o: string;
189
+ l: string;
190
+ } | undefined): void;
191
+ /** The moment of the last challenge, and the identity claimed then. `at` is 0 for none. */
192
+ get lastChallenge(): {
193
+ at: number;
194
+ shape: {
195
+ b: string;
196
+ o: string;
197
+ l: string;
198
+ } | undefined;
199
+ };
200
+ /** Records that a marker was handed to this actor on the way out. */
201
+ noteMarkerIssued(): void;
202
+ /** Records what this request's marker cookie turned out to be. */
203
+ noteMarker(returned: boolean, forged: boolean, drift: {
204
+ browser: boolean;
205
+ platform: boolean;
206
+ language: boolean;
207
+ } | undefined): void;
208
+ /** Markers handed to this actor, and how many came back. */
209
+ get markers(): {
210
+ issued: number;
211
+ returned: number;
212
+ forged: number;
213
+ };
214
+ /** Which parts of a claimed identity have ever changed under one marker. */
215
+ get identityDrift(): {
216
+ browser: boolean;
217
+ platform: boolean;
218
+ language: boolean;
219
+ events: number;
220
+ };
221
+ /** Records that this request carried a scanner payload, so later requests can know. */
222
+ notePayloadProbe(): void;
223
+ /** Every identity claimed so far, by id. */
224
+ get claimedIdentities(): ReadonlyMap<string, {
225
+ category: string;
226
+ verifiable: boolean;
227
+ }>;
228
+ /** How many of this actor's requests carried a scanner payload or target. */
229
+ get payloadProbes(): number;
230
+ /**
231
+ * Files a request under the shape of its path, if that path carries a number.
232
+ *
233
+ * The last numeric segment is the one taken to be the identifier: in `/api/v2/orders/42`
234
+ * the version is part of the shape and the order id is what is being walked.
235
+ */
236
+ private noteWalk;
237
+ /**
238
+ * The path shape this actor has walked hardest, with how far it reached.
239
+ *
240
+ * `span` is inclusive of both ends, so a walk of 1 to 30 spans 30. Comparing the count
241
+ * against it is what separates covering a range from visiting a few points in one.
242
+ */
243
+ densestWalk(): {
244
+ template: string;
245
+ count: number;
246
+ span: number;
247
+ } | undefined;
248
+ /** Every HTTP method this actor has used, in first-seen order. */
249
+ get methodsSeen(): readonly string[];
63
250
  /**
64
251
  * Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
65
252
  * still can. Over that many requests the distinct count is exact, so it is the only
@@ -159,3 +346,21 @@ export declare class ActorRegistry {
159
346
  /** Marks an actor as holding valid human clearance until `until`. */
160
347
  clearUntil(key: string, until: number): void;
161
348
  }
349
+ /**
350
+ * One step of a numeric walk: the shape of the path, and the number in it.
351
+ *
352
+ * `/api/v2/orders/42` becomes `/api/v2/orders/#` and `42`. Shared between the per-actor
353
+ * series and the site-wide one, which must agree on what a shape is or their counts
354
+ * cannot be compared.
355
+ *
356
+ * Both bounds are checked without allocating, because both exist to stop a client paying
357
+ * us to do work. `split` allocates a string per segment, so counting separators first is
358
+ * what makes the depth check worth having. Measured on a sixty-segment path: 3.65µs
359
+ * against 199ns for an ordinary one — every request eighteen times dearer, for free,
360
+ * from anyone willing to send a long URL. The concatenation was most of it; the split
361
+ * was the rest.
362
+ */
363
+ export declare function walkStepOf(path: string): {
364
+ template: string;
365
+ id: number;
366
+ } | undefined;
@@ -9,7 +9,6 @@ import type { Clock } from "../internal/clock.js";
9
9
  */
10
10
  export interface RedisLike {
11
11
  incr(key: string): Promise<number>;
12
- pexpire(key: string, ms: number): Promise<unknown>;
13
12
  /**
14
13
  * `SET key value PX ttl [NX]`.
15
14
  *
@@ -51,6 +50,30 @@ export declare class RedisStore implements BotHandlerStore {
51
50
  private readonly prefix;
52
51
  private readonly clock;
53
52
  constructor(client: RedisLike, options?: RedisStoreOptions);
53
+ /**
54
+ * Counts one request into the fixed window `key` is bucketed by.
55
+ *
56
+ * This used to be `INCR`, then `PEXPIRE` when the count came back as 1 — which is
57
+ * correct only if the process survives long enough to send the second command. A
58
+ * deploy, an OOM kill or a dropped connection in between left a counter key behind
59
+ * with no expiry at all, and nothing would ever clean it up: the next request falls
60
+ * into the next bucket, under a different key, so the orphan is never touched again.
61
+ * One per unlucky restart is nothing; the point is that it accumulates forever, in a
62
+ * Redis the operator may well be running with `noeviction`.
63
+ *
64
+ * So the expiry is armed by the command that *creates* the key rather than by a
65
+ * follow-up. `SET … PX … NX` writes the seed only if nothing is there, always with a
66
+ * lifetime, and does nothing at all once the bucket exists — so it neither costs a
67
+ * count nor re-arms a window under load. The `INCR` is issued without waiting for its
68
+ * reply, so both commands are on the wire together and this stays one round trip.
69
+ * Ordering holds because a Redis client writes commands to its connection in call
70
+ * order and Redis executes them in arrival order, which means the key has a lifetime
71
+ * from the instant it exists.
72
+ *
73
+ * Not a Lua script, which would make it a single command: `eval` is the one thing
74
+ * `ioredis` and `node-redis` spell differently enough that this interface could not
75
+ * describe both, and staying client-agnostic is worth more than the last round trip.
76
+ */
54
77
  increment(key: string, windowMs: number): Promise<number>;
55
78
  consumeOnce(key: string, ttlMs: number): Promise<boolean>;
56
79
  get(key: string): Promise<string | undefined>;
package/dist/types.d.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  * detection is a solved problem: probabilistic signals exist, they are useful, and
8
8
  * they are structurally barred from reaching a terminal action.
9
9
  */
10
+ import type { MarkerObservation } from "./probe/index.js";
10
11
  /** What kind of client we believe we are talking to. */
11
12
  export type BotClass =
12
13
  /** Positive evidence of a person driving a real browser. */
@@ -92,6 +93,15 @@ export interface Evidence {
92
93
  deterministicBasis?: string | undefined;
93
94
  /** Structured detail for logs and dashboards. Must be JSON-serialisable. */
94
95
  metadata?: Record<string, unknown> | undefined;
96
+ /**
97
+ * Set when this came from a detector named in {@link BotHandlerConfig.shadowDetectors}.
98
+ *
99
+ * It is a label rather than a mechanism: shadowed evidence never reaches scoring at
100
+ * all, because it is kept in {@link Assessment.shadowEvidence} rather than filtered out
101
+ * of {@link Assessment.evidence} later. The flag is here so that anything rendering the
102
+ * two lists together can say which is which.
103
+ */
104
+ shadow?: true | undefined;
95
105
  }
96
106
  /**
97
107
  * Every verdict and class, in a fixed order.
@@ -128,6 +138,21 @@ export interface RequestFacts {
128
138
  method: string;
129
139
  /** Path only, no query string. Always begins with `/`. */
130
140
  path: string;
141
+ /**
142
+ * The target as the client actually spelled it, present only when that is not how
143
+ * `path` reads.
144
+ *
145
+ * `path` is normalised — decoded once, backslashes and doubled slashes collapsed, dot
146
+ * segments resolved — because a rule scoped to `/admin` has to hold against `/%61dmin`
147
+ * and `/./admin` too. That normalisation is also the only thing that makes an evasive
148
+ * target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml` becomes `/app/config.yml`.
149
+ * This is where the difference is kept, so `target-integrity` can read it.
150
+ *
151
+ * Absent on the overwhelming majority of requests, which is the whole reason it is
152
+ * cheap: its presence already means the target was spelled unusually, though not
153
+ * necessarily suspiciously — a trailing slash is enough.
154
+ */
155
+ rawPath?: string | undefined;
131
156
  /** Decoded query parameters. Null-prototype so `?__proto__=x` is visible, not swallowed. */
132
157
  query: Record<string, string>;
133
158
  /** Lowercased header names to values. Multi-value headers are joined with `, `. */
@@ -178,6 +203,49 @@ export interface ActorSnapshot {
178
203
  requests: number;
179
204
  /** Distinct paths seen inside the window — breadth of crawl. */
180
205
  distinctPaths: number;
206
+ /**
207
+ * Distinct path-and-query combinations seen from this actor.
208
+ *
209
+ * Separate from `distinctPaths` because a scraper lives in the gap between them:
210
+ * `/products?page=1` through `?page=200` is one path and two hundred requests.
211
+ */
212
+ distinctQueries: number;
213
+ /** Whether `distinctQueries` stopped being able to grow. */
214
+ queriesSaturated: boolean;
215
+ /**
216
+ * Every HTTP method this actor has used.
217
+ *
218
+ * A browser navigating issues GET; an actor whose whole visit is HEAD is checking what
219
+ * exists rather than reading it.
220
+ */
221
+ methodsSeen: readonly string[];
222
+ /**
223
+ * Responses reported back for this actor, and how many of them were misses.
224
+ *
225
+ * Both zero unless something calls `recordOutcome` — the engine decides before a
226
+ * response exists, so this is knowledge only the application has.
227
+ */
228
+ responses: number;
229
+ /** Of `responses`, how many were 404 or 410. */
230
+ misses: number;
231
+ /**
232
+ * The path shape this actor has walked hardest — `/user/#` — with how many requests
233
+ * went to it and how wide a range of numbers they covered.
234
+ *
235
+ * Undefined when no path carried a number. `span` is inclusive, so ids 1 to 30 span 30.
236
+ */
237
+ walk?: {
238
+ template: string;
239
+ count: number;
240
+ span: number;
241
+ } | undefined;
242
+ /**
243
+ * A name somebody gave this actor. Never read by detection.
244
+ *
245
+ * An address is not a memory: whoever worked out that one belongs to a partner's price
246
+ * feed should be able to write it where the next person will see it.
247
+ */
248
+ label?: string | undefined;
181
249
  /** First and last sighting, ms since epoch. */
182
250
  firstSeen: number;
183
251
  lastSeen: number;
@@ -219,6 +287,36 @@ export interface Assessment {
219
287
  evidence: Evidence[];
220
288
  /** Every human-pointing observation. These rebut and dampen the score. */
221
289
  humanEvidence: Evidence[];
290
+ /**
291
+ * What the shadowed detectors said, in both directions, and what none of it did.
292
+ *
293
+ * A detector listed in {@link BotHandlerConfig.shadowDetectors} runs exactly as it
294
+ * otherwise would and its findings land here instead of in `evidence` — so they are
295
+ * counted, charted and readable, and they took no part in the verdict, the score, the
296
+ * class, the identity, or any rule. Not "weighted at zero": kept out of the arithmetic
297
+ * altogether, because a `certain` finding does not go through the arithmetic and a
298
+ * weight of zero would not have stopped it.
299
+ *
300
+ * This is how a new detector, or a threshold nobody is sure of yet, is answered with a
301
+ * week of your own traffic rather than with an argument.
302
+ */
303
+ shadowEvidence: Evidence[];
304
+ /**
305
+ * What this assessment would have been had the shadowed detectors been counted.
306
+ *
307
+ * Present only when a shadowed detector actually found something, which is what keeps
308
+ * the second pass off the hot path for the requests it would say nothing about.
309
+ *
310
+ * "It fired 312 times" is not the question anybody has. The question is what turning it
311
+ * on would *do*, and the only honest form of that is the verdict this request would
312
+ * have received — including, and especially, when it is a person.
313
+ */
314
+ shadowVerdict?: {
315
+ verdict: Verdict;
316
+ botClass: BotClass;
317
+ score: number;
318
+ certain: boolean;
319
+ } | undefined;
222
320
  actor: ActorSnapshot;
223
321
  /**
224
322
  * Set when detection was skipped rather than performed. `undefined` means every
@@ -232,6 +330,14 @@ export interface Assessment {
232
330
  /** Detectors that threw or timed out. Detection continues without them. */
233
331
  failures: DetectorFailure[];
234
332
  facts: RequestFacts;
333
+ /**
334
+ * What the marker cookie on this request turned out to be, when the probe is on.
335
+ *
336
+ * An input rather than a conclusion, kept here for the same reason `facts` is: the
337
+ * action path needs it to decide whether this response should carry a new marker, and
338
+ * recomputing it would mean verifying the same HMAC twice on every request.
339
+ */
340
+ marker?: MarkerObservation | undefined;
235
341
  }
236
342
  /** Why an assessment skipped detection. */
237
343
  export type BypassReason =
@@ -1,6 +1,6 @@
1
1
  # Lesson 5 — The detectors
2
2
 
3
- **Goal:** know what each of the twenty detectors reads, what it costs, and what it is
3
+ **Goal:** know what each of the detectors reads, what it costs, and what it is
4
4
  allowed to conclude — then turn one off and watch the score move.
5
5
 
6
6
  ← [Course](index.md) · Prev: [The guard](04-the-guard.md) · Next: [Identity and verification](06-identity.md)
@@ -36,7 +36,7 @@ browsing-coherence cheap always
36
36
  crawler-verification io confirming
37
37
  ```
38
38
 
39
- **Sixteen of twenty ship on by default.** The other four
39
+ **Twenty-two ship on by default.** The others
40
40
  `identity-rotation`, `tls-fingerprint`, `clearance`, `client-signals` — each need something
41
41
  from you, and you will switch three of them on later in the course.
42
42
 
@@ -61,7 +61,7 @@ fails closed is an outage with extra steps.
61
61
 
62
62
  | Detector | Ceiling | Reads |
63
63
  | -------- | ------- | ----- |
64
- | `self-identified` | `certain` | 161 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
64
+ | `self-identified` | `certain` | 205 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
65
65
  | `crawler-verification` | `certain` | Forward-confirmed reverse DNS, or published address ranges. **Confirms and refutes** |
66
66
 
67
67
  ### Single-request consistency
@@ -82,7 +82,12 @@ fails closed is an outage with extra steps.
82
82
  | `rate-anomaly` | `moderate` | Arrivals in a short window. Reports; never concludes |
83
83
  | `cadence` | `moderate` | Coefficient of variation of the gaps. Catches the polite scraper pacing itself *under* your rate limit |
84
84
  | `crawl-breadth` | `weak` | Distinct paths against total requests: reading a site against enumerating it |
85
+ | `parameter-sweep` | `weak` | Distinct query strings against the paths they sit on. Catches the collection that leaves the path unchanged — `?page=1..200` |
85
86
  | `session-integrity` | `moderate` | A "browser" that never carries a cookie |
87
+ | `blended-identity` | `strong` | The *set* of identities one actor claimed: two scanners, or two crawlers that cannot both be true |
88
+ | `id-enumeration` | `moderate` | A contiguous run of numeric ids under one path shape — walking `/user/1..n` rather than following links |
89
+ | `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; every bundled adapter wires it up |
90
+ | `transport-coherence` | `moderate` | The HTTP version and the verbs across a visit: a "Chrome" on HTTP/1.0, a visit made only of HEAD |
86
91
  | `identity-rotation` | `moderate` | One actor, several User-Agents. **Off by default** — under an IP actor key this fires on every corporate NAT |
87
92
  | `browsing-coherence` | `moderate` | The only detector arguing *for* the client. Human-pointing, so it discounts |
88
93
 
@@ -163,7 +168,7 @@ IP-based actor key.
163
168
 
164
169
  ## What you learned
165
170
 
166
- - Sixteen of twenty detectors are on by default; the rest need something from you
171
+ - Twenty-one detectors are on by default; the rest need something from you — a marker probe, a challenge, a site baseline, or data only you have
167
172
  - `cost` and `stage` decide when a detector runs, and confirming work is skipped when
168
173
  nothing claimed an identity
169
174
  - A failing detector is dropped, never the request
@@ -9,7 +9,7 @@ a name alone is worth so little.
9
9
 
10
10
  ## A name is a claim, not a fact
11
11
 
12
- `self-identified` matches 161 signatures across search, AI, SEO, social, monitoring,
12
+ `self-identified` matches 205 signatures across search, AI, SEO, social, monitoring,
13
13
  feeds, archives, security tooling, HTTP libraries and headless runtimes. When it fires you
14
14
  get an `identity` — `"googlebot"`, `"gptbot"` — and a `category`.
15
15
 
@@ -13,29 +13,35 @@ everything into a Serif you can defend.
13
13
 
14
14
  Three tools answer it, and they answer different halves.
15
15
 
16
- ## `check` — against 526 shapes of real traffic
16
+ ## `check` — against 548 shapes of real traffic
17
17
 
18
18
  ```bash
19
19
  npx @osqd/bothandlerjs check --preset protect-content
20
20
  ```
21
21
 
22
22
  ```
23
- protect-content against 522 shapes of real traffic
23
+ protect-content against 539 shapes of real traffic
24
24
 
25
- human 181 cases 3 tag, 175 allow, 3 challenge
26
- benign-bot 144 cases 6 allow, 86 tag, 48 block, 2 challenge, 2 rate-limit
27
- declared-bot 32 cases 18 tag, 2 rate-limit, 3 allow, 9 block
28
- unwanted-bot 105 cases 8 tag, 11 rate-limit, 3 block, 70 challenge, 13 allow
29
- hostile 27 cases 6 challenge, 17 block, 4 allow
30
- infrastructure 33 cases 7 allow, 12 tag, 14 challenge
25
+ human 184 cases 178 allow, 3 tag, 3 challenge
26
+ benign-bot 144 cases 86 tag, 48 block, 6 allow, 2 challenge, 2 rate-limit
27
+ declared-bot 32 cases 18 tag, 9 block, 3 allow, 2 rate-limit
28
+ unwanted-bot 109 cases 71 challenge, 16 allow, 11 rate-limit, 8 tag, 3 block
29
+ hostile 37 cases 20 block, 9 allow, 7 challenge, 1 tag
30
+ infrastructure 33 cases 14 challenge, 12 tag, 7 allow
31
31
 
32
32
  No case marked as a person was denied service.
33
33
  ```
34
34
 
35
+ The corpus holds 548 cases and this run reports 539, which is not a discrepancy: a case
36
+ naming a capability the configuration under test does not provide is *skipped* rather than
37
+ failed. The nine here need a marker probe or a site baseline, and judging a marker case
38
+ against a configuration that issues no markers would be a verdict about nothing. Turn those
39
+ sources on and the same command reports all 548.
40
+
35
41
  **It exits non-zero if any case marked `human` is denied**, which is what makes it a CI step
36
42
  rather than a report.
37
43
 
38
- The corpus is 526 cases with provenance: 182 of them people — 30 browser profiles, 40 in-app
44
+ The corpus is 548 cases with provenance: 186 of them people — 30 browser profiles, 40 in-app
39
45
  WebViews, Tor, screen readers, IE11, a car's infotainment screen, corporate proxies, CGNAT,
40
46
  an author signing in at `/wp-login.php`. Header order is reproduced rather than invented, and
41
47
  DNS is controlled rather than mocked away, so *"the operator's DNS disproves this"* and
@@ -58,7 +58,7 @@ lesson 11.
58
58
 
59
59
  | | | |
60
60
  |-|-|-|
61
- | 5 | [The detectors](05-detectors.md) | All twenty: what each reads, what each costs, what each may conclude. |
61
+ | 5 | [The detectors](05-detectors.md) | All twenty-two: what each reads, what each costs, what each may conclude. |
62
62
  | 6 | [Identity and verification](06-identity.md) | Recognising a crawler by name — and proving or refuting the claim. |
63
63
  | 7 | [Actors and behaviour](07-actors.md) | Who "the same client" is, and what watching one over time tells you. |
64
64
  | 8 | [Traps](08-traps.md) | The one detector that needs no statistics, and how to lay one properly. |
@@ -187,7 +187,7 @@ type checker catches something.
187
187
 
188
188
  ## The corpus is a published entry point, not a test fixture
189
189
 
190
- **The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 526 cases.
190
+ **The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 548 cases.
191
191
 
192
192
  **Why.** The claims in this documentation are only worth anything if you can check them
193
193
  against *your* configuration. A corpus locked inside the test suite proves things about the