@osqd/bothandlerjs 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -1
- package/README.md +18 -10
- package/dist/adapters/fastify.d.ts +10 -0
- package/dist/adapters/index.cjs +38 -10
- package/dist/adapters/index.cjs.map +1 -1
- package/dist/adapters/index.js +38 -10
- package/dist/adapters/index.js.map +1 -1
- package/dist/challenge/index.d.ts +40 -0
- package/dist/cli.cjs +2256 -103
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2256 -103
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +55 -0
- package/dist/core.d.ts +48 -21
- package/dist/corpus/index.cjs +365 -7
- package/dist/corpus/index.cjs.map +1 -1
- package/dist/corpus/index.js +365 -7
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/schema.d.ts +33 -0
- package/dist/crawler-ranges.d.ts +31 -0
- package/dist/dashboard/client/actions.d.ts +1 -1
- package/dist/dashboard/client/app.d.ts +9 -2
- package/dist/dashboard/client/boot.d.ts +32 -3
- package/dist/dashboard/client/query.d.ts +72 -12
- package/dist/dashboard/client/registry.d.ts +25 -0
- package/dist/dashboard/client/saved.d.ts +29 -0
- package/dist/dashboard/client/store.d.ts +16 -2
- package/dist/dashboard/client/types.d.ts +2 -0
- package/dist/dashboard/client.generated.d.ts +1 -1
- package/dist/dashboard/types.d.ts +15 -0
- package/dist/detectors/blended-identity.d.ts +34 -0
- package/dist/detectors/challenge-integrity.d.ts +26 -0
- package/dist/detectors/challenge-reaction.d.ts +39 -0
- package/dist/detectors/clearance.d.ts +1 -23
- package/dist/detectors/id-enumeration.d.ts +31 -0
- package/dist/detectors/index.d.ts +24 -1
- package/dist/detectors/known-bots.d.ts +11 -0
- package/dist/detectors/marker.d.ts +106 -0
- package/dist/detectors/parameter-sweep.d.ts +39 -0
- package/dist/detectors/probe-signature.d.ts +27 -0
- package/dist/detectors/probe-volume.d.ts +26 -0
- package/dist/detectors/site-baseline.d.ts +135 -0
- package/dist/detectors/target-integrity.d.ts +16 -0
- package/dist/detectors/transport-coherence.d.ts +31 -0
- package/dist/detectors/trap.d.ts +10 -3
- package/dist/detectors/types.d.ts +17 -0
- package/dist/element/index.cjs +730 -80
- package/dist/element/index.cjs.map +1 -1
- package/dist/element/index.js +730 -80
- package/dist/element/index.js.map +1 -1
- package/dist/index.cjs +2044 -123
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.js +2025 -123
- package/dist/index.js.map +1 -1
- package/dist/internal/async.d.ts +0 -3
- package/dist/internal/ip.d.ts +18 -0
- package/dist/internal/text.d.ts +28 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/probe/index.d.ts +153 -0
- package/dist/probe/marker.d.ts +119 -0
- package/dist/site/index.d.ts +122 -0
- package/dist/state.d.ts +205 -0
- package/dist/stores/redis.d.ts +24 -1
- package/dist/types.d.ts +106 -0
- package/docs/course/05-detectors.md +9 -4
- package/docs/course/06-identity.md +1 -1
- package/docs/course/16-proving-it.md +15 -9
- package/docs/course/index.md +1 -1
- package/docs/design/decisions.md +1 -1
- package/docs/detection/correlation.md +284 -0
- package/docs/detection/detectors.md +259 -1
- package/docs/detection/index.md +2 -1
- package/docs/detection/shadow-mode.md +147 -0
- package/docs/detection/signatures.md +10 -2
- package/docs/index.md +3 -2
- package/docs/integration/client-ip.md +16 -0
- package/docs/operations/dashboard.md +40 -1
- package/docs/operations/filters.md +143 -0
- package/docs/operations/index.md +1 -0
- package/docs/operations/metrics.md +18 -0
- package/docs/policy/presets.md +1 -1
- package/docs/start/choosing-a-policy.md +1 -1
- package/docs/start/first-integration.md +1 -1
- package/docs/start/installation.md +2 -2
- package/docs/testing/cli.md +7 -1
- package/docs/testing/corpus.md +12 -8
- package/docs/testing/index.md +1 -1
- package/docs/testing/try-it.md +1 -1
- package/package.json +4 -1
package/dist/state.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type { ActorSnapshot, RequestFacts } from "./types.js";
|
|
|
12
12
|
*/
|
|
13
13
|
export declare const MAX_TRACKED_ARRIVALS = 32;
|
|
14
14
|
export declare const MAX_TRACKED_PATHS = 64;
|
|
15
|
+
export declare const MAX_TRACKED_QUERIES = 64;
|
|
15
16
|
export declare const MAX_TRACKED_USER_AGENTS = 4;
|
|
16
17
|
export declare class ActorState {
|
|
17
18
|
readonly key: string;
|
|
@@ -53,6 +54,96 @@ export declare class ActorState {
|
|
|
53
54
|
private readonly paths;
|
|
54
55
|
private pathsOverflowed;
|
|
55
56
|
private pathsSaturatedAtTotal;
|
|
57
|
+
/**
|
|
58
|
+
* Distinct *parameterised* requests: the path together with its query.
|
|
59
|
+
*
|
|
60
|
+
* Counted apart from `paths` because the two answer different questions and a scraper
|
|
61
|
+
* lives in the gap between them. `/products?page=1` through `?page=200` is one path and
|
|
62
|
+
* two hundred requests, so breadth reads it as somebody rereading a single page — which
|
|
63
|
+
* is exactly what enumerating a catalogue looks like from the path alone.
|
|
64
|
+
*/
|
|
65
|
+
private readonly queries;
|
|
66
|
+
private queriesOverflowed;
|
|
67
|
+
/**
|
|
68
|
+
* Which HTTP methods this actor has used.
|
|
69
|
+
*
|
|
70
|
+
* A browser navigating issues GET. Something that has issued nothing but HEAD across a
|
|
71
|
+
* long visit is checking what exists rather than reading it, and that is a fact about
|
|
72
|
+
* the actor rather than about any one of its requests — which is why it is kept here.
|
|
73
|
+
*/
|
|
74
|
+
/**
|
|
75
|
+
* What has happened with this actor's marker cookie.
|
|
76
|
+
*
|
|
77
|
+
* Counted rather than listed: the useful questions are all "how often", and a list of
|
|
78
|
+
* marker ids would grow with a client's cookie jar for no benefit. The three drift
|
|
79
|
+
* flags are sticky — once a client has been seen claiming two different browsers under
|
|
80
|
+
* one marker it has done so, and a later request that looks tidy again does not undo
|
|
81
|
+
* it. That is the point of correlating a series rather than judging a request.
|
|
82
|
+
*/
|
|
83
|
+
/**
|
|
84
|
+
* When this actor was last challenged, and how it described itself at that moment.
|
|
85
|
+
*
|
|
86
|
+
* Kept so that what a client does *in response* to being challenged can be read. That
|
|
87
|
+
* reaction is better evidence than anything observed passively, because the stimulus
|
|
88
|
+
* was ours: we chose the moment, so a change of identity that follows it within
|
|
89
|
+
* seconds is a reaction to it rather than a coincidence we went looking for.
|
|
90
|
+
*/
|
|
91
|
+
/**
|
|
92
|
+
* Answers to challenges that were valid in form but wrong in a way only the series
|
|
93
|
+
* shows: a solution already spent, or one returned faster than the puzzle allows.
|
|
94
|
+
*/
|
|
95
|
+
/** Requests for a path no other client had ever asked this site for. */
|
|
96
|
+
private novelPaths;
|
|
97
|
+
private replayedSolutions;
|
|
98
|
+
private implausibleSolves;
|
|
99
|
+
private challengedAt;
|
|
100
|
+
private challengeShape;
|
|
101
|
+
private markerIssues;
|
|
102
|
+
private markerReturns;
|
|
103
|
+
private markerForgeries;
|
|
104
|
+
private driftSeen;
|
|
105
|
+
private driftEvents;
|
|
106
|
+
private readonly methods;
|
|
107
|
+
/**
|
|
108
|
+
* Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
|
|
109
|
+
*
|
|
110
|
+
* Three numbers per shape, deliberately — a count, a lowest and a highest — rather than
|
|
111
|
+
* the ids themselves. What separates enumeration from reading is not which ids were
|
|
112
|
+
* asked for but whether they *cover a range*: thirty requests spanning thirty
|
|
113
|
+
* consecutive ids is a walk, and thirty scattered across a hundred thousand is somebody
|
|
114
|
+
* following links. Both are answerable from a count and a span, and only the count and
|
|
115
|
+
* the span survive an actor asking for ten thousand of them.
|
|
116
|
+
*/
|
|
117
|
+
private readonly walks;
|
|
118
|
+
/**
|
|
119
|
+
* Every named identity this actor has claimed, and what kind each was.
|
|
120
|
+
*
|
|
121
|
+
* Kept because the interesting question is not what one request said but what the *set*
|
|
122
|
+
* of them says. One address claiming sqlmap and nikto is a scan; one claiming Googlebot
|
|
123
|
+
* and Bingbot is a forgery, since at most one of those can be true of an address. Neither
|
|
124
|
+
* observation exists inside a single request.
|
|
125
|
+
*/
|
|
126
|
+
private readonly identities;
|
|
127
|
+
/**
|
|
128
|
+
* A name somebody gave this actor.
|
|
129
|
+
*
|
|
130
|
+
* Nothing in detection reads it. It exists because an address is not a memory: the
|
|
131
|
+
* person who worked out that `198.51.100.4` is the partner's price feed should be able
|
|
132
|
+
* to write that down where the next person will see it, rather than in a ticket.
|
|
133
|
+
*/
|
|
134
|
+
private actorLabel;
|
|
135
|
+
/** Requests from this actor that carried a scanner payload or target. */
|
|
136
|
+
private probePayloads;
|
|
137
|
+
/**
|
|
138
|
+
* What the application answered, for the requests anybody bothered to tell us about.
|
|
139
|
+
*
|
|
140
|
+
* The engine decides *before* the response exists, so this arrives afterwards and only
|
|
141
|
+
* when the adapter reports it. Kept as two counters rather than a list because the one
|
|
142
|
+
* question worth asking is a ratio: an actor whose requests are almost all misses is
|
|
143
|
+
* looking for something rather than reading anything.
|
|
144
|
+
*/
|
|
145
|
+
private responsesSeen;
|
|
146
|
+
private missesSeen;
|
|
56
147
|
private readonly userAgents;
|
|
57
148
|
constructor(key: string, now: number);
|
|
58
149
|
/** Records an arrival. Called exactly once per request, by the engine. */
|
|
@@ -60,6 +151,102 @@ export declare class ActorState {
|
|
|
60
151
|
/** Distinct paths seen, by hash. Saturates at {@link PATH_CAP}; `pathsSaturated` says whether it did. */
|
|
61
152
|
get distinctPaths(): number;
|
|
62
153
|
get pathsSaturated(): boolean;
|
|
154
|
+
/** Distinct path-and-query combinations seen. Saturates at {@link QUERY_CAP}. */
|
|
155
|
+
get distinctQueries(): number;
|
|
156
|
+
get queriesSaturated(): boolean;
|
|
157
|
+
/**
|
|
158
|
+
* Records what the application answered. Called after the response, if at all.
|
|
159
|
+
*
|
|
160
|
+
* 404 and 410 only. A 403 is usually this library's own doing and counting it would
|
|
161
|
+
* make the detector that reads this argue with itself; a 500 is the site's problem and
|
|
162
|
+
* says nothing about the client.
|
|
163
|
+
*/
|
|
164
|
+
recordOutcome(status: number): void;
|
|
165
|
+
/** Responses reported for this actor. Zero unless something is reporting them. */
|
|
166
|
+
get responses(): number;
|
|
167
|
+
/** Of those, how many were 404 or 410. */
|
|
168
|
+
get misses(): number;
|
|
169
|
+
/** Names this actor, or clears the name when given nothing. Trimmed and bounded. */
|
|
170
|
+
setLabel(label: string | undefined): void;
|
|
171
|
+
get label(): string | undefined;
|
|
172
|
+
/** Records a named identity this actor claimed. Called once per matching signature. */
|
|
173
|
+
noteIdentity(id: string, category: string, verifiable: boolean): void;
|
|
174
|
+
/** Records that this request was for a path the site had never served to anybody. */
|
|
175
|
+
noteNovelPath(): void;
|
|
176
|
+
/** How many of this actor's requests were for a path nobody else had ever asked for. */
|
|
177
|
+
get novelPathCount(): number;
|
|
178
|
+
/** Records something wrong with a submitted solution that only its history reveals. */
|
|
179
|
+
noteChallengeAnomaly(kind: "replay" | "implausible-speed"): void;
|
|
180
|
+
/** Solutions this actor submitted that had already been spent, and ones returned too fast. */
|
|
181
|
+
get challengeAnomalies(): {
|
|
182
|
+
replays: number;
|
|
183
|
+
implausible: number;
|
|
184
|
+
};
|
|
185
|
+
/** Records that a challenge went out, and the identity claimed as it did. */
|
|
186
|
+
noteChallengeIssued(at: number, shape: {
|
|
187
|
+
b: string;
|
|
188
|
+
o: string;
|
|
189
|
+
l: string;
|
|
190
|
+
} | undefined): void;
|
|
191
|
+
/** The moment of the last challenge, and the identity claimed then. `at` is 0 for none. */
|
|
192
|
+
get lastChallenge(): {
|
|
193
|
+
at: number;
|
|
194
|
+
shape: {
|
|
195
|
+
b: string;
|
|
196
|
+
o: string;
|
|
197
|
+
l: string;
|
|
198
|
+
} | undefined;
|
|
199
|
+
};
|
|
200
|
+
/** Records that a marker was handed to this actor on the way out. */
|
|
201
|
+
noteMarkerIssued(): void;
|
|
202
|
+
/** Records what this request's marker cookie turned out to be. */
|
|
203
|
+
noteMarker(returned: boolean, forged: boolean, drift: {
|
|
204
|
+
browser: boolean;
|
|
205
|
+
platform: boolean;
|
|
206
|
+
language: boolean;
|
|
207
|
+
} | undefined): void;
|
|
208
|
+
/** Markers handed to this actor, and how many came back. */
|
|
209
|
+
get markers(): {
|
|
210
|
+
issued: number;
|
|
211
|
+
returned: number;
|
|
212
|
+
forged: number;
|
|
213
|
+
};
|
|
214
|
+
/** Which parts of a claimed identity have ever changed under one marker. */
|
|
215
|
+
get identityDrift(): {
|
|
216
|
+
browser: boolean;
|
|
217
|
+
platform: boolean;
|
|
218
|
+
language: boolean;
|
|
219
|
+
events: number;
|
|
220
|
+
};
|
|
221
|
+
/** Records that this request carried a scanner payload, so later requests can know. */
|
|
222
|
+
notePayloadProbe(): void;
|
|
223
|
+
/** Every identity claimed so far, by id. */
|
|
224
|
+
get claimedIdentities(): ReadonlyMap<string, {
|
|
225
|
+
category: string;
|
|
226
|
+
verifiable: boolean;
|
|
227
|
+
}>;
|
|
228
|
+
/** How many of this actor's requests carried a scanner payload or target. */
|
|
229
|
+
get payloadProbes(): number;
|
|
230
|
+
/**
|
|
231
|
+
* Files a request under the shape of its path, if that path carries a number.
|
|
232
|
+
*
|
|
233
|
+
* The last numeric segment is the one taken to be the identifier: in `/api/v2/orders/42`
|
|
234
|
+
* the version is part of the shape and the order id is what is being walked.
|
|
235
|
+
*/
|
|
236
|
+
private noteWalk;
|
|
237
|
+
/**
|
|
238
|
+
* The path shape this actor has walked hardest, with how far it reached.
|
|
239
|
+
*
|
|
240
|
+
* `span` is inclusive of both ends, so a walk of 1 to 30 spans 30. Comparing the count
|
|
241
|
+
* against it is what separates covering a range from visiting a few points in one.
|
|
242
|
+
*/
|
|
243
|
+
densestWalk(): {
|
|
244
|
+
template: string;
|
|
245
|
+
count: number;
|
|
246
|
+
span: number;
|
|
247
|
+
} | undefined;
|
|
248
|
+
/** Every HTTP method this actor has used, in first-seen order. */
|
|
249
|
+
get methodsSeen(): readonly string[];
|
|
63
250
|
/**
|
|
64
251
|
* Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
|
|
65
252
|
* still can. Over that many requests the distinct count is exact, so it is the only
|
|
@@ -159,3 +346,21 @@ export declare class ActorRegistry {
|
|
|
159
346
|
/** Marks an actor as holding valid human clearance until `until`. */
|
|
160
347
|
clearUntil(key: string, until: number): void;
|
|
161
348
|
}
|
|
349
|
+
/**
|
|
350
|
+
* One step of a numeric walk: the shape of the path, and the number in it.
|
|
351
|
+
*
|
|
352
|
+
* `/api/v2/orders/42` becomes `/api/v2/orders/#` and `42`. Shared between the per-actor
|
|
353
|
+
* series and the site-wide one, which must agree on what a shape is or their counts
|
|
354
|
+
* cannot be compared.
|
|
355
|
+
*
|
|
356
|
+
* Both bounds are checked without allocating, because both exist to stop a client paying
|
|
357
|
+
* us to do work. `split` allocates a string per segment, so counting separators first is
|
|
358
|
+
* what makes the depth check worth having. Measured on a sixty-segment path: 3.65µs
|
|
359
|
+
* against 199ns for an ordinary one — every request eighteen times dearer, for free,
|
|
360
|
+
* from anyone willing to send a long URL. The concatenation was most of it; the split
|
|
361
|
+
* was the rest.
|
|
362
|
+
*/
|
|
363
|
+
export declare function walkStepOf(path: string): {
|
|
364
|
+
template: string;
|
|
365
|
+
id: number;
|
|
366
|
+
} | undefined;
|
package/dist/stores/redis.d.ts
CHANGED
|
@@ -9,7 +9,6 @@ import type { Clock } from "../internal/clock.js";
|
|
|
9
9
|
*/
|
|
10
10
|
export interface RedisLike {
|
|
11
11
|
incr(key: string): Promise<number>;
|
|
12
|
-
pexpire(key: string, ms: number): Promise<unknown>;
|
|
13
12
|
/**
|
|
14
13
|
* `SET key value PX ttl [NX]`.
|
|
15
14
|
*
|
|
@@ -51,6 +50,30 @@ export declare class RedisStore implements BotHandlerStore {
|
|
|
51
50
|
private readonly prefix;
|
|
52
51
|
private readonly clock;
|
|
53
52
|
constructor(client: RedisLike, options?: RedisStoreOptions);
|
|
53
|
+
/**
|
|
54
|
+
* Counts one request into the fixed window `key` is bucketed by.
|
|
55
|
+
*
|
|
56
|
+
* This used to be `INCR`, then `PEXPIRE` when the count came back as 1 — which is
|
|
57
|
+
* correct only if the process survives long enough to send the second command. A
|
|
58
|
+
* deploy, an OOM kill or a dropped connection in between left a counter key behind
|
|
59
|
+
* with no expiry at all, and nothing would ever clean it up: the next request falls
|
|
60
|
+
* into the next bucket, under a different key, so the orphan is never touched again.
|
|
61
|
+
* One per unlucky restart is nothing; the point is that it accumulates forever, in a
|
|
62
|
+
* Redis the operator may well be running with `noeviction`.
|
|
63
|
+
*
|
|
64
|
+
* So the expiry is armed by the command that *creates* the key rather than by a
|
|
65
|
+
* follow-up. `SET … PX … NX` writes the seed only if nothing is there, always with a
|
|
66
|
+
* lifetime, and does nothing at all once the bucket exists — so it neither costs a
|
|
67
|
+
* count nor re-arms a window under load. The `INCR` is issued without waiting for its
|
|
68
|
+
* reply, so both commands are on the wire together and this stays one round trip.
|
|
69
|
+
* Ordering holds because a Redis client writes commands to its connection in call
|
|
70
|
+
* order and Redis executes them in arrival order, which means the key has a lifetime
|
|
71
|
+
* from the instant it exists.
|
|
72
|
+
*
|
|
73
|
+
* Not a Lua script, which would make it a single command: `eval` is the one thing
|
|
74
|
+
* `ioredis` and `node-redis` spell differently enough that this interface could not
|
|
75
|
+
* describe both, and staying client-agnostic is worth more than the last round trip.
|
|
76
|
+
*/
|
|
54
77
|
increment(key: string, windowMs: number): Promise<number>;
|
|
55
78
|
consumeOnce(key: string, ttlMs: number): Promise<boolean>;
|
|
56
79
|
get(key: string): Promise<string | undefined>;
|
package/dist/types.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* detection is a solved problem: probabilistic signals exist, they are useful, and
|
|
8
8
|
* they are structurally barred from reaching a terminal action.
|
|
9
9
|
*/
|
|
10
|
+
import type { MarkerObservation } from "./probe/index.js";
|
|
10
11
|
/** What kind of client we believe we are talking to. */
|
|
11
12
|
export type BotClass =
|
|
12
13
|
/** Positive evidence of a person driving a real browser. */
|
|
@@ -92,6 +93,15 @@ export interface Evidence {
|
|
|
92
93
|
deterministicBasis?: string | undefined;
|
|
93
94
|
/** Structured detail for logs and dashboards. Must be JSON-serialisable. */
|
|
94
95
|
metadata?: Record<string, unknown> | undefined;
|
|
96
|
+
/**
|
|
97
|
+
* Set when this came from a detector named in {@link BotHandlerConfig.shadowDetectors}.
|
|
98
|
+
*
|
|
99
|
+
* It is a label rather than a mechanism: shadowed evidence never reaches scoring at
|
|
100
|
+
* all, because it is kept in {@link Assessment.shadowEvidence} rather than filtered out
|
|
101
|
+
* of {@link Assessment.evidence} later. The flag is here so that anything rendering the
|
|
102
|
+
* two lists together can say which is which.
|
|
103
|
+
*/
|
|
104
|
+
shadow?: true | undefined;
|
|
95
105
|
}
|
|
96
106
|
/**
|
|
97
107
|
* Every verdict and class, in a fixed order.
|
|
@@ -128,6 +138,21 @@ export interface RequestFacts {
|
|
|
128
138
|
method: string;
|
|
129
139
|
/** Path only, no query string. Always begins with `/`. */
|
|
130
140
|
path: string;
|
|
141
|
+
/**
|
|
142
|
+
* The target as the client actually spelled it, present only when that is not how
|
|
143
|
+
* `path` reads.
|
|
144
|
+
*
|
|
145
|
+
* `path` is normalised — decoded once, backslashes and doubled slashes collapsed, dot
|
|
146
|
+
* segments resolved — because a rule scoped to `/admin` has to hold against `/%61dmin`
|
|
147
|
+
* and `/./admin` too. That normalisation is also the only thing that makes an evasive
|
|
148
|
+
* target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml` becomes `/app/config.yml`.
|
|
149
|
+
* This is where the difference is kept, so `target-integrity` can read it.
|
|
150
|
+
*
|
|
151
|
+
* Absent on the overwhelming majority of requests, which is the whole reason it is
|
|
152
|
+
* cheap: its presence already means the target was spelled unusually, though not
|
|
153
|
+
* necessarily suspiciously — a trailing slash is enough.
|
|
154
|
+
*/
|
|
155
|
+
rawPath?: string | undefined;
|
|
131
156
|
/** Decoded query parameters. Null-prototype so `?__proto__=x` is visible, not swallowed. */
|
|
132
157
|
query: Record<string, string>;
|
|
133
158
|
/** Lowercased header names to values. Multi-value headers are joined with `, `. */
|
|
@@ -178,6 +203,49 @@ export interface ActorSnapshot {
|
|
|
178
203
|
requests: number;
|
|
179
204
|
/** Distinct paths seen inside the window — breadth of crawl. */
|
|
180
205
|
distinctPaths: number;
|
|
206
|
+
/**
|
|
207
|
+
* Distinct path-and-query combinations seen from this actor.
|
|
208
|
+
*
|
|
209
|
+
* Separate from `distinctPaths` because a scraper lives in the gap between them:
|
|
210
|
+
* `/products?page=1` through `?page=200` is one path and two hundred requests.
|
|
211
|
+
*/
|
|
212
|
+
distinctQueries: number;
|
|
213
|
+
/** Whether `distinctQueries` stopped being able to grow. */
|
|
214
|
+
queriesSaturated: boolean;
|
|
215
|
+
/**
|
|
216
|
+
* Every HTTP method this actor has used.
|
|
217
|
+
*
|
|
218
|
+
* A browser navigating issues GET; an actor whose whole visit is HEAD is checking what
|
|
219
|
+
* exists rather than reading it.
|
|
220
|
+
*/
|
|
221
|
+
methodsSeen: readonly string[];
|
|
222
|
+
/**
|
|
223
|
+
* Responses reported back for this actor, and how many of them were misses.
|
|
224
|
+
*
|
|
225
|
+
* Both zero unless something calls `recordOutcome` — the engine decides before a
|
|
226
|
+
* response exists, so this is knowledge only the application has.
|
|
227
|
+
*/
|
|
228
|
+
responses: number;
|
|
229
|
+
/** Of `responses`, how many were 404 or 410. */
|
|
230
|
+
misses: number;
|
|
231
|
+
/**
|
|
232
|
+
* The path shape this actor has walked hardest — `/user/#` — with how many requests
|
|
233
|
+
* went to it and how wide a range of numbers they covered.
|
|
234
|
+
*
|
|
235
|
+
* Undefined when no path carried a number. `span` is inclusive, so ids 1 to 30 span 30.
|
|
236
|
+
*/
|
|
237
|
+
walk?: {
|
|
238
|
+
template: string;
|
|
239
|
+
count: number;
|
|
240
|
+
span: number;
|
|
241
|
+
} | undefined;
|
|
242
|
+
/**
|
|
243
|
+
* A name somebody gave this actor. Never read by detection.
|
|
244
|
+
*
|
|
245
|
+
* An address is not a memory: whoever worked out that one belongs to a partner's price
|
|
246
|
+
* feed should be able to write it where the next person will see it.
|
|
247
|
+
*/
|
|
248
|
+
label?: string | undefined;
|
|
181
249
|
/** First and last sighting, ms since epoch. */
|
|
182
250
|
firstSeen: number;
|
|
183
251
|
lastSeen: number;
|
|
@@ -219,6 +287,36 @@ export interface Assessment {
|
|
|
219
287
|
evidence: Evidence[];
|
|
220
288
|
/** Every human-pointing observation. These rebut and dampen the score. */
|
|
221
289
|
humanEvidence: Evidence[];
|
|
290
|
+
/**
|
|
291
|
+
* What the shadowed detectors said, in both directions, and what none of it did.
|
|
292
|
+
*
|
|
293
|
+
* A detector listed in {@link BotHandlerConfig.shadowDetectors} runs exactly as it
|
|
294
|
+
* otherwise would and its findings land here instead of in `evidence` — so they are
|
|
295
|
+
* counted, charted and readable, and they took no part in the verdict, the score, the
|
|
296
|
+
* class, the identity, or any rule. Not "weighted at zero": kept out of the arithmetic
|
|
297
|
+
* altogether, because a `certain` finding does not go through the arithmetic and a
|
|
298
|
+
* weight of zero would not have stopped it.
|
|
299
|
+
*
|
|
300
|
+
* This is how a new detector, or a threshold nobody is sure of yet, is answered with a
|
|
301
|
+
* week of your own traffic rather than with an argument.
|
|
302
|
+
*/
|
|
303
|
+
shadowEvidence: Evidence[];
|
|
304
|
+
/**
|
|
305
|
+
* What this assessment would have been had the shadowed detectors been counted.
|
|
306
|
+
*
|
|
307
|
+
* Present only when a shadowed detector actually found something, which is what keeps
|
|
308
|
+
* the second pass off the hot path for the requests it would say nothing about.
|
|
309
|
+
*
|
|
310
|
+
* "It fired 312 times" is not the question anybody has. The question is what turning it
|
|
311
|
+
* on would *do*, and the only honest form of that is the verdict this request would
|
|
312
|
+
* have received — including, and especially, when it is a person.
|
|
313
|
+
*/
|
|
314
|
+
shadowVerdict?: {
|
|
315
|
+
verdict: Verdict;
|
|
316
|
+
botClass: BotClass;
|
|
317
|
+
score: number;
|
|
318
|
+
certain: boolean;
|
|
319
|
+
} | undefined;
|
|
222
320
|
actor: ActorSnapshot;
|
|
223
321
|
/**
|
|
224
322
|
* Set when detection was skipped rather than performed. `undefined` means every
|
|
@@ -232,6 +330,14 @@ export interface Assessment {
|
|
|
232
330
|
/** Detectors that threw or timed out. Detection continues without them. */
|
|
233
331
|
failures: DetectorFailure[];
|
|
234
332
|
facts: RequestFacts;
|
|
333
|
+
/**
|
|
334
|
+
* What the marker cookie on this request turned out to be, when the probe is on.
|
|
335
|
+
*
|
|
336
|
+
* An input rather than a conclusion, kept here for the same reason `facts` is: the
|
|
337
|
+
* action path needs it to decide whether this response should carry a new marker, and
|
|
338
|
+
* recomputing it would mean verifying the same HMAC twice on every request.
|
|
339
|
+
*/
|
|
340
|
+
marker?: MarkerObservation | undefined;
|
|
235
341
|
}
|
|
236
342
|
/** Why an assessment skipped detection. */
|
|
237
343
|
export type BypassReason =
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Lesson 5 — The detectors
|
|
2
2
|
|
|
3
|
-
**Goal:** know what each of the
|
|
3
|
+
**Goal:** know what each of the detectors reads, what it costs, and what it is
|
|
4
4
|
allowed to conclude — then turn one off and watch the score move.
|
|
5
5
|
|
|
6
6
|
← [Course](index.md) · Prev: [The guard](04-the-guard.md) · Next: [Identity and verification](06-identity.md)
|
|
@@ -36,7 +36,7 @@ browsing-coherence cheap always
|
|
|
36
36
|
crawler-verification io confirming
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
**
|
|
39
|
+
**Twenty-two ship on by default.** The others —
|
|
40
40
|
`identity-rotation`, `tls-fingerprint`, `clearance`, `client-signals` — each need something
|
|
41
41
|
from you, and you will switch three of them on later in the course.
|
|
42
42
|
|
|
@@ -61,7 +61,7 @@ fails closed is an outage with extra steps.
|
|
|
61
61
|
|
|
62
62
|
| Detector | Ceiling | Reads |
|
|
63
63
|
| -------- | ------- | ----- |
|
|
64
|
-
| `self-identified` | `certain` |
|
|
64
|
+
| `self-identified` | `certain` | 205 signatures, 389 tokens, in one Aho–Corasick pass. Also catches unrecognised crawlers that name a contact URL, and bare client tokens with no browser preamble |
|
|
65
65
|
| `crawler-verification` | `certain` | Forward-confirmed reverse DNS, or published address ranges. **Confirms and refutes** |
|
|
66
66
|
|
|
67
67
|
### Single-request consistency
|
|
@@ -82,7 +82,12 @@ fails closed is an outage with extra steps.
|
|
|
82
82
|
| `rate-anomaly` | `moderate` | Arrivals in a short window. Reports; never concludes |
|
|
83
83
|
| `cadence` | `moderate` | Coefficient of variation of the gaps. Catches the polite scraper pacing itself *under* your rate limit |
|
|
84
84
|
| `crawl-breadth` | `weak` | Distinct paths against total requests: reading a site against enumerating it |
|
|
85
|
+
| `parameter-sweep` | `weak` | Distinct query strings against the paths they sit on. Catches the collection that leaves the path unchanged — `?page=1..200` |
|
|
85
86
|
| `session-integrity` | `moderate` | A "browser" that never carries a cookie |
|
|
87
|
+
| `blended-identity` | `strong` | The *set* of identities one actor claimed: two scanners, or two crawlers that cannot both be true |
|
|
88
|
+
| `id-enumeration` | `moderate` | A contiguous run of numeric ids under one path shape — walking `/user/1..n` rather than following links |
|
|
89
|
+
| `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; every bundled adapter wires it up |
|
|
90
|
+
| `transport-coherence` | `moderate` | The HTTP version and the verbs across a visit: a "Chrome" on HTTP/1.0, a visit made only of HEAD |
|
|
86
91
|
| `identity-rotation` | `moderate` | One actor, several User-Agents. **Off by default** — under an IP actor key this fires on every corporate NAT |
|
|
87
92
|
| `browsing-coherence` | `moderate` | The only detector arguing *for* the client. Human-pointing, so it discounts |
|
|
88
93
|
|
|
@@ -163,7 +168,7 @@ IP-based actor key.
|
|
|
163
168
|
|
|
164
169
|
## What you learned
|
|
165
170
|
|
|
166
|
-
-
|
|
171
|
+
- Twenty-one detectors are on by default; the rest need something from you — a marker probe, a challenge, a site baseline, or data only you have
|
|
167
172
|
- `cost` and `stage` decide when a detector runs, and confirming work is skipped when
|
|
168
173
|
nothing claimed an identity
|
|
169
174
|
- A failing detector is dropped, never the request
|
|
@@ -9,7 +9,7 @@ a name alone is worth so little.
|
|
|
9
9
|
|
|
10
10
|
## A name is a claim, not a fact
|
|
11
11
|
|
|
12
|
-
`self-identified` matches
|
|
12
|
+
`self-identified` matches 205 signatures across search, AI, SEO, social, monitoring,
|
|
13
13
|
feeds, archives, security tooling, HTTP libraries and headless runtimes. When it fires you
|
|
14
14
|
get an `identity` — `"googlebot"`, `"gptbot"` — and a `category`.
|
|
15
15
|
|
|
@@ -13,29 +13,35 @@ everything into a Serif you can defend.
|
|
|
13
13
|
|
|
14
14
|
Three tools answer it, and they answer different halves.
|
|
15
15
|
|
|
16
|
-
## `check` — against
|
|
16
|
+
## `check` — against 548 shapes of real traffic
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
19
|
npx @osqd/bothandlerjs check --preset protect-content
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
```
|
|
23
|
-
protect-content against
|
|
23
|
+
protect-content against 539 shapes of real traffic
|
|
24
24
|
|
|
25
|
-
human
|
|
26
|
-
benign-bot 144 cases
|
|
27
|
-
declared-bot 32 cases 18 tag,
|
|
28
|
-
unwanted-bot
|
|
29
|
-
hostile
|
|
30
|
-
infrastructure 33 cases
|
|
25
|
+
human 184 cases 178 allow, 3 tag, 3 challenge
|
|
26
|
+
benign-bot 144 cases 86 tag, 48 block, 6 allow, 2 challenge, 2 rate-limit
|
|
27
|
+
declared-bot 32 cases 18 tag, 9 block, 3 allow, 2 rate-limit
|
|
28
|
+
unwanted-bot 109 cases 71 challenge, 16 allow, 11 rate-limit, 8 tag, 3 block
|
|
29
|
+
hostile 37 cases 20 block, 9 allow, 7 challenge, 1 tag
|
|
30
|
+
infrastructure 33 cases 14 challenge, 12 tag, 7 allow
|
|
31
31
|
|
|
32
32
|
No case marked as a person was denied service.
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
The corpus holds 548 cases and this run reports 539, which is not a discrepancy: a case
|
|
36
|
+
naming a capability the configuration under test does not provide is *skipped* rather than
|
|
37
|
+
failed. The nine here need a marker probe or a site baseline, and judging a marker case
|
|
38
|
+
against a configuration that issues no markers would be a verdict about nothing. Turn those
|
|
39
|
+
sources on and the same command reports all 548.
|
|
40
|
+
|
|
35
41
|
**It exits non-zero if any case marked `human` is denied**, which is what makes it a CI step
|
|
36
42
|
rather than a report.
|
|
37
43
|
|
|
38
|
-
The corpus is
|
|
44
|
+
The corpus is 548 cases with provenance: 186 of them people — 30 browser profiles, 40 in-app
|
|
39
45
|
WebViews, Tor, screen readers, IE11, a car's infotainment screen, corporate proxies, CGNAT,
|
|
40
46
|
an author signing in at `/wp-login.php`. Header order is reproduced rather than invented, and
|
|
41
47
|
DNS is controlled rather than mocked away, so *"the operator's DNS disproves this"* and
|
package/docs/course/index.md
CHANGED
|
@@ -58,7 +58,7 @@ lesson 11.
|
|
|
58
58
|
|
|
59
59
|
| | | |
|
|
60
60
|
|-|-|-|
|
|
61
|
-
| 5 | [The detectors](05-detectors.md) | All twenty: what each reads, what each costs, what each may conclude. |
|
|
61
|
+
| 5 | [The detectors](05-detectors.md) | All twenty-two: what each reads, what each costs, what each may conclude. |
|
|
62
62
|
| 6 | [Identity and verification](06-identity.md) | Recognising a crawler by name — and proving or refuting the claim. |
|
|
63
63
|
| 7 | [Actors and behaviour](07-actors.md) | Who "the same client" is, and what watching one over time tells you. |
|
|
64
64
|
| 8 | [Traps](08-traps.md) | The one detector that needs no statistics, and how to lay one properly. |
|
package/docs/design/decisions.md
CHANGED
|
@@ -187,7 +187,7 @@ type checker catches something.
|
|
|
187
187
|
|
|
188
188
|
## The corpus is a published entry point, not a test fixture
|
|
189
189
|
|
|
190
|
-
**The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all
|
|
190
|
+
**The decision.** `@osqd/bothandlerjs/corpus` ships `runCorpus` and all 548 cases.
|
|
191
191
|
|
|
192
192
|
**Why.** The claims in this documentation are only worth anything if you can check them
|
|
193
193
|
against *your* configuration. A corpus locked inside the test suite proves things about the
|