@osqd/bothandlerjs 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -1
- package/README.md +18 -10
- package/dist/adapters/fastify.d.ts +10 -0
- package/dist/adapters/index.cjs +38 -13
- package/dist/adapters/index.cjs.map +1 -1
- package/dist/adapters/index.js +38 -13
- package/dist/adapters/index.js.map +1 -1
- package/dist/challenge/index.d.ts +40 -0
- package/dist/cli.cjs +1783 -83
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1783 -83
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +55 -0
- package/dist/core.d.ts +33 -21
- package/dist/corpus/index.cjs +282 -8
- package/dist/corpus/index.cjs.map +1 -1
- package/dist/corpus/index.js +282 -8
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/schema.d.ts +25 -0
- package/dist/dashboard/client/actions.d.ts +1 -1
- package/dist/dashboard/client/app.d.ts +9 -2
- package/dist/dashboard/client/boot.d.ts +32 -3
- package/dist/dashboard/client/query.d.ts +72 -12
- package/dist/dashboard/client/registry.d.ts +25 -0
- package/dist/dashboard/client/saved.d.ts +29 -0
- package/dist/dashboard/client/store.d.ts +16 -2
- package/dist/dashboard/client/types.d.ts +2 -0
- package/dist/dashboard/client.generated.d.ts +1 -1
- package/dist/dashboard/types.d.ts +15 -0
- package/dist/detectors/blended-identity.d.ts +34 -0
- package/dist/detectors/challenge-integrity.d.ts +26 -0
- package/dist/detectors/challenge-reaction.d.ts +39 -0
- package/dist/detectors/clearance.d.ts +1 -23
- package/dist/detectors/index.d.ts +16 -1
- package/dist/detectors/marker.d.ts +106 -0
- package/dist/detectors/probe-signature.d.ts +27 -0
- package/dist/detectors/site-baseline.d.ts +135 -0
- package/dist/detectors/target-integrity.d.ts +16 -0
- package/dist/detectors/trap.d.ts +10 -3
- package/dist/detectors/types.d.ts +17 -0
- package/dist/element/index.cjs +726 -79
- package/dist/element/index.cjs.map +1 -1
- package/dist/element/index.js +726 -79
- package/dist/element/index.js.map +1 -1
- package/dist/index.cjs +1657 -125
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1643 -125
- package/dist/index.js.map +1 -1
- package/dist/internal/async.d.ts +0 -3
- package/dist/internal/ip.d.ts +18 -0
- package/dist/internal/text.d.ts +28 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/probe/index.d.ts +153 -0
- package/dist/probe/marker.d.ts +119 -0
- package/dist/site/index.d.ts +122 -0
- package/dist/state.d.ts +130 -0
- package/dist/stores/redis.d.ts +24 -1
- package/dist/types.d.ts +70 -0
- package/docs/course/05-detectors.md +6 -5
- package/docs/course/06-identity.md +1 -1
- package/docs/course/16-proving-it.md +15 -9
- package/docs/course/index.md +1 -1
- package/docs/design/decisions.md +1 -1
- package/docs/detection/correlation.md +284 -0
- package/docs/detection/detectors.md +139 -3
- package/docs/detection/index.md +2 -1
- package/docs/detection/shadow-mode.md +147 -0
- package/docs/detection/signatures.md +1 -1
- package/docs/index.md +3 -2
- package/docs/integration/client-ip.md +16 -0
- package/docs/operations/dashboard.md +40 -1
- package/docs/operations/filters.md +143 -0
- package/docs/operations/index.md +1 -0
- package/docs/operations/metrics.md +18 -0
- package/docs/policy/presets.md +1 -1
- package/docs/start/choosing-a-policy.md +1 -1
- package/docs/start/first-integration.md +1 -1
- package/docs/start/installation.md +2 -2
- package/docs/testing/cli.md +7 -1
- package/docs/testing/corpus.md +12 -8
- package/docs/testing/index.md +1 -1
- package/docs/testing/try-it.md +1 -1
- package/package.json +4 -1
package/dist/internal/async.d.ts
CHANGED
|
@@ -11,6 +11,3 @@
|
|
|
11
11
|
* unhandled rejection.
|
|
12
12
|
*/
|
|
13
13
|
export declare function withTimeout<T>(work: Promise<T>, ms: number, fallback: T): Promise<T>;
|
|
14
|
-
/** `Promise.allSettled` semantics without allocating a settled-result wrapper per item. */
|
|
15
|
-
export declare function settleAll(work: Iterable<Promise<void>>): Promise<void>;
|
|
16
|
-
export declare function isPromise(value: unknown): value is Promise<unknown>;
|
package/dist/internal/ip.d.ts
CHANGED
|
@@ -13,6 +13,24 @@ export type IpBytes = Uint8Array;
|
|
|
13
13
|
* address. Deliberately strict: no octal, no hex, no shorthand octets, no zone
|
|
14
14
|
* ids. Anything ambiguous is rejected rather than guessed at.
|
|
15
15
|
*/
|
|
16
|
+
/**
|
|
17
|
+
* An address with its source port removed, when it carried one.
|
|
18
|
+
*
|
|
19
|
+
* Forwarded headers are not consistent about this. Most proxies write a bare address,
|
|
20
|
+
* but Azure's Application Gateway and Front Door write `1.2.3.4:5678`, and the
|
|
21
|
+
* bracketed `[2001:db8::1]:5678` is the form RFC 7239 defines for IPv6. An entry that
|
|
22
|
+
* carries a port parses as nothing at all, and the consequence is not that one entry is
|
|
23
|
+
* skipped: every entry in the chain looks the same way, the chain empties, and the whole
|
|
24
|
+
* internet collapses onto the proxy's own address as a single actor. Rate limits, actor
|
|
25
|
+
* history and reputation then apply to everyone at once, so one bot locks out every real
|
|
26
|
+
* visitor — and none of it announces itself.
|
|
27
|
+
*
|
|
28
|
+
* The rule has to be narrow, because a bare IPv6 address is *made of* colons and must
|
|
29
|
+
* never be mistaken for a host and port. Only two shapes are a port: a bracketed host,
|
|
30
|
+
* which is unambiguous, and a single colon whose left side is an IPv4 address. Anything
|
|
31
|
+
* with more colons and no brackets is IPv6 and is returned untouched.
|
|
32
|
+
*/
|
|
33
|
+
export declare function stripPort(value: string): string;
|
|
16
34
|
export declare function parseIp(value: string): IpBytes | null;
|
|
17
35
|
/** Canonical string form, so the same host always produces the same actor key. */
|
|
18
36
|
export declare function formatIp(bytes: IpBytes): string;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text that is safe to print.
|
|
3
|
+
*
|
|
4
|
+
* Everything here exists because this library quotes its input back: an evidence
|
|
5
|
+
* summary names the path that was requested and the header that was sent, and an actor
|
|
6
|
+
* label is whatever an operator typed. That text is then written to a log file, a
|
|
7
|
+
* terminal and a JSON feed — three readers that treat some characters as instructions
|
|
8
|
+
* rather than as letters.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Makes one line of prose safe to print.
|
|
12
|
+
*
|
|
13
|
+
* Evidence summaries name what was seen, and naming it often means quoting the client:
|
|
14
|
+
* the path it asked for, the header it sent, the identity it claimed. That text is
|
|
15
|
+
* written by whoever sent the request, and it lands in three places that read control
|
|
16
|
+
* characters as instructions rather than as letters — a log file, where a carriage
|
|
17
|
+
* return and a newline let a client forge a second log line of its own composition; a
|
|
18
|
+
* terminal, where an escape sequence repaints or erases what is already on screen; and
|
|
19
|
+
* a JSON feed, where a lone surrogate is not encodable.
|
|
20
|
+
*
|
|
21
|
+
* So the characters that mean something other than themselves are replaced with U+FFFD
|
|
22
|
+
* and the text is capped. What a client says about itself is quoted, never obeyed.
|
|
23
|
+
*
|
|
24
|
+
* The scan allocates nothing and returns the original string when there is nothing to
|
|
25
|
+
* fix, which is every request that is not an attack: this runs on every piece of
|
|
26
|
+
* evidence on every request.
|
|
27
|
+
*/
|
|
28
|
+
export declare function safeSummary(text: string): string;
|
package/dist/metrics.d.ts
CHANGED
|
@@ -29,6 +29,22 @@ export interface MetricsSnapshot {
|
|
|
29
29
|
detectorFirings: Record<string, number>;
|
|
30
30
|
/** How often each detector threw or timed out. */
|
|
31
31
|
detectorFailures: Record<string, number>;
|
|
32
|
+
/**
|
|
33
|
+
* How often each *shadowed* detector produced evidence, counted separately.
|
|
34
|
+
*
|
|
35
|
+
* Separately on purpose: a shadowed detector took no part in any verdict, so folding
|
|
36
|
+
* its firings into `detectorFirings` would put work into a chart of decisions that
|
|
37
|
+
* decided nothing. See {@link BotHandlerConfig.shadowDetectors}.
|
|
38
|
+
*/
|
|
39
|
+
shadowFirings: Record<string, number>;
|
|
40
|
+
/**
|
|
41
|
+
* Assessments whose verdict the shadowed detectors would have changed.
|
|
42
|
+
*
|
|
43
|
+
* Keyed by the verdict they would have produced, so the number that matters is legible
|
|
44
|
+
* on its own: `suspected-bot` here, against a `human` count that did not move, is the
|
|
45
|
+
* shape of a detector about to start challenging people.
|
|
46
|
+
*/
|
|
47
|
+
shadowChanges: Record<Verdict, number>;
|
|
32
48
|
/**
|
|
33
49
|
* Time spent inside each detector, when `metrics.perDetectorTiming` is on.
|
|
34
50
|
*
|
|
@@ -100,6 +116,8 @@ export declare class Metrics {
|
|
|
100
116
|
private readonly detectorFirings;
|
|
101
117
|
private readonly detectorFailures;
|
|
102
118
|
private readonly detectorTimings;
|
|
119
|
+
private readonly shadowFirings;
|
|
120
|
+
private readonly shadowChanges;
|
|
103
121
|
private challengesIssued;
|
|
104
122
|
private challengesSolved;
|
|
105
123
|
private challengesRejected;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import type { IdentityShape, MarkerReading, ShapeDrift } from "./marker.js";
|
|
2
|
+
import type { RequestFacts } from "../types.js";
|
|
3
|
+
import type { ParsedUserAgent } from "../internal/ua.js";
|
|
4
|
+
import type { Clock } from "../internal/clock.js";
|
|
5
|
+
export type { IdentityShape, MarkerClaims, MarkerReading, ShapeDrift } from "./marker.js";
|
|
6
|
+
export { identityShape, driftBetween, readMarker } from "./marker.js";
|
|
7
|
+
/**
|
|
8
|
+
* Asking the client to hold something, and reading what comes back.
|
|
9
|
+
*
|
|
10
|
+
* This is the one part of the library that *acts* in order to detect, rather than
|
|
11
|
+
* reading what a request happened to carry. That difference is worth being explicit
|
|
12
|
+
* about, because it changes what the evidence is worth: everything else here is an
|
|
13
|
+
* inference about a client, while a marker is a controlled experiment on one. We choose
|
|
14
|
+
* the stimulus, we sign it, and the response is either the cookie we issued or it is
|
|
15
|
+
* not. There is very little room left for a coincidence to explain.
|
|
16
|
+
*
|
|
17
|
+
* It is **off by default**, for two reasons that have nothing to do with detection.
|
|
18
|
+
* A `Set-Cookie` on a response makes it uncacheable by most shared caches and CDNs, so
|
|
19
|
+
* switching this on without knowing that can quietly move a site's cache-hit ratio; the
|
|
20
|
+
* probe therefore issues a marker only when the client does not already hold a valid
|
|
21
|
+
* one, which for an ordinary visitor is the first request of a session and no other.
|
|
22
|
+
* And a cookie is a cookie: it is first-party, carries no identifier of a person and
|
|
23
|
+
* expires on its own, but the decision to set one belongs to the operator rather than
|
|
24
|
+
* to a library they installed to read headers.
|
|
25
|
+
*/
|
|
26
|
+
export interface MarkerProbeOptions {
|
|
27
|
+
/**
|
|
28
|
+
* HMAC secrets. The first signs; all of them verify.
|
|
29
|
+
*
|
|
30
|
+
* Required, and deliberately not defaulted to something generated at startup. A
|
|
31
|
+
* per-process secret would mean every marker minted by one replica reads as *forged*
|
|
32
|
+
* on every other one, and as forged again after a restart — turning the strongest
|
|
33
|
+
* signal here into a machine for accusing ordinary visitors.
|
|
34
|
+
*/
|
|
35
|
+
secrets: readonly string[];
|
|
36
|
+
/** Cookie name. Default `__bh_m`. */
|
|
37
|
+
cookieName?: string;
|
|
38
|
+
/** How long a marker stands. Default 12 hours. */
|
|
39
|
+
ttlMs?: number;
|
|
40
|
+
sameSite?: "Strict" | "Lax" | "None";
|
|
41
|
+
/** Set the `Secure` attribute. Default true; turn it off only for local HTTP. */
|
|
42
|
+
secure?: boolean;
|
|
43
|
+
domain?: string;
|
|
44
|
+
/**
|
|
45
|
+
* How many markers to track network fan-out for. Default 20000; 0 turns it off.
|
|
46
|
+
*
|
|
47
|
+
* Bounded rather than shared, on purpose. The rest of this library keeps its
|
|
48
|
+
* behavioural series in process — see `state.ts` — because a store round trip on the
|
|
49
|
+
* request path buys precision nobody asked for at a cost everybody pays. Across
|
|
50
|
+
* replicas each one therefore sees its own share of a client's addresses, which
|
|
51
|
+
* *understates* fan-out and so errs towards saying nothing.
|
|
52
|
+
*/
|
|
53
|
+
maxTrackedMarkers?: number;
|
|
54
|
+
/**
|
|
55
|
+
* Estimated networks at which the count saturates. Default 96.
|
|
56
|
+
*
|
|
57
|
+
* The networks are not remembered individually — see `noteNetwork`. Raising this does
|
|
58
|
+
* not cost memory; it only extends the range over which the estimate stays useful.
|
|
59
|
+
*/
|
|
60
|
+
maxNetworksPerMarker?: number;
|
|
61
|
+
/**
|
|
62
|
+
* How many verified markers to remember, so a session is not re-verified per request.
|
|
63
|
+
* Default 5000; 0 turns the cache off.
|
|
64
|
+
*/
|
|
65
|
+
maxVerifiedMarkers?: number;
|
|
66
|
+
}
|
|
67
|
+
/** What this request's marker turned out to be. */
|
|
68
|
+
export interface MarkerObservation {
|
|
69
|
+
reading: MarkerReading;
|
|
70
|
+
/**
|
|
71
|
+
* How the identity claimed now differs from the identity claimed when the marker was
|
|
72
|
+
* issued. Present only for a marker that verified, because a drift measured against
|
|
73
|
+
* an unsigned claim measures nothing.
|
|
74
|
+
*/
|
|
75
|
+
drift: ShapeDrift | undefined;
|
|
76
|
+
/** This request's shape, so the issuing path does not compute it twice. */
|
|
77
|
+
shape: IdentityShape;
|
|
78
|
+
/**
|
|
79
|
+
* Distinct networks this marker has now been presented from, counted in this process.
|
|
80
|
+
*
|
|
81
|
+
* `0` when the marker did not verify or tracking is off. Saturates at
|
|
82
|
+
* `maxNetworksPerMarker`, because the question it answers — "is one client moving
|
|
83
|
+
* across a pool of addresses" — is already answered long before the number is large.
|
|
84
|
+
*/
|
|
85
|
+
networks: number;
|
|
86
|
+
}
|
|
87
|
+
export declare class MarkerProbe {
|
|
88
|
+
readonly cookieName: string;
|
|
89
|
+
private readonly secrets;
|
|
90
|
+
private readonly ttlMs;
|
|
91
|
+
private readonly cookieOptions;
|
|
92
|
+
private readonly clock;
|
|
93
|
+
/**
|
|
94
|
+
* Marker id to a 128-bit sketch of the networks it has been presented from.
|
|
95
|
+
*
|
|
96
|
+
* A `Set` of network strings is the obvious structure and measured at **55.6 MB** with
|
|
97
|
+
* both caps full — twenty thousand markers each seen from a few dozen networks — which
|
|
98
|
+
* is far too much to hand somebody for switching on a detector. The question being
|
|
99
|
+
* asked is only ever "has this marker come from more than about sixteen networks", and
|
|
100
|
+
* a bitmap answers that in sixteen bytes by linear counting: hash each network to a
|
|
101
|
+
* bit, then estimate the distinct count from how many bits are set.
|
|
102
|
+
*
|
|
103
|
+
* The estimate carries a few percent of error in **either** direction — measured, 16
|
|
104
|
+
* real networks read as 17 and 32 read as 33 — so the threshold it feeds is a soft
|
|
105
|
+
* boundary rather than a hard one. That is honest for this signal in particular, which
|
|
106
|
+
* cannot separate a proxy pool from a heavily mobile person at any resolution, and is
|
|
107
|
+
* why it is capped at `moderate` and never denies anybody by itself.
|
|
108
|
+
*/
|
|
109
|
+
private readonly fanout;
|
|
110
|
+
private readonly maxNetworks;
|
|
111
|
+
/**
|
|
112
|
+
* Markers already verified, by the exact cookie value that verified.
|
|
113
|
+
*
|
|
114
|
+
* A browsing session sends one identical cookie on every request, and verifying it is
|
|
115
|
+
* an HMAC — which measured at roughly twenty microseconds, nearly doubling the cost of
|
|
116
|
+
* an assessment to re-establish a fact that had not changed. The cache is only ever
|
|
117
|
+
* populated with *successes*: caching failures would let anyone flood it with unique
|
|
118
|
+
* junk, and a failure is cheap to reach anyway.
|
|
119
|
+
*
|
|
120
|
+
* Expiry is still checked on every hit, so a cached marker stops being accepted at the
|
|
121
|
+
* moment it should. The key is the whole signed value, so a cache hit is only possible
|
|
122
|
+
* for a string that already carried a valid signature.
|
|
123
|
+
*/
|
|
124
|
+
private readonly verified;
|
|
125
|
+
constructor(options: MarkerProbeOptions & {
|
|
126
|
+
clock: Clock;
|
|
127
|
+
});
|
|
128
|
+
/** Reads the marker this request carried, and measures it against the request. */
|
|
129
|
+
observe(facts: RequestFacts, ua: ParsedUserAgent): MarkerObservation;
|
|
130
|
+
/**
|
|
131
|
+
* Whether this response should carry a marker.
|
|
132
|
+
*
|
|
133
|
+
* Only when the client is not already holding a good one. An ordinary visitor is
|
|
134
|
+
* therefore issued a cookie once and then browses with uncached-by-`Set-Cookie`
|
|
135
|
+
* responses never again; a client that discards cookies is issued one every time,
|
|
136
|
+
* which is itself the observation `marker-persistence` is built on.
|
|
137
|
+
*/
|
|
138
|
+
shouldIssue(observation: MarkerObservation): boolean;
|
|
139
|
+
/** Verifies a presented marker, reusing an earlier verification of the same value. */
|
|
140
|
+
private read;
|
|
141
|
+
/**
|
|
142
|
+
* Files this presentation under the marker's own id and returns how many distinct
|
|
143
|
+
* networks it has now come from.
|
|
144
|
+
*
|
|
145
|
+
* A `/24` rather than an address, because a single visitor's address changes for
|
|
146
|
+
* ordinary reasons all day — a phone moving between cells, a router relearning a
|
|
147
|
+
* lease — while the network it sits behind usually does not. Counting addresses would
|
|
148
|
+
* report every commuter.
|
|
149
|
+
*/
|
|
150
|
+
private noteNetwork;
|
|
151
|
+
/** The `Set-Cookie` handing this client a marker bound to the identity it just claimed. */
|
|
152
|
+
issue(observation: MarkerObservation): string;
|
|
153
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { RequestFacts } from "../types.js";
|
|
2
|
+
import type { ParsedUserAgent } from "../internal/ua.js";
|
|
3
|
+
/**
|
|
4
|
+
* A marker the client carries, so its requests can be read as a series.
|
|
5
|
+
*
|
|
6
|
+
* Everything else in this library correlates requests by **actor key**, which is
|
|
7
|
+
* derived from the address. That is the only join available without asking the client
|
|
8
|
+
* to hold anything, and it is wrong in both directions: a household, an office and a
|
|
9
|
+
* mobile carrier put thousands of unrelated people behind one key, while a single
|
|
10
|
+
* scraper on a proxy pool spreads one operator across thousands of keys. The
|
|
11
|
+
* consequence was a detector this library could not write — `identity-rotation`, the
|
|
12
|
+
* client that arrives as Chrome, then as curl, then as Googlebot. From headers alone
|
|
13
|
+
* it is indistinguishable from three people sharing an address, so it was left unbuilt.
|
|
14
|
+
*
|
|
15
|
+
* A signed cookie closes exactly that gap. It is minted by this server, carries an
|
|
16
|
+
* HMAC only this server can produce, and comes back only from the client that received
|
|
17
|
+
* it. Two requests bearing the same marker are the same browser profile — not the same
|
|
18
|
+
* address, not the same network, the same *client* — and that is what makes a change of
|
|
19
|
+
* claimed identity between them evidence rather than speculation.
|
|
20
|
+
*
|
|
21
|
+
* **What is in it, and what is deliberately not.** Tokens here are signed and never
|
|
22
|
+
* encrypted, so the client can read every claim and nothing secret may go in one. The
|
|
23
|
+
* marker holds a random id, the usual validity window, and three short hashes standing
|
|
24
|
+
* for the identity the client claimed when it was issued. The hashes are of data the
|
|
25
|
+
* client sent us in the first place, so they tell it nothing it did not already know,
|
|
26
|
+
* and hashing them keeps the cookie from being a readable fingerprint echoed back on
|
|
27
|
+
* every response.
|
|
28
|
+
*
|
|
29
|
+
* **Why the identity is stored coarsely.** A browser that updates from version 130 to
|
|
30
|
+
* 131 has not changed identity, and a detector that says otherwise would report every
|
|
31
|
+
* visitor in the week after a Chrome release. So the version is excluded and the three
|
|
32
|
+
* parts are kept apart rather than hashed together, because *which* part changed is the
|
|
33
|
+
* difference between a strong signal and a benign one: a browser family that changes
|
|
34
|
+
* from Chrome to curl has no innocent reading, while a platform that changes from
|
|
35
|
+
* iPhone to Mac is what "Request desktop site" does to a real person's phone.
|
|
36
|
+
*/
|
|
37
|
+
/** Claims inside a marker. `sub` is the marker's own id, not an actor: see below. */
|
|
38
|
+
export interface MarkerClaims {
|
|
39
|
+
v: 1;
|
|
40
|
+
/**
|
|
41
|
+
* The marker's own random id.
|
|
42
|
+
*
|
|
43
|
+
* Deliberately *not* the actor. Binding a marker to an address-derived key would
|
|
44
|
+
* invalidate it the moment a phone moved between wifi and cellular, which is the
|
|
45
|
+
* ordinary behaviour of the visitors this is supposed to leave alone — and it would
|
|
46
|
+
* throw away the property that makes the marker worth having, that it identifies a
|
|
47
|
+
* client across exactly those changes.
|
|
48
|
+
*/
|
|
49
|
+
sub: string;
|
|
50
|
+
iat: number;
|
|
51
|
+
exp: number;
|
|
52
|
+
/** Browser family, coarse. */
|
|
53
|
+
b: string;
|
|
54
|
+
/** Operating system or platform, coarse. */
|
|
55
|
+
o: string;
|
|
56
|
+
/** Primary language subtag. */
|
|
57
|
+
l: string;
|
|
58
|
+
}
|
|
59
|
+
/** How the claimed identity looked, reduced to the parts worth comparing. */
|
|
60
|
+
export interface IdentityShape {
|
|
61
|
+
b: string;
|
|
62
|
+
o: string;
|
|
63
|
+
l: string;
|
|
64
|
+
}
|
|
65
|
+
/** Which parts of a claimed identity differ between two requests. */
|
|
66
|
+
export interface ShapeDrift {
|
|
67
|
+
browser: boolean;
|
|
68
|
+
platform: boolean;
|
|
69
|
+
language: boolean;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* The claimed identity, reduced to three coarse parts.
|
|
73
|
+
*
|
|
74
|
+
* Stored as short plain text rather than hashed, which is both cheaper and more honest.
|
|
75
|
+
* An earlier version ran each part through an HMAC so the cookie would not carry a
|
|
76
|
+
* "readable fingerprint" — but there is no privacy in hashing a value the client wrote
|
|
77
|
+
* itself and sent to us, in a token the client can already read. It bought nothing and
|
|
78
|
+
* cost three HMACs on every single request, which more than doubled the price of an
|
|
79
|
+
* assessment. Now it costs a `slice`.
|
|
80
|
+
*
|
|
81
|
+
* The version is deliberately absent. A browser updating from 130 to 131 has not changed
|
|
82
|
+
* identity, and a detector that says otherwise reports every visitor in the week after a
|
|
83
|
+
* Chrome release.
|
|
84
|
+
*/
|
|
85
|
+
export declare function identityShape(facts: RequestFacts, ua: ParsedUserAgent): IdentityShape;
|
|
86
|
+
/** Which parts changed. All three false means the client looks the same as it did. */
|
|
87
|
+
export declare function driftBetween(issued: IdentityShape, now: IdentityShape): ShapeDrift;
|
|
88
|
+
export declare function newMarker(shape: IdentityShape, ttlMs: number, now: number): MarkerClaims;
|
|
89
|
+
/** The `Set-Cookie` that hands a client its marker. */
|
|
90
|
+
export declare function markerCookie(name: string, claims: MarkerClaims, secrets: readonly string[], options: MarkerCookieOptions): string;
|
|
91
|
+
export interface MarkerCookieOptions {
|
|
92
|
+
sameSite?: "Strict" | "Lax" | "None";
|
|
93
|
+
secure?: boolean;
|
|
94
|
+
domain?: string;
|
|
95
|
+
}
|
|
96
|
+
export type MarkerReading =
|
|
97
|
+
/** No marker was presented. */
|
|
98
|
+
{
|
|
99
|
+
kind: "absent";
|
|
100
|
+
}
|
|
101
|
+
/** Presented, signed by us, still valid. */
|
|
102
|
+
| {
|
|
103
|
+
kind: "valid";
|
|
104
|
+
claims: MarkerClaims;
|
|
105
|
+
}
|
|
106
|
+
/** Presented and past its expiry — ordinary, and not evidence of anything. */
|
|
107
|
+
| {
|
|
108
|
+
kind: "expired";
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Presented, and not something this server signed.
|
|
112
|
+
*
|
|
113
|
+
* The interesting case. A browser does not edit its own cookies, so a marker that
|
|
114
|
+
* fails its signature was changed by whoever is holding it.
|
|
115
|
+
*/
|
|
116
|
+
| {
|
|
117
|
+
kind: "forged";
|
|
118
|
+
};
|
|
119
|
+
export declare function readMarker(value: string | undefined, secrets: readonly string[], now: number): MarkerReading;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { Clock } from "../internal/clock.js";
|
|
2
|
+
/**
|
|
3
|
+
* What this site normally looks like.
|
|
4
|
+
*
|
|
5
|
+
* Everything else in this library judges a client against a fixed idea of what clients
|
|
6
|
+
* do. This judges one against *the rest of the traffic*, which answers questions the
|
|
7
|
+
* fixed rules cannot. A path nobody else has ever asked for is a wordlist entry without
|
|
8
|
+
* needing to be in anybody's wordlist. A client whose requests are almost all misses
|
|
9
|
+
* matters only relative to a site's own miss rate, which on a site mid-migration is
|
|
10
|
+
* enormous and innocent. And an enumeration split across five hundred addresses is
|
|
11
|
+
* invisible to every per-actor threshold by construction, but perfectly visible in the
|
|
12
|
+
* union of what those addresses asked for.
|
|
13
|
+
*
|
|
14
|
+
* **It is off by default and it stays quiet for a long time after being switched on.**
|
|
15
|
+
* Both of those are load-bearing. A baseline is a claim about normal, and a claim about
|
|
16
|
+
* normal drawn from four hundred requests is not one — a quiet site at three in the
|
|
17
|
+
* morning would otherwise produce "this path is unique to this client" for every path,
|
|
18
|
+
* because every path is. So nothing is reported until `warmupRequests` have been seen,
|
|
19
|
+
* and the failure mode of the whole module is silence.
|
|
20
|
+
*
|
|
21
|
+
* **It is kept in process, like the rest of the behavioural state.** Across replicas
|
|
22
|
+
* each one therefore sees its own share of the traffic. That understates every count
|
|
23
|
+
* here, and understating is the direction that costs a missed detection rather than an
|
|
24
|
+
* accusation — which is the correct way round for a module whose mistakes would
|
|
25
|
+
* otherwise land on everybody at once.
|
|
26
|
+
*/
|
|
27
|
+
export interface SiteProfileOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Requests to observe before anything may be reported. Default 5000.
|
|
30
|
+
*
|
|
31
|
+
* The single most important number here. Below it every question this module answers
|
|
32
|
+
* has the same wrong answer — everything looks rare, because everything is.
|
|
33
|
+
*/
|
|
34
|
+
warmupRequests?: number;
|
|
35
|
+
/** Distinct paths remembered. Default 50000. */
|
|
36
|
+
maxPaths?: number;
|
|
37
|
+
/** Numeric path shapes tracked for distributed walks. Default 256. */
|
|
38
|
+
maxTemplates?: number;
|
|
39
|
+
/** Actors remembered per template before the count saturates. Default 64. */
|
|
40
|
+
maxActorsPerTemplate?: number;
|
|
41
|
+
/** How long an observation counts for. Default one hour. */
|
|
42
|
+
windowMs?: number;
|
|
43
|
+
/** Newly-appeared paths watched for a surge. Default 2048; 0 turns it off. */
|
|
44
|
+
maxWatchedPaths?: number;
|
|
45
|
+
/** Clients remembered per watched path before the count saturates. Default 64. */
|
|
46
|
+
maxActorsPerPath?: number;
|
|
47
|
+
}
|
|
48
|
+
/** What has happened to a newly-appeared path since it appeared. */
|
|
49
|
+
export interface PathSurge {
|
|
50
|
+
clients: number;
|
|
51
|
+
ageMs: number;
|
|
52
|
+
answered: number;
|
|
53
|
+
misses: number;
|
|
54
|
+
}
|
|
55
|
+
/** What the site knows about one numeric path shape right now. */
|
|
56
|
+
export interface WalkSpread {
|
|
57
|
+
actors: number;
|
|
58
|
+
/**
|
|
59
|
+
* Distinct ids requested, estimated.
|
|
60
|
+
*
|
|
61
|
+
* Exact while the bitmap is at full resolution, and `buckets * scale` once it has
|
|
62
|
+
* coarsened — which is the right estimate for the thing this exists to find, because a
|
|
63
|
+
* walk that covers its range touches every id in every bucket it touches. It
|
|
64
|
+
* *overestimates* for sparse traffic at a coarse scale, so it is never used alone: the
|
|
65
|
+
* detector requires coverage and the revisit ratio to agree.
|
|
66
|
+
*/
|
|
67
|
+
ids: number;
|
|
68
|
+
/** Buckets of the range that were touched. Exact distinct count while `scale` is 1. */
|
|
69
|
+
buckets: number;
|
|
70
|
+
/** Ids per bucket. 1 until the range outgrows the bitmap. */
|
|
71
|
+
scale: number;
|
|
72
|
+
visits: number;
|
|
73
|
+
/**
|
|
74
|
+
* Fraction of the range between the lowest and highest id that was touched.
|
|
75
|
+
*
|
|
76
|
+
* Always a number, and never zero: the range is measured from the lowest id seen to the
|
|
77
|
+
* highest, so it is at least one bucket wide and that bucket was touched by definition.
|
|
78
|
+
* A single id is a range of one, fully covered.
|
|
79
|
+
*/
|
|
80
|
+
coverage: number;
|
|
81
|
+
}
|
|
82
|
+
export declare class SiteProfile {
|
|
83
|
+
private readonly options;
|
|
84
|
+
private readonly paths;
|
|
85
|
+
private readonly walks;
|
|
86
|
+
private readonly watched;
|
|
87
|
+
private readonly clock;
|
|
88
|
+
private observed;
|
|
89
|
+
private misses;
|
|
90
|
+
private answered;
|
|
91
|
+
constructor(options: SiteProfileOptions & {
|
|
92
|
+
clock: Clock;
|
|
93
|
+
});
|
|
94
|
+
/**
|
|
95
|
+
* Whether enough traffic has been seen for any of this to mean anything.
|
|
96
|
+
*
|
|
97
|
+
* Every reader checks this. A profile that answers during warmup is worse than one
|
|
98
|
+
* that does not exist, because it answers confidently and wrongly.
|
|
99
|
+
*/
|
|
100
|
+
get warm(): boolean;
|
|
101
|
+
get requestsObserved(): number;
|
|
102
|
+
/** The share of answered requests that were misses, or `undefined` before warmup. */
|
|
103
|
+
get missRate(): number | undefined;
|
|
104
|
+
/** Files a request. Called once per assessed request, before the detectors run. */
|
|
105
|
+
record(path: string, actorKey: string): void;
|
|
106
|
+
/** Files what the application answered, for the site's miss rate and each watched path. */
|
|
107
|
+
recordOutcome(path: string, status: number): void;
|
|
108
|
+
/** What has happened to a path since it first appeared. `undefined` if not watched. */
|
|
109
|
+
surgeOf(path: string): PathSurge | undefined;
|
|
110
|
+
/**
|
|
111
|
+
* How many times the site has served this path, to anybody.
|
|
112
|
+
*
|
|
113
|
+
* `undefined` before warmup, and `0` for a path this process has not seen — which is
|
|
114
|
+
* not the same as one the site does not have, and is why the detector reading this
|
|
115
|
+
* needs a great many of them before it says anything.
|
|
116
|
+
*/
|
|
117
|
+
timesSeen(path: string): number | undefined;
|
|
118
|
+
/** Files one step of a numeric walk against the shape it belongs to. */
|
|
119
|
+
recordWalk(template: string, id: number, actorKey: string): void;
|
|
120
|
+
/** What the whole site has done with one numeric shape. `undefined` before warmup. */
|
|
121
|
+
spreadOf(template: string): WalkSpread | undefined;
|
|
122
|
+
}
|
package/dist/state.d.ts
CHANGED
|
@@ -71,6 +71,38 @@ export declare class ActorState {
|
|
|
71
71
|
* long visit is checking what exists rather than reading it, and that is a fact about
|
|
72
72
|
* the actor rather than about any one of its requests — which is why it is kept here.
|
|
73
73
|
*/
|
|
74
|
+
/**
|
|
75
|
+
* What has happened with this actor's marker cookie.
|
|
76
|
+
*
|
|
77
|
+
* Counted rather than listed: the useful questions are all "how often", and a list of
|
|
78
|
+
* marker ids would grow with a client's cookie jar for no benefit. The three drift
|
|
79
|
+
* flags are sticky — once a client has been seen claiming two different browsers under
|
|
80
|
+
* one marker it has done so, and a later request that looks tidy again does not undo
|
|
81
|
+
* it. That is the point of correlating a series rather than judging a request.
|
|
82
|
+
*/
|
|
83
|
+
/**
|
|
84
|
+
* When this actor was last challenged, and how it described itself at that moment.
|
|
85
|
+
*
|
|
86
|
+
* Kept so that what a client does *in response* to being challenged can be read. That
|
|
87
|
+
* reaction is better evidence than anything observed passively, because the stimulus
|
|
88
|
+
* was ours: we chose the moment, so a change of identity that follows it within
|
|
89
|
+
* seconds is a reaction to it rather than a coincidence we went looking for.
|
|
90
|
+
*/
|
|
91
|
+
/**
|
|
92
|
+
* Answers to challenges that were valid in form but wrong in a way only the series
|
|
93
|
+
* shows: a solution already spent, or one returned faster than the puzzle allows.
|
|
94
|
+
*/
|
|
95
|
+
/** Requests for a path no other client had ever asked this site for. */
|
|
96
|
+
private novelPaths;
|
|
97
|
+
private replayedSolutions;
|
|
98
|
+
private implausibleSolves;
|
|
99
|
+
private challengedAt;
|
|
100
|
+
private challengeShape;
|
|
101
|
+
private markerIssues;
|
|
102
|
+
private markerReturns;
|
|
103
|
+
private markerForgeries;
|
|
104
|
+
private driftSeen;
|
|
105
|
+
private driftEvents;
|
|
74
106
|
private readonly methods;
|
|
75
107
|
/**
|
|
76
108
|
* Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
|
|
@@ -83,6 +115,25 @@ export declare class ActorState {
|
|
|
83
115
|
* the span survive an actor asking for ten thousand of them.
|
|
84
116
|
*/
|
|
85
117
|
private readonly walks;
|
|
118
|
+
/**
|
|
119
|
+
* Every named identity this actor has claimed, and what kind each was.
|
|
120
|
+
*
|
|
121
|
+
* Kept because the interesting question is not what one request said but what the *set*
|
|
122
|
+
* of them says. One address claiming sqlmap and nikto is a scan; one claiming Googlebot
|
|
123
|
+
* and Bingbot is a forgery, since at most one of those can be true of an address. Neither
|
|
124
|
+
* observation exists inside a single request.
|
|
125
|
+
*/
|
|
126
|
+
private readonly identities;
|
|
127
|
+
/**
|
|
128
|
+
* A name somebody gave this actor.
|
|
129
|
+
*
|
|
130
|
+
* Nothing in detection reads it. It exists because an address is not a memory: the
|
|
131
|
+
* person who worked out that `198.51.100.4` is the partner's price feed should be able
|
|
132
|
+
* to write that down where the next person will see it, rather than in a ticket.
|
|
133
|
+
*/
|
|
134
|
+
private actorLabel;
|
|
135
|
+
/** Requests from this actor that carried a scanner payload or target. */
|
|
136
|
+
private probePayloads;
|
|
86
137
|
/**
|
|
87
138
|
* What the application answered, for the requests anybody bothered to tell us about.
|
|
88
139
|
*
|
|
@@ -115,6 +166,67 @@ export declare class ActorState {
|
|
|
115
166
|
get responses(): number;
|
|
116
167
|
/** Of those, how many were 404 or 410. */
|
|
117
168
|
get misses(): number;
|
|
169
|
+
/** Names this actor, or clears the name when given nothing. Trimmed and bounded. */
|
|
170
|
+
setLabel(label: string | undefined): void;
|
|
171
|
+
get label(): string | undefined;
|
|
172
|
+
/** Records a named identity this actor claimed. Called once per matching signature. */
|
|
173
|
+
noteIdentity(id: string, category: string, verifiable: boolean): void;
|
|
174
|
+
/** Records that this request was for a path the site had never served to anybody. */
|
|
175
|
+
noteNovelPath(): void;
|
|
176
|
+
/** How many of this actor's requests were for a path nobody else had ever asked for. */
|
|
177
|
+
get novelPathCount(): number;
|
|
178
|
+
/** Records something wrong with a submitted solution that only its history reveals. */
|
|
179
|
+
noteChallengeAnomaly(kind: "replay" | "implausible-speed"): void;
|
|
180
|
+
/** Solutions this actor submitted that had already been spent, and ones returned too fast. */
|
|
181
|
+
get challengeAnomalies(): {
|
|
182
|
+
replays: number;
|
|
183
|
+
implausible: number;
|
|
184
|
+
};
|
|
185
|
+
/** Records that a challenge went out, and the identity claimed as it did. */
|
|
186
|
+
noteChallengeIssued(at: number, shape: {
|
|
187
|
+
b: string;
|
|
188
|
+
o: string;
|
|
189
|
+
l: string;
|
|
190
|
+
} | undefined): void;
|
|
191
|
+
/** The moment of the last challenge, and the identity claimed then. `at` is 0 for none. */
|
|
192
|
+
get lastChallenge(): {
|
|
193
|
+
at: number;
|
|
194
|
+
shape: {
|
|
195
|
+
b: string;
|
|
196
|
+
o: string;
|
|
197
|
+
l: string;
|
|
198
|
+
} | undefined;
|
|
199
|
+
};
|
|
200
|
+
/** Records that a marker was handed to this actor on the way out. */
|
|
201
|
+
noteMarkerIssued(): void;
|
|
202
|
+
/** Records what this request's marker cookie turned out to be. */
|
|
203
|
+
noteMarker(returned: boolean, forged: boolean, drift: {
|
|
204
|
+
browser: boolean;
|
|
205
|
+
platform: boolean;
|
|
206
|
+
language: boolean;
|
|
207
|
+
} | undefined): void;
|
|
208
|
+
/** Markers handed to this actor, and how many came back. */
|
|
209
|
+
get markers(): {
|
|
210
|
+
issued: number;
|
|
211
|
+
returned: number;
|
|
212
|
+
forged: number;
|
|
213
|
+
};
|
|
214
|
+
/** Which parts of a claimed identity have ever changed under one marker. */
|
|
215
|
+
get identityDrift(): {
|
|
216
|
+
browser: boolean;
|
|
217
|
+
platform: boolean;
|
|
218
|
+
language: boolean;
|
|
219
|
+
events: number;
|
|
220
|
+
};
|
|
221
|
+
/** Records that this request carried a scanner payload, so later requests can know. */
|
|
222
|
+
notePayloadProbe(): void;
|
|
223
|
+
/** Every identity claimed so far, by id. */
|
|
224
|
+
get claimedIdentities(): ReadonlyMap<string, {
|
|
225
|
+
category: string;
|
|
226
|
+
verifiable: boolean;
|
|
227
|
+
}>;
|
|
228
|
+
/** How many of this actor's requests carried a scanner payload or target. */
|
|
229
|
+
get payloadProbes(): number;
|
|
118
230
|
/**
|
|
119
231
|
* Files a request under the shape of its path, if that path carries a number.
|
|
120
232
|
*
|
|
@@ -234,3 +346,21 @@ export declare class ActorRegistry {
|
|
|
234
346
|
/** Marks an actor as holding valid human clearance until `until`. */
|
|
235
347
|
clearUntil(key: string, until: number): void;
|
|
236
348
|
}
|
|
349
|
+
/**
|
|
350
|
+
* One step of a numeric walk: the shape of the path, and the number in it.
|
|
351
|
+
*
|
|
352
|
+
* `/api/v2/orders/42` becomes `/api/v2/orders/#` and `42`. Shared between the per-actor
|
|
353
|
+
* series and the site-wide one, which must agree on what a shape is or their counts
|
|
354
|
+
* cannot be compared.
|
|
355
|
+
*
|
|
356
|
+
* Both bounds are checked without allocating, because both exist to stop a client paying
|
|
357
|
+
* us to do work. `split` allocates a string per segment, so counting separators first is
|
|
358
|
+
* what makes the depth check worth having. Measured on a sixty-segment path: 3.65µs
|
|
359
|
+
* against 199ns for an ordinary one — every request eighteen times dearer, for free,
|
|
360
|
+
* from anyone willing to send a long URL. The concatenation was most of it; the split
|
|
361
|
+
* was the rest.
|
|
362
|
+
*/
|
|
363
|
+
export declare function walkStepOf(path: string): {
|
|
364
|
+
template: string;
|
|
365
|
+
id: number;
|
|
366
|
+
} | undefined;
|