@osqd/bothandlerjs 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -1
- package/README.md +18 -10
- package/dist/adapters/fastify.d.ts +10 -0
- package/dist/adapters/index.cjs +38 -10
- package/dist/adapters/index.cjs.map +1 -1
- package/dist/adapters/index.js +38 -10
- package/dist/adapters/index.js.map +1 -1
- package/dist/challenge/index.d.ts +40 -0
- package/dist/cli.cjs +2256 -103
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2256 -103
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +55 -0
- package/dist/core.d.ts +48 -21
- package/dist/corpus/index.cjs +365 -7
- package/dist/corpus/index.cjs.map +1 -1
- package/dist/corpus/index.js +365 -7
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/schema.d.ts +33 -0
- package/dist/crawler-ranges.d.ts +31 -0
- package/dist/dashboard/client/actions.d.ts +1 -1
- package/dist/dashboard/client/app.d.ts +9 -2
- package/dist/dashboard/client/boot.d.ts +32 -3
- package/dist/dashboard/client/query.d.ts +72 -12
- package/dist/dashboard/client/registry.d.ts +25 -0
- package/dist/dashboard/client/saved.d.ts +29 -0
- package/dist/dashboard/client/store.d.ts +16 -2
- package/dist/dashboard/client/types.d.ts +2 -0
- package/dist/dashboard/client.generated.d.ts +1 -1
- package/dist/dashboard/types.d.ts +15 -0
- package/dist/detectors/blended-identity.d.ts +34 -0
- package/dist/detectors/challenge-integrity.d.ts +26 -0
- package/dist/detectors/challenge-reaction.d.ts +39 -0
- package/dist/detectors/clearance.d.ts +1 -23
- package/dist/detectors/id-enumeration.d.ts +31 -0
- package/dist/detectors/index.d.ts +24 -1
- package/dist/detectors/known-bots.d.ts +11 -0
- package/dist/detectors/marker.d.ts +106 -0
- package/dist/detectors/parameter-sweep.d.ts +39 -0
- package/dist/detectors/probe-signature.d.ts +27 -0
- package/dist/detectors/probe-volume.d.ts +26 -0
- package/dist/detectors/site-baseline.d.ts +135 -0
- package/dist/detectors/target-integrity.d.ts +16 -0
- package/dist/detectors/transport-coherence.d.ts +31 -0
- package/dist/detectors/trap.d.ts +10 -3
- package/dist/detectors/types.d.ts +17 -0
- package/dist/element/index.cjs +730 -80
- package/dist/element/index.cjs.map +1 -1
- package/dist/element/index.js +730 -80
- package/dist/element/index.js.map +1 -1
- package/dist/index.cjs +2044 -123
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.js +2025 -123
- package/dist/index.js.map +1 -1
- package/dist/internal/async.d.ts +0 -3
- package/dist/internal/ip.d.ts +18 -0
- package/dist/internal/text.d.ts +28 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/probe/index.d.ts +153 -0
- package/dist/probe/marker.d.ts +119 -0
- package/dist/site/index.d.ts +122 -0
- package/dist/state.d.ts +205 -0
- package/dist/stores/redis.d.ts +24 -1
- package/dist/types.d.ts +106 -0
- package/docs/course/05-detectors.md +9 -4
- package/docs/course/06-identity.md +1 -1
- package/docs/course/16-proving-it.md +15 -9
- package/docs/course/index.md +1 -1
- package/docs/design/decisions.md +1 -1
- package/docs/detection/correlation.md +284 -0
- package/docs/detection/detectors.md +259 -1
- package/docs/detection/index.md +2 -1
- package/docs/detection/shadow-mode.md +147 -0
- package/docs/detection/signatures.md +10 -2
- package/docs/index.md +3 -2
- package/docs/integration/client-ip.md +16 -0
- package/docs/operations/dashboard.md +40 -1
- package/docs/operations/filters.md +143 -0
- package/docs/operations/index.md +1 -0
- package/docs/operations/metrics.md +18 -0
- package/docs/policy/presets.md +1 -1
- package/docs/start/choosing-a-policy.md +1 -1
- package/docs/start/first-integration.md +1 -1
- package/docs/start/installation.md +2 -2
- package/docs/testing/cli.md +7 -1
- package/docs/testing/corpus.md +12 -8
- package/docs/testing/index.md +1 -1
- package/docs/testing/try-it.md +1 -1
- package/package.json +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,127 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
8
8
|
|
|
9
9
|
### Added
|
|
10
10
|
|
|
11
|
+
- **Shadow mode: run a detector and let it decide nothing.** `shadowDetectors: ["path-novelty"]`
|
|
12
|
+
runs the named detectors on every request exactly as they otherwise would and keeps their
|
|
13
|
+
findings out of the verdict, the score, the class, the identity and every rule. They land
|
|
14
|
+
in `assessment.shadowEvidence`, counted and charted next to the evidence that did decide.
|
|
15
|
+
This is deliberately not a weight of zero: a weight is consulted only on the probabilistic
|
|
16
|
+
path, and `certain` evidence never reaches that path, so a shadowed detector emitting it
|
|
17
|
+
would have blocked people with its weight sitting at zero the whole time.
|
|
18
|
+
|
|
19
|
+
`assessment.shadowVerdict` carries what the verdict *would* have been, computed only when
|
|
20
|
+
a shadowed detector actually found something. "It fired 312 times" is not a number anybody
|
|
21
|
+
can act on; "it would have moved 41 requests to `suspected-bot`" is. Prometheus gains
|
|
22
|
+
`bothandler_shadow_firings_total` and `bothandler_shadow_verdict_changes_total`, both
|
|
23
|
+
absent entirely while nothing is shadowed.
|
|
24
|
+
|
|
25
|
+
The correlation sources are why this exists: several of their detectors fire at `moderate`
|
|
26
|
+
on real people by design — a phone roaming between networks, a crowd on a broken link,
|
|
27
|
+
somebody tapping "Request desktop site" — and whether the thresholds are right *for your
|
|
28
|
+
site* is not a thing this library can know. See `docs/detection/shadow-mode.md`.
|
|
29
|
+
|
|
30
|
+
- **`target-integrity`, and the raw request target it reads.** `facts.path` is normalised so
|
|
31
|
+
that a rule scoped to `/admin` holds against `/%61dmin` and `/./admin`. That normalisation
|
|
32
|
+
is also what makes an evasive target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml`
|
|
33
|
+
arrives as `/app/config.yml`, which is on no wordlist and reads like a broken link.
|
|
34
|
+
`facts.rawPath` now keeps the target as it was spelled — and only when it differs from the
|
|
35
|
+
normalised form, so ordinary traffic pays nothing for it.
|
|
36
|
+
|
|
37
|
+
The detector reports encoded traversals, double encoding, encoded control characters,
|
|
38
|
+
absolute-form targets addressed to a proxy, and separators hidden inside a segment. None
|
|
39
|
+
of it is `certain` and the closest call says why: a path segment carrying a URL as *data*
|
|
40
|
+
is encoded once to sit in a path and again by whatever built the link around it, which
|
|
41
|
+
produces `%252e` honestly on a site that has done nothing wrong.
|
|
42
|
+
|
|
43
|
+
- **The feed filter takes `$and`, `$or`, `$not`, `$in` and `$notin`.** Adjacent terms
|
|
44
|
+
still mean `AND` and `-term` still negates, so every existing query and saved filter
|
|
45
|
+
reads the same — but the parser produces a tree now rather than a flat list, which is
|
|
46
|
+
what `$or` needs and what the previous version communicated by silently ignoring the
|
|
47
|
+
word. `$not` binds tightest, then `$and`, then `$or`, and brackets group. Operators
|
|
48
|
+
carry a `$` because a bare `or` appears in User-Agents and paths, and a language where
|
|
49
|
+
an ordinary search word becomes an operator lies about what it matched. Nothing throws:
|
|
50
|
+
an unclosed bracket, a dangling `$or` and a half-typed `$in(` are all the normal state
|
|
51
|
+
of a live search box.
|
|
52
|
+
|
|
53
|
+
- **The Actors screen lists either the registry or the actors in the feed.** A toggle
|
|
54
|
+
above the table. The registry answers "who is hitting me hardest"; once a filter is on,
|
|
55
|
+
the question is usually the other one — "who is in *this*" — and the screen could not
|
|
56
|
+
answer it. The feed-derived list leaves `Per min`, `Cadence` and `Unsolved` blank rather
|
|
57
|
+
than computing them from a few hundred requests, because those are properties of a
|
|
58
|
+
client's whole history and a confident number under the wrong heading is worse than a
|
|
59
|
+
dash.
|
|
60
|
+
|
|
61
|
+
### Changed
|
|
62
|
+
|
|
63
|
+
- **The browser suite runs on Chromium, Firefox and WebKit.** `BROWSER_ENGINE` picks one
|
|
64
|
+
locally (`npm run test:browser:firefox`, `:webkit`); CI runs all three as a matrix.
|
|
65
|
+
Testing a single engine is how the Safari header bug shipped, and Firefox earned its
|
|
66
|
+
place on the first run by catching the challenge page's CSP error.
|
|
67
|
+
|
|
68
|
+
- **The feed's column headers stick in Safari.** The tables collapsed their borders, which
|
|
69
|
+
is a long-standing sore point for sticky table cells in WebKit — the CSSWG has an open
|
|
70
|
+
issue on collapsed borders not following a cell when it sticks, and Safari is widely
|
|
71
|
+
reported to drop the stickiness of a `th` outright. The header held in Chromium and
|
|
72
|
+
Firefox and was reported adrift in Safari, which is what a sticky element that has
|
|
73
|
+
stopped sticking looks like. They separate their
|
|
74
|
+
borders with zero spacing now, which no border in these tables relied on: the only
|
|
75
|
+
measurable difference is the accent column starting two pixels earlier, because
|
|
76
|
+
collapsing left half of that 3px border outside the cell. A browser test scrolls the
|
|
77
|
+
page and asserts the header holds, since one keyword can undo this silently.
|
|
78
|
+
|
|
79
|
+
- **The feed's Exclude button is gone; `$not` replaces it.** It kept a hidden list in one
|
|
80
|
+
person's browser, which meant a view with the noise taken out could not be shared. A
|
|
81
|
+
`$not` lives in the URL like every other narrowing.
|
|
82
|
+
|
|
83
|
+
- **Labelling an actor edits in place instead of calling `prompt()`.** A sandboxed iframe
|
|
84
|
+
blocks `prompt()` outright, so on an embedded dashboard the Label button did nothing at
|
|
85
|
+
all, with no error and no way to tell.
|
|
86
|
+
|
|
87
|
+
- **Correlating a client's own requests, through a marker cookie.** `probe` is a new,
|
|
88
|
+
opt-in source: the engine issues a signed first-party cookie and reads it back, so two
|
|
89
|
+
requests can be attributed to one *client* rather than to one address. That closes a gap
|
|
90
|
+
the library had documented and declined to guess at — `identity-rotation` reads "one
|
|
91
|
+
actor, several User-Agents" as lying, and under an address-derived actor key that
|
|
92
|
+
describes every office and carrier on the internet, so it has always shipped switched
|
|
93
|
+
off. A marker carries an HMAC only this server can produce, which makes the same
|
|
94
|
+
observation evidence instead of speculation.
|
|
95
|
+
|
|
96
|
+
Four detectors arrive with it. `identity-drift` compares the identity claimed now with
|
|
97
|
+
the one claimed when the marker was issued, weighing a changed *browser family* at
|
|
98
|
+
`strong` and a changed platform at `moderate` — because "Request desktop site" on a
|
|
99
|
+
phone does the latter and the person doing it is a person. `marker-integrity` reports a
|
|
100
|
+
marker signed with a key we do not have. `marker-fanout` counts the networks one marker
|
|
101
|
+
has been presented from. `marker-persistence` reports a client that sends cookies but
|
|
102
|
+
never ours, deliberately narrower than `session-integrity`, which already covers a
|
|
103
|
+
client that sends none.
|
|
104
|
+
|
|
105
|
+
It is off by default and sets a cookie only when the client holds no valid one, so an
|
|
106
|
+
ordinary visitor is issued one on the first request of a session and no other. `secrets`
|
|
107
|
+
is required and has no default: a secret generated at startup would read every marker
|
|
108
|
+
minted by another replica as forged.
|
|
109
|
+
|
|
110
|
+
- **Reading what a client does when it is challenged.** `challenge-reaction` reports an
|
|
111
|
+
identity that changes within seconds of a challenge, and a client challenged repeatedly
|
|
112
|
+
that has never answered. It is the only detector here whose stimulus this library chose,
|
|
113
|
+
which is what lets it reach `strong` — and it reports the same observation a tier lower
|
|
114
|
+
when only an address ties the two requests together, because that is how much less an
|
|
115
|
+
address-based join is worth. `challenge-integrity` reports solutions that were replayed,
|
|
116
|
+
or returned faster than the proof of work can be computed in a browser.
|
|
117
|
+
|
|
118
|
+
- **Comparing a client with the rest of your traffic.** `site` is a second opt-in source
|
|
119
|
+
holding a bounded, warmed-up baseline of what a site normally serves. `distributed-walk`
|
|
120
|
+
finds a numeric range walked across many clients where none walks enough of it alone —
|
|
121
|
+
the one threat per-actor thresholds miss by construction. `path-novelty` is a
|
|
122
|
+
self-maintaining wordlist. `path-campaign` reports a path the site never served that
|
|
123
|
+
many unrelated clients suddenly want, requiring the miss rate so that a successful
|
|
124
|
+
launch is not reported as an attack. `miss-baseline` is `probe-volume` measured against
|
|
125
|
+
the site's own rate rather than a fixed threshold.
|
|
126
|
+
|
|
127
|
+
Nothing is reported until `warmupRequests` have been observed, because every path is
|
|
128
|
+
rare when nothing has been seen. Walk ids are held as a coarsening bitmap and marker
|
|
129
|
+
fan-out as a 128-bit sketch, so neither table grows with what a client chooses to
|
|
130
|
+
request.
|
|
131
|
+
|
|
11
132
|
- **`indexers-only`, a preset for sites that want search traffic and nothing else.**
|
|
12
133
|
Serves a bot only when its identity has been confirmed by DNS or a published range
|
|
13
134
|
*and* it is a `search` or `social` crawler; refuses every other proven bot; challenges
|
|
@@ -106,6 +227,111 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
106
227
|
|
|
107
228
|
### Fixed
|
|
108
229
|
|
|
230
|
+
- **An embedded dashboard could point at the host application instead of at `src`.**
|
|
231
|
+
`disconnectedCallback` imports the stream module to close the stream, that module
|
|
232
|
+
imports the boot module, and the boot module read its mount path off the global once,
|
|
233
|
+
when it was first evaluated. A host page that mounted, unmounted and remounted the
|
|
234
|
+
element before the first bootstrap fetch returned — React 18's development double-mount
|
|
235
|
+
— ran that import while the global was still undefined, so the base froze at `""` for
|
|
236
|
+
the life of the page.
|
|
237
|
+
|
|
238
|
+
Nothing threw. Every request the dashboard made afterwards went to the host page's own
|
|
239
|
+
origin root rather than to `src`, so it drew no traffic while sending `/api/stream` and
|
|
240
|
+
`/api/stats` to somebody else's router, and the only outward sign was an `EventSource`
|
|
241
|
+
complaining about a MIME type on whichever engine reports that. The element now hands
|
|
242
|
+
the payload over explicitly, so being evaluated early costs nothing.
|
|
243
|
+
|
|
244
|
+
- **Pressing "Load them" left the "not streamed" badge on screen.** The count of how much
|
|
245
|
+
of the gap had been closed was taken from the page's own snapshot, which is refreshed on
|
|
246
|
+
a timer and so was usually several seconds out of date; a snapshot arriving afterwards
|
|
247
|
+
with a larger count reopened a gap that had just been closed. `/api/feed` now returns
|
|
248
|
+
`skipped` alongside the backlog, so the number comes from the response that closed the
|
|
249
|
+
gap rather than from one taken before it.
|
|
250
|
+
|
|
251
|
+
- **`RedisStore.increment` could leave a counter key that never expired.** `INCR` then
|
|
252
|
+
`PEXPIRE` is two commands, and a process killed between them orphaned a key nothing would
|
|
253
|
+
ever revisit — the next request falls into the next bucket, under another key. The expiry
|
|
254
|
+
is now armed by the command that *creates* the key: `SET … PX … NX` issued without waiting
|
|
255
|
+
for its reply, then `INCR`, so both are on the wire together and this stays one round trip.
|
|
256
|
+
Not a Lua script, because `eval` is the one command `ioredis` and `node-redis` spell
|
|
257
|
+
differently enough that `RedisLike` could not describe both. `pexpire` has left that
|
|
258
|
+
interface, which needs four commands now rather than five.
|
|
259
|
+
|
|
260
|
+
- **The interaction challenge marked down somebody who paused mid-movement, twice over.**
|
|
261
|
+
Samples arriving more than a quarter of a second apart are dropped before anything is
|
|
262
|
+
measured, because the distance across a pause is not a distance a hand travelled — but
|
|
263
|
+
`fractionalShare` was dividing the samples it kept by the count of everything that
|
|
264
|
+
arrived, reporting *fewer* sub-pixel coordinates than the samples it was computed from
|
|
265
|
+
actually had. That reads as "these coordinates are integers", which is the signature of an
|
|
266
|
+
interpolated path. Separately, whether a path was measurable at all was decided from the
|
|
267
|
+
raw array length while the scoring judged the kept samples: four pointer samples with a
|
|
268
|
+
pause before each one were four to the array and none to the analysis, so the report was
|
|
269
|
+
called measurable and then scored at zero. Both cost the reading pattern most likely to
|
|
270
|
+
produce them — move the pointer, stop to read, move again.
|
|
271
|
+
|
|
272
|
+
- **Save and Cancel on the actor label editor, which three earlier attempts could not fit.**
|
|
273
|
+
`input[type="text"] { width: 100% }` outranks a bare class selector, so the editor's
|
|
274
|
+
`width: 15ch` had never applied: the box filled its shrink-to-fit container and pushed the
|
|
275
|
+
buttons past the right edge of the panel, where they could be seen and not clicked.
|
|
276
|
+
Qualifying the selector fixed the cause; the row's other actions now stand aside while the
|
|
277
|
+
editor is open, which is also the right thing on its own — Allowlist and Forget are not
|
|
278
|
+
what somebody naming a client is reaching for.
|
|
279
|
+
|
|
280
|
+
- **The Actors scope was not in the URL.** `#actors?a=feed`, pushed rather than replaced,
|
|
281
|
+
because switching between the registry and the feed is a discrete act like clicking a tab
|
|
282
|
+
and the back button should undo it. The feed-scoped list also shows a dash for
|
|
283
|
+
`Confirmations` like the three columns beside it, rather than a confident zero under a
|
|
284
|
+
heading that means "how many times has this client been proven a bot".
|
|
285
|
+
|
|
286
|
+
- **`bothandlerjs detectors --preset <typo>` answered anyway.** An unknown preset fell
|
|
287
|
+
through to a handler with no preset and printed the default list at exit code 0 — a
|
|
288
|
+
confident wrong answer to the one question the command exists for, while `robots` in the
|
|
289
|
+
same file refused the same typo. It is refused now, with the same message. The command
|
|
290
|
+
also notes that a preset selects rules rather than detectors, and that `challenge`,
|
|
291
|
+
`probe` and `site` are what change the list; the note goes to stderr, so redirecting the
|
|
292
|
+
list stays clean.
|
|
293
|
+
|
|
294
|
+
- **The default notification sink discarded the content of every error.** `consoleNotifier`
|
|
295
|
+
never referenced `event.error`. An error carrying no assessment fell into the branch
|
|
296
|
+
written for anomalies and printed `[bothandler] error unknown — ` at *warning* level, with
|
|
297
|
+
the failing source and the message both dropped; one that did carry an assessment printed
|
|
298
|
+
the request's evidence instead. Errors are how a failed detector, sink or store is
|
|
299
|
+
reported, and this is the sink an operator gets without configuring one — so the default
|
|
300
|
+
way to find out that a detector had been throwing showed neither which one nor why.
|
|
301
|
+
|
|
302
|
+
- **Every person shown a challenge page in Firefox got a security error in their
|
|
303
|
+
console.** The page declares no icon, so the browser asks for `/favicon.ico` by itself;
|
|
304
|
+
under `default-src 'none'` that request is refused, and Firefox reports the refusal as a
|
|
305
|
+
CSP violation on a page whose entire purpose is to reassure somebody that nothing is
|
|
306
|
+
wrong. The page now declares an empty icon so the request is never made, and the policy
|
|
307
|
+
allows `img-src data:` — which permits nothing off the machine, a `data:` URI being
|
|
308
|
+
inline by definition — so that declaration is honoured. Found by running the browser
|
|
309
|
+
suite on Firefox for the first time.
|
|
310
|
+
|
|
311
|
+
- **Cookies split across several header fields were misparsed, losing every cookie after
|
|
312
|
+
the first.** HTTP/2 permits a client to send its cookies as separate header fields and
|
|
313
|
+
Node's `http2` surfaces them as an array; RFC 9113 §8.2.3 says a receiver concatenates
|
|
314
|
+
them with `"; "`. They were joined with `", "` like every other header, which parses as
|
|
315
|
+
one cookie whose value is the rest of the line — so a clearance token in the second
|
|
316
|
+
field was invisible, and an HTTP/2 visitor who had solved a challenge was challenged
|
|
317
|
+
again on every request.
|
|
318
|
+
|
|
319
|
+
- **`X-Forwarded-For` entries carrying a port were dropped.** Azure's Application Gateway
|
|
320
|
+
and Front Door write `1.2.3.4:5678`, and RFC 7239 spells IPv6 as `[2001:db8::1]:5678`.
|
|
321
|
+
Neither parsed, and because every entry in such a chain carries a port the whole chain
|
|
322
|
+
emptied and every client behind that proxy resolved to the proxy's own address — sharing
|
|
323
|
+
one actor, one history and one rate-limit bucket, so a single bot could lock out every
|
|
324
|
+
real visitor.
|
|
325
|
+
|
|
326
|
+
- **Evidence summaries quoted client-controlled text without neutralising it.** A request
|
|
327
|
+
path containing CRLF came back inside an `id-enumeration` summary exactly as sent, which
|
|
328
|
+
forges log lines; escape sequences repainted terminals. Summaries and operator labels are
|
|
329
|
+
now cleaned centrally, so detectors written elsewhere are covered too.
|
|
330
|
+
|
|
331
|
+
- **`probe-volume` was inert on three of the four adapters.** Only the Node adapter
|
|
332
|
+
reported the status the application answered, so on Fastify, Koa and every Fetch runtime
|
|
333
|
+
the detector was installed, listed, and structurally unable to fire.
|
|
334
|
+
|
|
109
335
|
- **The "N not streamed" badge outlived the feed it described.** The badge adds two
|
|
110
336
|
counts: the server's rate-cap `skipped`, which `FeedRing.clear()` resets, and the
|
|
111
337
|
connection's lagged drops, which nothing did. After a Reset — or after the replace-sync
|
|
@@ -281,7 +507,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
281
507
|
looks like a challenge that worked, which is why it needed saying. `AuditWindow` gains
|
|
282
508
|
`challengesSolved` and `challengeSolveRate`.
|
|
283
509
|
|
|
284
|
-
- **`bothandlerjs check` — your policy against
|
|
510
|
+
- **`bothandlerjs check` — your policy against 545 shapes of real traffic.** The question
|
|
285
511
|
the library is organised around, asked offline and before a deploy: *if I point this
|
|
286
512
|
configuration at the actual internet, who gets hurt?* It exits non-zero if any case
|
|
287
513
|
marked `human` is denied service, which is what makes it a CI step rather than a
|
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ No runtime dependencies. The library imports nothing but `node:` builtins, and C
|
|
|
23
23
|
fails if that ever stops being true — so nothing here can hand your project a
|
|
24
24
|
transitive advisory, an install script, or a version conflict with something you
|
|
25
25
|
already run. Redis, if you use it, is your client passed in: `RedisStore` describes
|
|
26
|
-
the
|
|
26
|
+
the four commands it needs structurally and imports neither `ioredis` nor
|
|
27
27
|
`node-redis`.
|
|
28
28
|
|
|
29
29
|
---
|
|
@@ -31,15 +31,15 @@ the five commands it needs structurally and imports neither `ioredis` nor
|
|
|
31
31
|
## Documentation
|
|
32
32
|
|
|
33
33
|
The README is the argument and the shortest path to a working integration. Everything
|
|
34
|
-
else lives in **[`docs/`](docs/index.md)** —
|
|
35
|
-
|
|
34
|
+
else lives in **[`docs/`](docs/index.md)** — a page per question, each explaining why a
|
|
35
|
+
thing exists as well as how to use it.
|
|
36
36
|
|
|
37
37
|
| | |
|
|
38
38
|
| --- | --- |
|
|
39
39
|
| **[The course](docs/course/index.md)** | Sixteen lessons that build one integration, from a first assessment to a policy you can defend. Start here if the library is new to you. |
|
|
40
40
|
| **[Start here](docs/index.md)** | [Installation](docs/start/installation.md) · [Your first integration](docs/start/first-integration.md) · [Choosing a policy](docs/start/choosing-a-policy.md) · [Upgrading](docs/start/upgrading.md) |
|
|
41
41
|
| **Concepts** | [Evidence and certainty](docs/concepts/evidence.md) · [Verdicts and scores](docs/concepts/verdicts.md) · [The safety guard](docs/concepts/the-guard.md) · [Actors](docs/concepts/actors.md) · [Threat model](docs/concepts/threat-model.md) |
|
|
42
|
-
| **[Detection](docs/detection/index.md)** | [The
|
|
42
|
+
| **[Detection](docs/detection/index.md)** | [The detectors](docs/detection/detectors.md) · [Correlation](docs/detection/correlation.md) · [Signatures](docs/detection/signatures.md) · [Verification](docs/detection/verification.md) · [Browser signals](docs/detection/client-signals.md) · [Writing a detector](docs/detection/writing-a-detector.md) |
|
|
43
43
|
| **[Policy](docs/policy/index.md)** | [Rules](docs/policy/rules.md) · [Actions](docs/policy/actions.md) · [Presets](docs/policy/presets.md) · [robots.txt](docs/policy/robots.md) · [The challenge](docs/challenge/index.md) |
|
|
44
44
|
| **[Operations](docs/operations/index.md)** | [The dashboard](docs/operations/dashboard.md) · [Embedding it](docs/operations/embedding.md) · [Metrics](docs/operations/metrics.md) · [The audit](docs/operations/audit.md) · [Notifications](docs/operations/notifications.md) · [Runtime changes](docs/operations/runtime-changes.md) |
|
|
45
45
|
| **[Integration](docs/integration/index.md)** | [Adapters](docs/integration/adapters.md) · [The client IP](docs/integration/client-ip.md) · [Stores](docs/integration/stores.md) |
|
|
@@ -233,12 +233,20 @@ all three.
|
|
|
233
233
|
stale mapping is a false positive with a long half-life. Bring your own, from a
|
|
234
234
|
source you refresh and can audit.
|
|
235
235
|
- **Be right about a shared address.** Behind CGNAT, "one actor" is thousands of
|
|
236
|
-
people. That is why the behavioural signals are capped where they are.
|
|
236
|
+
people. That is why the behavioural signals are capped where they are. The optional
|
|
237
|
+
[marker cookie](docs/detection/correlation.md) narrows this — two requests carrying one
|
|
238
|
+
signed marker are one client rather than one address — but only for clients that keep
|
|
239
|
+
cookies, and the address remains what everything else is keyed on.
|
|
237
240
|
- **Escalate on a wordlist walk.** `probe-signature` reads one request at a time, so a
|
|
238
241
|
scanner working through five hundred paths produces five hundred separate
|
|
239
242
|
observations rather than a mounting case. That is the price of a detector that runs
|
|
240
243
|
unchanged over a log file; enumeration over time is what `rate-anomaly`, `cadence`
|
|
241
244
|
and `crawl-breadth` are for.
|
|
245
|
+
- **See an enumeration split across enough clients — unless you ask it to.** Divide a
|
|
246
|
+
range between five hundred addresses and every per-actor threshold is defeated by
|
|
247
|
+
construction, because each actor is genuinely unremarkable. `distributed-walk` finds it
|
|
248
|
+
in the union of what those addresses asked for, and needs the optional
|
|
249
|
+
[site baseline](docs/detection/correlation.md) and enough traffic to have one.
|
|
242
250
|
|
|
243
251
|
---
|
|
244
252
|
|
|
@@ -254,13 +262,13 @@ npm run build # ESM + CJS + declarations
|
|
|
254
262
|
npm run demo # protected site :9673 + live dashboard :9674
|
|
255
263
|
npm run demo:roles # the same dashboard behind roles: analyst :9684, operator :9685, admin :9686
|
|
256
264
|
npm run simulate # eighteen curated scenarios against the demo
|
|
257
|
-
npm run simulate:corpus # replay all
|
|
265
|
+
npm run simulate:corpus # replay all 548 corpus cases over a real socket
|
|
258
266
|
npm run bench # hot-path benchmark, median of several rounds
|
|
259
|
-
npm run corpus #
|
|
267
|
+
npm run corpus # 548 shapes of real traffic against your policy
|
|
260
268
|
npm run example # a minimal Express integration on :3000
|
|
261
269
|
|
|
262
270
|
npx @osqd/bothandlerjs replay access.log # what your policy would have done
|
|
263
|
-
npx @osqd/bothandlerjs check # your policy against
|
|
271
|
+
npx @osqd/bothandlerjs check # your policy against 548 shapes of real traffic
|
|
264
272
|
npx @osqd/bothandlerjs explain "curl/8.4.0" # one request, and the evidence behind the verdict
|
|
265
273
|
```
|
|
266
274
|
|
|
@@ -276,7 +284,7 @@ src/
|
|
|
276
284
|
config.ts validation, defaults, client-IP resolution
|
|
277
285
|
facts.ts request normalisation
|
|
278
286
|
state.ts bounded per-actor behavioural memory
|
|
279
|
-
detectors/
|
|
287
|
+
detectors/ the detectors + the signature database
|
|
280
288
|
policy/ rules, matcher, the safety guard, presets
|
|
281
289
|
actions/ decision -> framework-neutral outcome
|
|
282
290
|
challenge/ proof of work, signed tokens, the interstitial
|
|
@@ -287,7 +295,7 @@ src/
|
|
|
287
295
|
adapters/ Express/Connect, Fastify, Koa, Fetch
|
|
288
296
|
client/ browser-side signal script
|
|
289
297
|
internal/ IP, crypto, UA, Aho-Corasick, LRU, DNS, HTTP
|
|
290
|
-
corpus/
|
|
298
|
+
corpus/ 548 shapes of real traffic, with provenance, and the harness
|
|
291
299
|
that runs them against your configuration
|
|
292
300
|
demo/ the protected site and the live dashboard
|
|
293
301
|
scripts/simulate.ts the traffic simulator
|
|
@@ -15,6 +15,16 @@ export interface FastifyLikeReply {
|
|
|
15
15
|
header(name: string, value: string): FastifyLikeReply;
|
|
16
16
|
send(body: unknown): unknown;
|
|
17
17
|
hijack?(): void;
|
|
18
|
+
/**
|
|
19
|
+
* The underlying response. Optional because this type describes the shape the adapter
|
|
20
|
+
* needs rather than Fastify's own, and a test double should not have to build one —
|
|
21
|
+
* but Fastify always provides it, and without it the status this request ends up
|
|
22
|
+
* answering is never reported back.
|
|
23
|
+
*/
|
|
24
|
+
raw?: {
|
|
25
|
+
once(event: "finish", listener: () => void): unknown;
|
|
26
|
+
statusCode: number;
|
|
27
|
+
};
|
|
18
28
|
}
|
|
19
29
|
export interface FastifyAdapterOptions {
|
|
20
30
|
tlsFingerprintHeader?: string;
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -294,6 +294,7 @@ var SPECIAL_USE = new IpRangeSet(SPECIAL_USE_RANGES);
|
|
|
294
294
|
|
|
295
295
|
// src/facts.ts
|
|
296
296
|
var MAX_URL_LENGTH = 8192;
|
|
297
|
+
var MAX_RAW_PATH = 512;
|
|
297
298
|
var MAX_QUERY_PARAMS = 64;
|
|
298
299
|
function createFacts(input) {
|
|
299
300
|
const rawUrl = input.url ?? "/";
|
|
@@ -302,18 +303,21 @@ function createFacts(input) {
|
|
|
302
303
|
const rawPath = queryStart === -1 ? url : url.slice(0, queryStart);
|
|
303
304
|
const headers = /* @__PURE__ */ Object.create(null);
|
|
304
305
|
for (const [name, value] of Object.entries(input.headers)) {
|
|
305
|
-
const
|
|
306
|
-
|
|
306
|
+
const lower = name.toLowerCase();
|
|
307
|
+
const joined = lower === "cookie" && Array.isArray(value) ? value.join("; ") : joinHeaderValue(value);
|
|
308
|
+
if (joined !== void 0) headers[lower] = joined;
|
|
307
309
|
}
|
|
310
|
+
const normalized = normalizePath(rawPath);
|
|
308
311
|
const facts = {
|
|
309
312
|
method: (input.method ?? "GET").toUpperCase(),
|
|
310
|
-
path:
|
|
313
|
+
path: normalized,
|
|
311
314
|
query: parseQuery(queryStart === -1 ? "" : url.slice(queryStart + 1)),
|
|
312
315
|
headers,
|
|
313
316
|
headerOrder: extractOrder(input.rawHeaders, headers),
|
|
314
317
|
ip: normalizeIp(input.ip) ?? input.ip,
|
|
315
318
|
timestamp: input.timestamp ?? Date.now()
|
|
316
319
|
};
|
|
320
|
+
if (rawPath !== normalized) facts.rawPath = rawPath.length > MAX_RAW_PATH ? rawPath.slice(0, MAX_RAW_PATH) : rawPath;
|
|
317
321
|
const cookieHeader = headers["cookie"];
|
|
318
322
|
if (cookieHeader !== void 0) facts.cookies = parseCookies(cookieHeader);
|
|
319
323
|
if (input.protocol !== void 0) facts.protocol = input.protocol;
|
|
@@ -346,12 +350,23 @@ function parseQuery(search) {
|
|
|
346
350
|
const query = /* @__PURE__ */ Object.create(null);
|
|
347
351
|
if (search.length === 0) return query;
|
|
348
352
|
let count = 0;
|
|
349
|
-
for (const [key, value] of new URLSearchParams(search)) {
|
|
353
|
+
for (const [key, value] of new URLSearchParams(boundedSearch(search))) {
|
|
350
354
|
if (count++ >= MAX_QUERY_PARAMS) break;
|
|
351
355
|
query[key] = value.length > 1024 ? value.slice(0, 1024) : value;
|
|
352
356
|
}
|
|
353
357
|
return query;
|
|
354
358
|
}
|
|
359
|
+
function boundedSearch(search) {
|
|
360
|
+
let seen = 0;
|
|
361
|
+
let at = search.charCodeAt(0) === 63 ? 1 : 0;
|
|
362
|
+
while (at < search.length) {
|
|
363
|
+
let end = search.indexOf("&", at);
|
|
364
|
+
if (end === -1) end = search.length;
|
|
365
|
+
if (end !== at && ++seen > MAX_QUERY_PARAMS) return search.slice(0, at - 1);
|
|
366
|
+
at = end + 1;
|
|
367
|
+
}
|
|
368
|
+
return search;
|
|
369
|
+
}
|
|
355
370
|
var MAX_ORDERED_HEADERS = 64;
|
|
356
371
|
function extractOrder(rawHeaders, headers) {
|
|
357
372
|
if (!rawHeaders || rawHeaders.length === 0) return EMPTY_ORDER;
|
|
@@ -474,6 +489,11 @@ function botHandler(handler, options = {}) {
|
|
|
474
489
|
for (const [name, value] of Object.entries(outcome.responseHeaders)) response.setHeader(name, value);
|
|
475
490
|
}
|
|
476
491
|
if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
|
|
492
|
+
if (typeof response.once === "function") {
|
|
493
|
+
response.once("finish", () => {
|
|
494
|
+
handler.recordOutcome(facts, response.statusCode);
|
|
495
|
+
});
|
|
496
|
+
}
|
|
477
497
|
handedOff = true;
|
|
478
498
|
next();
|
|
479
499
|
} catch (error) {
|
|
@@ -570,6 +590,7 @@ function withBotHandler(handler, next, options = {}) {
|
|
|
570
590
|
const decision = await evaluate3(request, context);
|
|
571
591
|
if (decision.response) return decision.response;
|
|
572
592
|
const response = await next(decision.request, context);
|
|
593
|
+
if (decision.result !== void 0) handler.recordOutcome(decision.result.assessment.facts, response.status);
|
|
573
594
|
const extra = decision.result?.outcome.kind === "continue" ? decision.result.outcome.responseHeaders : void 0;
|
|
574
595
|
if (!extra) return response;
|
|
575
596
|
const merged = new Response(response.body, response);
|
|
@@ -653,6 +674,8 @@ async function evaluate(handler, options, mountChallenge, request, reply) {
|
|
|
653
674
|
for (const [name, value] of Object.entries(outcome.responseHeaders)) reply.header(name, value);
|
|
654
675
|
}
|
|
655
676
|
if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
|
|
677
|
+
const raw = reply.raw;
|
|
678
|
+
if (raw !== void 0) raw.once("finish", () => handler.recordOutcome(facts, raw.statusCode));
|
|
656
679
|
}
|
|
657
680
|
|
|
658
681
|
// src/adapters/koa.ts
|
|
@@ -660,13 +683,18 @@ function koaBotHandler(handler, options = {}) {
|
|
|
660
683
|
const mountChallenge = options.mountChallengeEndpoint ?? true;
|
|
661
684
|
return async function botHandlerMiddleware(context, next) {
|
|
662
685
|
let proceed;
|
|
686
|
+
let facts;
|
|
663
687
|
try {
|
|
664
|
-
|
|
688
|
+
const decision = await evaluate2(handler, mountChallenge, context);
|
|
689
|
+
proceed = decision.proceed;
|
|
690
|
+
facts = decision.facts;
|
|
665
691
|
} catch (error) {
|
|
666
692
|
handler.config.onError(error, { source: "adapter:koa" });
|
|
667
693
|
proceed = true;
|
|
668
694
|
}
|
|
669
|
-
if (proceed)
|
|
695
|
+
if (!proceed) return;
|
|
696
|
+
await next();
|
|
697
|
+
if (facts !== void 0) handler.recordOutcome(facts, context.status);
|
|
670
698
|
};
|
|
671
699
|
}
|
|
672
700
|
async function evaluate2(handler, mountChallenge, context) {
|
|
@@ -690,18 +718,18 @@ async function evaluate2(handler, mountChallenge, context) {
|
|
|
690
718
|
context.set("cache-control", "no-store");
|
|
691
719
|
if (verification.ok) context.set("set-cookie", verification.setCookie);
|
|
692
720
|
context.body = verificationBody(verification);
|
|
693
|
-
return false;
|
|
721
|
+
return { proceed: false };
|
|
694
722
|
}
|
|
695
723
|
const { outcome } = await handler.handle(facts);
|
|
696
724
|
if (outcome.kind === "drop") {
|
|
697
725
|
context.req.socket.destroy();
|
|
698
|
-
return false;
|
|
726
|
+
return { proceed: false };
|
|
699
727
|
}
|
|
700
728
|
if (outcome.kind === "respond") {
|
|
701
729
|
context.status = outcome.status;
|
|
702
730
|
for (const [name, value] of Object.entries(outcome.headers)) context.set(name, value);
|
|
703
731
|
context.body = outcome.body;
|
|
704
|
-
return false;
|
|
732
|
+
return { proceed: false };
|
|
705
733
|
}
|
|
706
734
|
for (const [name, value] of Object.entries(outcome.requestHeaders ?? {})) {
|
|
707
735
|
context.req.headers[name] = value;
|
|
@@ -710,7 +738,7 @@ async function evaluate2(handler, mountChallenge, context) {
|
|
|
710
738
|
for (const [name, value] of Object.entries(outcome.responseHeaders)) context.set(name, value);
|
|
711
739
|
}
|
|
712
740
|
if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
|
|
713
|
-
return true;
|
|
741
|
+
return { proceed: true, facts };
|
|
714
742
|
}
|
|
715
743
|
// Annotate the CommonJS export names for ESM import in node:
|
|
716
744
|
0 && (module.exports = {
|