@osqd/bothandlerjs 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +227 -1
  2. package/README.md +18 -10
  3. package/dist/adapters/fastify.d.ts +10 -0
  4. package/dist/adapters/index.cjs +38 -10
  5. package/dist/adapters/index.cjs.map +1 -1
  6. package/dist/adapters/index.js +38 -10
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/challenge/index.d.ts +40 -0
  9. package/dist/cli.cjs +2256 -103
  10. package/dist/cli.cjs.map +1 -1
  11. package/dist/cli.js +2256 -103
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts +55 -0
  14. package/dist/core.d.ts +48 -21
  15. package/dist/corpus/index.cjs +365 -7
  16. package/dist/corpus/index.cjs.map +1 -1
  17. package/dist/corpus/index.js +365 -7
  18. package/dist/corpus/index.js.map +1 -1
  19. package/dist/corpus/schema.d.ts +33 -0
  20. package/dist/crawler-ranges.d.ts +31 -0
  21. package/dist/dashboard/client/actions.d.ts +1 -1
  22. package/dist/dashboard/client/app.d.ts +9 -2
  23. package/dist/dashboard/client/boot.d.ts +32 -3
  24. package/dist/dashboard/client/query.d.ts +72 -12
  25. package/dist/dashboard/client/registry.d.ts +25 -0
  26. package/dist/dashboard/client/saved.d.ts +29 -0
  27. package/dist/dashboard/client/store.d.ts +16 -2
  28. package/dist/dashboard/client/types.d.ts +2 -0
  29. package/dist/dashboard/client.generated.d.ts +1 -1
  30. package/dist/dashboard/types.d.ts +15 -0
  31. package/dist/detectors/blended-identity.d.ts +34 -0
  32. package/dist/detectors/challenge-integrity.d.ts +26 -0
  33. package/dist/detectors/challenge-reaction.d.ts +39 -0
  34. package/dist/detectors/clearance.d.ts +1 -23
  35. package/dist/detectors/id-enumeration.d.ts +31 -0
  36. package/dist/detectors/index.d.ts +24 -1
  37. package/dist/detectors/known-bots.d.ts +11 -0
  38. package/dist/detectors/marker.d.ts +106 -0
  39. package/dist/detectors/parameter-sweep.d.ts +39 -0
  40. package/dist/detectors/probe-signature.d.ts +27 -0
  41. package/dist/detectors/probe-volume.d.ts +26 -0
  42. package/dist/detectors/site-baseline.d.ts +135 -0
  43. package/dist/detectors/target-integrity.d.ts +16 -0
  44. package/dist/detectors/transport-coherence.d.ts +31 -0
  45. package/dist/detectors/trap.d.ts +10 -3
  46. package/dist/detectors/types.d.ts +17 -0
  47. package/dist/element/index.cjs +730 -80
  48. package/dist/element/index.cjs.map +1 -1
  49. package/dist/element/index.js +730 -80
  50. package/dist/element/index.js.map +1 -1
  51. package/dist/index.cjs +2044 -123
  52. package/dist/index.cjs.map +1 -1
  53. package/dist/index.d.ts +6 -2
  54. package/dist/index.js +2025 -123
  55. package/dist/index.js.map +1 -1
  56. package/dist/internal/async.d.ts +0 -3
  57. package/dist/internal/ip.d.ts +18 -0
  58. package/dist/internal/text.d.ts +28 -0
  59. package/dist/metrics.d.ts +18 -0
  60. package/dist/probe/index.d.ts +153 -0
  61. package/dist/probe/marker.d.ts +119 -0
  62. package/dist/site/index.d.ts +122 -0
  63. package/dist/state.d.ts +205 -0
  64. package/dist/stores/redis.d.ts +24 -1
  65. package/dist/types.d.ts +106 -0
  66. package/docs/course/05-detectors.md +9 -4
  67. package/docs/course/06-identity.md +1 -1
  68. package/docs/course/16-proving-it.md +15 -9
  69. package/docs/course/index.md +1 -1
  70. package/docs/design/decisions.md +1 -1
  71. package/docs/detection/correlation.md +284 -0
  72. package/docs/detection/detectors.md +259 -1
  73. package/docs/detection/index.md +2 -1
  74. package/docs/detection/shadow-mode.md +147 -0
  75. package/docs/detection/signatures.md +10 -2
  76. package/docs/index.md +3 -2
  77. package/docs/integration/client-ip.md +16 -0
  78. package/docs/operations/dashboard.md +40 -1
  79. package/docs/operations/filters.md +143 -0
  80. package/docs/operations/index.md +1 -0
  81. package/docs/operations/metrics.md +18 -0
  82. package/docs/policy/presets.md +1 -1
  83. package/docs/start/choosing-a-policy.md +1 -1
  84. package/docs/start/first-integration.md +1 -1
  85. package/docs/start/installation.md +2 -2
  86. package/docs/testing/cli.md +7 -1
  87. package/docs/testing/corpus.md +12 -8
  88. package/docs/testing/index.md +1 -1
  89. package/docs/testing/try-it.md +1 -1
  90. package/package.json +4 -1
package/CHANGELOG.md CHANGED
@@ -8,6 +8,127 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
8
8
 
9
9
  ### Added
10
10
 
11
+ - **Shadow mode: run a detector and let it decide nothing.** `shadowDetectors: ["path-novelty"]`
12
+ runs the named detectors on every request exactly as they otherwise would and keeps their
13
+ findings out of the verdict, the score, the class, the identity and every rule. They land
14
+ in `assessment.shadowEvidence`, counted and charted next to the evidence that did decide.
15
+ This is deliberately not a weight of zero: a weight is consulted only on the probabilistic
16
+ path, and `certain` evidence never reaches that path, so a shadowed detector emitting it
17
+ would have blocked people with its weight sitting at zero the whole time.
18
+
19
+ `assessment.shadowVerdict` carries what the verdict *would* have been, computed only when
20
+ a shadowed detector actually found something. "It fired 312 times" is not a number anybody
21
+ can act on; "it would have moved 41 requests to `suspected-bot`" is. Prometheus gains
22
+ `bothandler_shadow_firings_total` and `bothandler_shadow_verdict_changes_total`, both
23
+ absent entirely while nothing is shadowed.
24
+
25
+ The correlation sources are why this exists: several of their detectors fire at `moderate`
26
+ on real people by design — a phone roaming between networks, a crowd on a broken link,
27
+ somebody tapping "Request desktop site" — and whether the thresholds are right *for your
28
+ site* is not a thing this library can know. See `docs/detection/shadow-mode.md`.
29
+
30
+ - **`target-integrity`, and the raw request target it reads.** `facts.path` is normalised so
31
+ that a rule scoped to `/admin` holds against `/%61dmin` and `/./admin`. That normalisation
32
+ is also what makes an evasive target look ordinary: `/%2e%2e%2f%2e%2e%2fapp/config.yml`
33
+ arrives as `/app/config.yml`, which is on no wordlist and reads like a broken link.
34
+ `facts.rawPath` now keeps the target as it was spelled — and only when it differs from the
35
+ normalised form, so ordinary traffic pays nothing for it.
36
+
37
+ The detector reports encoded traversals, double encoding, encoded control characters,
38
+ absolute-form targets addressed to a proxy, and separators hidden inside a segment. None
39
+ of it is `certain` and the closest call says why: a path segment carrying a URL as *data*
40
+ is encoded once to sit in a path and again by whatever built the link around it, which
41
+ produces `%252e` honestly on a site that has done nothing wrong.
42
+
43
+ - **The feed filter takes `$and`, `$or`, `$not`, `$in` and `$notin`.** Adjacent terms
44
+ still mean `AND` and `-term` still negates, so every existing query and saved filter
45
+ reads the same — but the parser produces a tree now rather than a flat list, which is
46
+ what `$or` needs and what the previous version communicated by silently ignoring the
47
+ word. `$not` binds tightest, then `$and`, then `$or`, and brackets group. Operators
48
+ carry a `$` because a bare `or` appears in User-Agents and paths, and a language where
49
+ an ordinary search word becomes an operator lies about what it matched. Nothing throws:
50
+ an unclosed bracket, a dangling `$or` and a half-typed `$in(` are all the normal state
51
+ of a live search box.
52
+
53
+ - **The Actors screen lists either the registry or the actors in the feed.** A toggle
54
+ above the table. The registry answers "who is hitting me hardest"; once a filter is on,
55
+ the question is usually the other one — "who is in *this*" — and the screen could not
56
+ answer it. The feed-derived list leaves `Per min`, `Cadence` and `Unsolved` blank rather
57
+ than computing them from a few hundred requests, because those are properties of a
58
+ client's whole history and a confident number under the wrong heading is worse than a
59
+ dash.
60
+
61
+ ### Changed
62
+
63
+ - **The browser suite runs on Chromium, Firefox and WebKit.** `BROWSER_ENGINE` picks one
64
+ locally (`npm run test:browser:firefox`, `:webkit`); CI runs all three as a matrix.
65
+ Testing a single engine is how the Safari header bug shipped, and Firefox earned its
66
+ place on the first run by catching the challenge page's CSP error.
67
+
68
+ - **The feed's column headers stick in Safari.** The tables collapsed their borders, which
69
+ is a long-standing sore point for sticky table cells in WebKit — the CSSWG has an open
70
+ issue on collapsed borders not following a cell when it sticks, and Safari is widely
71
+ reported to drop the stickiness of a `th` outright. The header held in Chromium and
72
+ Firefox and was reported adrift in Safari, which is what a sticky element that has
73
+ stopped sticking looks like. They separate their
74
+ borders with zero spacing now, which no border in these tables relied on: the only
75
+ measurable difference is the accent column starting two pixels earlier, because
76
+ collapsing left half of that 3px border outside the cell. A browser test scrolls the
77
+ page and asserts the header holds, since one keyword can undo this silently.
78
+
79
+ - **The feed's Exclude button is gone; `$not` replaces it.** It kept a hidden list in one
80
+ person's browser, which meant a view with the noise taken out could not be shared. A
81
+ `$not` lives in the URL like every other narrowing.
82
+
83
+ - **Labelling an actor edits in place instead of calling `prompt()`.** A sandboxed iframe
84
+ blocks `prompt()` outright, so on an embedded dashboard the Label button did nothing at
85
+ all, with no error and no way to tell.
86
+
87
+ - **Correlating a client's own requests, through a marker cookie.** `probe` is a new,
88
+ opt-in source: the engine issues a signed first-party cookie and reads it back, so two
89
+ requests can be attributed to one *client* rather than to one address. That closes a gap
90
+ the library had documented and declined to guess at — `identity-rotation` reads "one
91
+ actor, several User-Agents" as lying, and under an address-derived actor key that
92
+ describes every office and carrier on the internet, so it has always shipped switched
93
+ off. A marker carries an HMAC only this server can produce, which makes the same
94
+ observation evidence instead of speculation.
95
+
96
+ Four detectors arrive with it. `identity-drift` compares the identity claimed now with
97
+ the one claimed when the marker was issued, weighing a changed *browser family* at
98
+ `strong` and a changed platform at `moderate` — because "Request desktop site" on a
99
+ phone does the latter and the person doing it is a person. `marker-integrity` reports a
100
+ marker signed with a key we do not have. `marker-fanout` counts the networks one marker
101
+ has been presented from. `marker-persistence` reports a client that sends cookies but
102
+ never ours, deliberately narrower than `session-integrity`, which already covers a
103
+ client that sends none.
104
+
105
+ It is off by default and sets a cookie only when the client holds no valid one, so an
106
+ ordinary visitor is issued one on the first request of a session and no other. `secrets`
107
+ is required and has no default: a secret generated at startup would read every marker
108
+ minted by another replica as forged.
109
+
110
+ - **Reading what a client does when it is challenged.** `challenge-reaction` reports an
111
+ identity that changes within seconds of a challenge, and a client challenged repeatedly
112
+ that has never answered. It is the only detector here whose stimulus this library chose,
113
+ which is what lets it reach `strong` — and it reports the same observation a tier lower
114
+ when only an address ties the two requests together, because that is how much less an
115
+ address-based join is worth. `challenge-integrity` reports solutions that were replayed,
116
+ or returned faster than the proof of work can be computed in a browser.
117
+
118
+ - **Comparing a client with the rest of your traffic.** `site` is a second opt-in source
119
+ holding a bounded, warmed-up baseline of what a site normally serves. `distributed-walk`
120
+ finds a numeric range walked across many clients where none walks enough of it alone —
121
+ the one threat per-actor thresholds miss by construction. `path-novelty` is a
122
+ self-maintaining wordlist. `path-campaign` reports a path the site never served that
123
+ many unrelated clients suddenly want, requiring the miss rate so that a successful
124
+ launch is not reported as an attack. `miss-baseline` is `probe-volume` measured against
125
+ the site's own rate rather than a fixed threshold.
126
+
127
+ Nothing is reported until `warmupRequests` have been observed, because every path is
128
+ rare when nothing has been seen. Walk ids are held as a coarsening bitmap and marker
129
+ fan-out as a 128-bit sketch, so neither table grows with what a client chooses to
130
+ request.
131
+
11
132
  - **`indexers-only`, a preset for sites that want search traffic and nothing else.**
12
133
  Serves a bot only when its identity has been confirmed by DNS or a published range
13
134
  *and* it is a `search` or `social` crawler; refuses every other proven bot; challenges
@@ -106,6 +227,111 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
106
227
 
107
228
  ### Fixed
108
229
 
230
+ - **An embedded dashboard could point at the host application instead of at `src`.**
231
+ `disconnectedCallback` imports the stream module to close the stream, that module
232
+ imports the boot module, and the boot module read its mount path off the global once,
233
+ when it was first evaluated. A host page that mounted, unmounted and remounted the
234
+ element before the first bootstrap fetch returned — React 18's development double-mount
235
+ — ran that import while the global was still undefined, so the base froze at `""` for
236
+ the life of the page.
237
+
238
+ Nothing threw. Every request the dashboard made afterwards went to the host page's own
239
+ origin root rather than to `src`, so it drew no traffic while sending `/api/stream` and
240
+ `/api/stats` to somebody else's router, and the only outward sign was an `EventSource`
241
+ complaining about a MIME type on whichever engine reports that. The element now hands
242
+ the payload over explicitly, so being evaluated early costs nothing.
243
+
244
+ - **Pressing "Load them" left the "not streamed" badge on screen.** The count of how much
245
+ of the gap had been closed was taken from the page's own snapshot, which is refreshed on
246
+ a timer and so was usually several seconds out of date; a snapshot arriving afterwards
247
+ with a larger count reopened a gap that had just been closed. `/api/feed` now returns
248
+ `skipped` alongside the backlog, so the number comes from the response that closed the
249
+ gap rather than from one taken before it.
250
+
251
+ - **`RedisStore.increment` could leave a counter key that never expired.** `INCR` then
252
+ `PEXPIRE` is two commands, and a process killed between them orphaned a key nothing would
253
+ ever revisit — the next request falls into the next bucket, under another key. The expiry
254
+ is now armed by the command that *creates* the key: `SET … PX … NX` issued without waiting
255
+ for its reply, then `INCR`, so both are on the wire together and this stays one round trip.
256
+ Not a Lua script, because `eval` is the one command `ioredis` and `node-redis` spell
257
+ differently enough that `RedisLike` could not describe both. `pexpire` has left that
258
+ interface, which needs four commands now rather than five.
259
+
260
+ - **The interaction challenge marked down somebody who paused mid-movement, twice over.**
261
+ Samples arriving more than a quarter of a second apart are dropped before anything is
262
+ measured, because the distance across a pause is not a distance a hand travelled — but
263
+ `fractionalShare` was dividing the samples it kept by the count of everything that
264
+ arrived, reporting *fewer* sub-pixel coordinates than the samples it was computed from
265
+ actually had. That reads as "these coordinates are integers", which is the signature of an
266
+ interpolated path. Separately, whether a path was measurable at all was decided from the
267
+ raw array length while the scoring judged the kept samples: four pointer samples with a
268
+ pause before each one were four to the array and none to the analysis, so the report was
269
+ called measurable and then scored at zero. Both cost the reading pattern most likely to
270
+ produce them — move the pointer, stop to read, move again.
271
+
272
+ - **Save and Cancel on the actor label editor, which three earlier attempts could not fit.**
273
+ `input[type="text"] { width: 100% }` outranks a bare class selector, so the editor's
274
+ `width: 15ch` had never applied: the box filled its shrink-to-fit container and pushed the
275
+ buttons past the right edge of the panel, where they could be seen and not clicked.
276
+ Qualifying the selector fixed the cause; the row's other actions now stand aside while the
277
+ editor is open, which is also the right thing on its own — Allowlist and Forget are not
278
+ what somebody naming a client is reaching for.
279
+
280
+ - **The Actors scope was not in the URL.** `#actors?a=feed`, pushed rather than replaced,
281
+ because switching between the registry and the feed is a discrete act like clicking a tab
282
+ and the back button should undo it. The feed-scoped list also shows a dash for
283
+ `Confirmations` like the three columns beside it, rather than a confident zero under a
284
+ heading that means "how many times has this client been proven a bot".
285
+
286
+ - **`bothandlerjs detectors --preset <typo>` answered anyway.** An unknown preset fell
287
+ through to a handler with no preset and printed the default list at exit code 0 — a
288
+ confident wrong answer to the one question the command exists for, while `robots` in the
289
+ same file refused the same typo. It is refused now, with the same message. The command
290
+ also notes that a preset selects rules rather than detectors, and that `challenge`,
291
+ `probe` and `site` are what change the list; the note goes to stderr, so redirecting the
292
+ list stays clean.
293
+
294
+ - **The default notification sink discarded the content of every error.** `consoleNotifier`
295
+ never referenced `event.error`. An error carrying no assessment fell into the branch
296
+ written for anomalies and printed `[bothandler] error unknown — ` at *warning* level, with
297
+ the failing source and the message both dropped; one that did carry an assessment printed
298
+ the request's evidence instead. Errors are how a failed detector, sink or store is
299
+ reported, and this is the sink an operator gets without configuring one — so the default
300
+ way to find out that a detector had been throwing showed neither which one nor why.
301
+
302
+ - **Every person shown a challenge page in Firefox got a security error in their
303
+ console.** The page declares no icon, so the browser asks for `/favicon.ico` by itself;
304
+ under `default-src 'none'` that request is refused, and Firefox reports the refusal as a
305
+ CSP violation on a page whose entire purpose is to reassure somebody that nothing is
306
+ wrong. The page now declares an empty icon so the request is never made, and the policy
307
+ allows `img-src data:` — which permits nothing off the machine, a `data:` URI being
308
+ inline by definition — so that declaration is honoured. Found by running the browser
309
+ suite on Firefox for the first time.
310
+
311
+ - **Cookies split across several header fields were misparsed, losing every cookie after
312
+ the first.** HTTP/2 permits a client to send its cookies as separate header fields and
313
+ Node's `http2` surfaces them as an array; RFC 9113 §8.2.3 says a receiver concatenates
314
+ them with `"; "`. They were joined with `", "` like every other header, which parses as
315
+ one cookie whose value is the rest of the line — so a clearance token in the second
316
+ field was invisible, and an HTTP/2 visitor who had solved a challenge was challenged
317
+ again on every request.
318
+
319
+ - **`X-Forwarded-For` entries carrying a port were dropped.** Azure's Application Gateway
320
+ and Front Door write `1.2.3.4:5678`, and RFC 7239 spells IPv6 as `[2001:db8::1]:5678`.
321
+ Neither parsed, and because every entry in such a chain carries a port the whole chain
322
+ emptied and every client behind that proxy resolved to the proxy's own address — sharing
323
+ one actor, one history and one rate-limit bucket, so a single bot could lock out every
324
+ real visitor.
325
+
326
+ - **Evidence summaries quoted client-controlled text without neutralising it.** A request
327
+ path containing CRLF came back inside an `id-enumeration` summary exactly as sent, which
328
+ forges log lines; escape sequences repainted terminals. Summaries and operator labels are
329
+ now cleaned centrally, so detectors written elsewhere are covered too.
330
+
331
+ - **`probe-volume` was inert on three of the four adapters.** Only the Node adapter
332
+ reported the status the application answered, so on Fastify, Koa and every Fetch runtime
333
+ the detector was installed, listed, and structurally unable to fire.
334
+
109
335
  - **The "N not streamed" badge outlived the feed it described.** The badge adds two
110
336
  counts: the server's rate-cap `skipped`, which `FeedRing.clear()` resets, and the
111
337
  connection's lagged drops, which nothing did. After a Reset — or after the replace-sync
@@ -281,7 +507,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
281
507
  looks like a challenge that worked, which is why it needed saying. `AuditWindow` gains
282
508
  `challengesSolved` and `challengeSolveRate`.
283
509
 
284
- - **`bothandlerjs check` — your policy against 526 shapes of real traffic.** The question
510
+ - **`bothandlerjs check` — your policy against 545 shapes of real traffic.** The question
285
511
  the library is organised around, asked offline and before a deploy: *if I point this
286
512
  configuration at the actual internet, who gets hurt?* It exits non-zero if any case
287
513
  marked `human` is denied service, which is what makes it a CI step rather than a
package/README.md CHANGED
@@ -23,7 +23,7 @@ No runtime dependencies. The library imports nothing but `node:` builtins, and C
23
23
  fails if that ever stops being true — so nothing here can hand your project a
24
24
  transitive advisory, an install script, or a version conflict with something you
25
25
  already run. Redis, if you use it, is your client passed in: `RedisStore` describes
26
- the five commands it needs structurally and imports neither `ioredis` nor
26
+ the four commands it needs structurally and imports neither `ioredis` nor
27
27
  `node-redis`.
28
28
 
29
29
  ---
@@ -31,15 +31,15 @@ the five commands it needs structurally and imports neither `ioredis` nor
31
31
  ## Documentation
32
32
 
33
33
  The README is the argument and the shortest path to a working integration. Everything
34
- else lives in **[`docs/`](docs/index.md)** — thirty pages, one per question, each
35
- explaining why a thing exists as well as how to use it.
34
+ else lives in **[`docs/`](docs/index.md)** — a page per question, each explaining why a
35
+ thing exists as well as how to use it.
36
36
 
37
37
  | | |
38
38
  | --- | --- |
39
39
  | **[The course](docs/course/index.md)** | Sixteen lessons that build one integration, from a first assessment to a policy you can defend. Start here if the library is new to you. |
40
40
  | **[Start here](docs/index.md)** | [Installation](docs/start/installation.md) · [Your first integration](docs/start/first-integration.md) · [Choosing a policy](docs/start/choosing-a-policy.md) · [Upgrading](docs/start/upgrading.md) |
41
41
  | **Concepts** | [Evidence and certainty](docs/concepts/evidence.md) · [Verdicts and scores](docs/concepts/verdicts.md) · [The safety guard](docs/concepts/the-guard.md) · [Actors](docs/concepts/actors.md) · [Threat model](docs/concepts/threat-model.md) |
42
- | **[Detection](docs/detection/index.md)** | [The 20 detectors](docs/detection/detectors.md) · [Signatures](docs/detection/signatures.md) · [Verification](docs/detection/verification.md) · [Browser signals](docs/detection/client-signals.md) · [Writing a detector](docs/detection/writing-a-detector.md) |
42
+ | **[Detection](docs/detection/index.md)** | [The detectors](docs/detection/detectors.md) · [Correlation](docs/detection/correlation.md) · [Signatures](docs/detection/signatures.md) · [Verification](docs/detection/verification.md) · [Browser signals](docs/detection/client-signals.md) · [Writing a detector](docs/detection/writing-a-detector.md) |
43
43
  | **[Policy](docs/policy/index.md)** | [Rules](docs/policy/rules.md) · [Actions](docs/policy/actions.md) · [Presets](docs/policy/presets.md) · [robots.txt](docs/policy/robots.md) · [The challenge](docs/challenge/index.md) |
44
44
  | **[Operations](docs/operations/index.md)** | [The dashboard](docs/operations/dashboard.md) · [Embedding it](docs/operations/embedding.md) · [Metrics](docs/operations/metrics.md) · [The audit](docs/operations/audit.md) · [Notifications](docs/operations/notifications.md) · [Runtime changes](docs/operations/runtime-changes.md) |
45
45
  | **[Integration](docs/integration/index.md)** | [Adapters](docs/integration/adapters.md) · [The client IP](docs/integration/client-ip.md) · [Stores](docs/integration/stores.md) |
@@ -233,12 +233,20 @@ all three.
233
233
  stale mapping is a false positive with a long half-life. Bring your own, from a
234
234
  source you refresh and can audit.
235
235
  - **Be right about a shared address.** Behind CGNAT, "one actor" is thousands of
236
- people. That is why the behavioural signals are capped where they are.
236
+ people. That is why the behavioural signals are capped where they are. The optional
237
+ [marker cookie](docs/detection/correlation.md) narrows this — two requests carrying one
238
+ signed marker are one client rather than one address — but only for clients that keep
239
+ cookies, and the address remains what everything else is keyed on.
237
240
  - **Escalate on a wordlist walk.** `probe-signature` reads one request at a time, so a
238
241
  scanner working through five hundred paths produces five hundred separate
239
242
  observations rather than a mounting case. That is the price of a detector that runs
240
243
  unchanged over a log file; enumeration over time is what `rate-anomaly`, `cadence`
241
244
  and `crawl-breadth` are for.
245
+ - **See an enumeration split across enough clients — unless you ask it to.** Divide a
246
+ range between five hundred addresses and every per-actor threshold is defeated by
247
+ construction, because each actor is genuinely unremarkable. `distributed-walk` finds it
248
+ in the union of what those addresses asked for, and needs the optional
249
+ [site baseline](docs/detection/correlation.md) and enough traffic to have one.
242
250
 
243
251
  ---
244
252
 
@@ -254,13 +262,13 @@ npm run build # ESM + CJS + declarations
254
262
  npm run demo # protected site :9673 + live dashboard :9674
255
263
  npm run demo:roles # the same dashboard behind roles: analyst :9684, operator :9685, admin :9686
256
264
  npm run simulate # eighteen curated scenarios against the demo
257
- npm run simulate:corpus # replay all 526 corpus cases over a real socket
265
+ npm run simulate:corpus # replay all 548 corpus cases over a real socket
258
266
  npm run bench # hot-path benchmark, median of several rounds
259
- npm run corpus # 526 shapes of real traffic against your policy
267
+ npm run corpus # 548 shapes of real traffic against your policy
260
268
  npm run example # a minimal Express integration on :3000
261
269
 
262
270
  npx @osqd/bothandlerjs replay access.log # what your policy would have done
263
- npx @osqd/bothandlerjs check # your policy against 526 shapes of real traffic
271
+ npx @osqd/bothandlerjs check # your policy against 548 shapes of real traffic
264
272
  npx @osqd/bothandlerjs explain "curl/8.4.0" # one request, and the evidence behind the verdict
265
273
  ```
266
274
 
@@ -276,7 +284,7 @@ src/
276
284
  config.ts validation, defaults, client-IP resolution
277
285
  facts.ts request normalisation
278
286
  state.ts bounded per-actor behavioural memory
279
- detectors/ twenty detectors + the signature database
287
+ detectors/ the detectors + the signature database
280
288
  policy/ rules, matcher, the safety guard, presets
281
289
  actions/ decision -> framework-neutral outcome
282
290
  challenge/ proof of work, signed tokens, the interstitial
@@ -287,7 +295,7 @@ src/
287
295
  adapters/ Express/Connect, Fastify, Koa, Fetch
288
296
  client/ browser-side signal script
289
297
  internal/ IP, crypto, UA, Aho-Corasick, LRU, DNS, HTTP
290
- corpus/ 526 shapes of real traffic, with provenance, and the harness
298
+ corpus/ 548 shapes of real traffic, with provenance, and the harness
291
299
  that runs them against your configuration
292
300
  demo/ the protected site and the live dashboard
293
301
  scripts/simulate.ts the traffic simulator
@@ -15,6 +15,16 @@ export interface FastifyLikeReply {
15
15
  header(name: string, value: string): FastifyLikeReply;
16
16
  send(body: unknown): unknown;
17
17
  hijack?(): void;
18
+ /**
19
+ * The underlying response. Optional because this type describes the shape the adapter
20
+ * needs rather than Fastify's own, and a test double should not have to build one —
21
+ * but Fastify always provides it, and without it the status this request ends up
22
+ * answering is never reported back.
23
+ */
24
+ raw?: {
25
+ once(event: "finish", listener: () => void): unknown;
26
+ statusCode: number;
27
+ };
18
28
  }
19
29
  export interface FastifyAdapterOptions {
20
30
  tlsFingerprintHeader?: string;
@@ -294,6 +294,7 @@ var SPECIAL_USE = new IpRangeSet(SPECIAL_USE_RANGES);
294
294
 
295
295
  // src/facts.ts
296
296
  var MAX_URL_LENGTH = 8192;
297
+ var MAX_RAW_PATH = 512;
297
298
  var MAX_QUERY_PARAMS = 64;
298
299
  function createFacts(input) {
299
300
  const rawUrl = input.url ?? "/";
@@ -302,18 +303,21 @@ function createFacts(input) {
302
303
  const rawPath = queryStart === -1 ? url : url.slice(0, queryStart);
303
304
  const headers = /* @__PURE__ */ Object.create(null);
304
305
  for (const [name, value] of Object.entries(input.headers)) {
305
- const joined = joinHeaderValue(value);
306
- if (joined !== void 0) headers[name.toLowerCase()] = joined;
306
+ const lower = name.toLowerCase();
307
+ const joined = lower === "cookie" && Array.isArray(value) ? value.join("; ") : joinHeaderValue(value);
308
+ if (joined !== void 0) headers[lower] = joined;
307
309
  }
310
+ const normalized = normalizePath(rawPath);
308
311
  const facts = {
309
312
  method: (input.method ?? "GET").toUpperCase(),
310
- path: normalizePath(rawPath),
313
+ path: normalized,
311
314
  query: parseQuery(queryStart === -1 ? "" : url.slice(queryStart + 1)),
312
315
  headers,
313
316
  headerOrder: extractOrder(input.rawHeaders, headers),
314
317
  ip: normalizeIp(input.ip) ?? input.ip,
315
318
  timestamp: input.timestamp ?? Date.now()
316
319
  };
320
+ if (rawPath !== normalized) facts.rawPath = rawPath.length > MAX_RAW_PATH ? rawPath.slice(0, MAX_RAW_PATH) : rawPath;
317
321
  const cookieHeader = headers["cookie"];
318
322
  if (cookieHeader !== void 0) facts.cookies = parseCookies(cookieHeader);
319
323
  if (input.protocol !== void 0) facts.protocol = input.protocol;
@@ -346,12 +350,23 @@ function parseQuery(search) {
346
350
  const query = /* @__PURE__ */ Object.create(null);
347
351
  if (search.length === 0) return query;
348
352
  let count = 0;
349
- for (const [key, value] of new URLSearchParams(search)) {
353
+ for (const [key, value] of new URLSearchParams(boundedSearch(search))) {
350
354
  if (count++ >= MAX_QUERY_PARAMS) break;
351
355
  query[key] = value.length > 1024 ? value.slice(0, 1024) : value;
352
356
  }
353
357
  return query;
354
358
  }
359
+ function boundedSearch(search) {
360
+ let seen = 0;
361
+ let at = search.charCodeAt(0) === 63 ? 1 : 0;
362
+ while (at < search.length) {
363
+ let end = search.indexOf("&", at);
364
+ if (end === -1) end = search.length;
365
+ if (end !== at && ++seen > MAX_QUERY_PARAMS) return search.slice(0, at - 1);
366
+ at = end + 1;
367
+ }
368
+ return search;
369
+ }
355
370
  var MAX_ORDERED_HEADERS = 64;
356
371
  function extractOrder(rawHeaders, headers) {
357
372
  if (!rawHeaders || rawHeaders.length === 0) return EMPTY_ORDER;
@@ -474,6 +489,11 @@ function botHandler(handler, options = {}) {
474
489
  for (const [name, value] of Object.entries(outcome.responseHeaders)) response.setHeader(name, value);
475
490
  }
476
491
  if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
492
+ if (typeof response.once === "function") {
493
+ response.once("finish", () => {
494
+ handler.recordOutcome(facts, response.statusCode);
495
+ });
496
+ }
477
497
  handedOff = true;
478
498
  next();
479
499
  } catch (error) {
@@ -570,6 +590,7 @@ function withBotHandler(handler, next, options = {}) {
570
590
  const decision = await evaluate3(request, context);
571
591
  if (decision.response) return decision.response;
572
592
  const response = await next(decision.request, context);
593
+ if (decision.result !== void 0) handler.recordOutcome(decision.result.assessment.facts, response.status);
573
594
  const extra = decision.result?.outcome.kind === "continue" ? decision.result.outcome.responseHeaders : void 0;
574
595
  if (!extra) return response;
575
596
  const merged = new Response(response.body, response);
@@ -653,6 +674,8 @@ async function evaluate(handler, options, mountChallenge, request, reply) {
653
674
  for (const [name, value] of Object.entries(outcome.responseHeaders)) reply.header(name, value);
654
675
  }
655
676
  if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
677
+ const raw = reply.raw;
678
+ if (raw !== void 0) raw.once("finish", () => handler.recordOutcome(facts, raw.statusCode));
656
679
  }
657
680
 
658
681
  // src/adapters/koa.ts
@@ -660,13 +683,18 @@ function koaBotHandler(handler, options = {}) {
660
683
  const mountChallenge = options.mountChallengeEndpoint ?? true;
661
684
  return async function botHandlerMiddleware(context, next) {
662
685
  let proceed;
686
+ let facts;
663
687
  try {
664
- proceed = await evaluate2(handler, mountChallenge, context);
688
+ const decision = await evaluate2(handler, mountChallenge, context);
689
+ proceed = decision.proceed;
690
+ facts = decision.facts;
665
691
  } catch (error) {
666
692
  handler.config.onError(error, { source: "adapter:koa" });
667
693
  proceed = true;
668
694
  }
669
- if (proceed) await next();
695
+ if (!proceed) return;
696
+ await next();
697
+ if (facts !== void 0) handler.recordOutcome(facts, context.status);
670
698
  };
671
699
  }
672
700
  async function evaluate2(handler, mountChallenge, context) {
@@ -690,18 +718,18 @@ async function evaluate2(handler, mountChallenge, context) {
690
718
  context.set("cache-control", "no-store");
691
719
  if (verification.ok) context.set("set-cookie", verification.setCookie);
692
720
  context.body = verificationBody(verification);
693
- return false;
721
+ return { proceed: false };
694
722
  }
695
723
  const { outcome } = await handler.handle(facts);
696
724
  if (outcome.kind === "drop") {
697
725
  context.req.socket.destroy();
698
- return false;
726
+ return { proceed: false };
699
727
  }
700
728
  if (outcome.kind === "respond") {
701
729
  context.status = outcome.status;
702
730
  for (const [name, value] of Object.entries(outcome.headers)) context.set(name, value);
703
731
  context.body = outcome.body;
704
- return false;
732
+ return { proceed: false };
705
733
  }
706
734
  for (const [name, value] of Object.entries(outcome.requestHeaders ?? {})) {
707
735
  context.req.headers[name] = value;
@@ -710,7 +738,7 @@ async function evaluate2(handler, mountChallenge, context) {
710
738
  for (const [name, value] of Object.entries(outcome.responseHeaders)) context.set(name, value);
711
739
  }
712
740
  if (outcome.delayMs !== void 0) await pause(outcome.delayMs);
713
- return true;
741
+ return { proceed: true, facts };
714
742
  }
715
743
  // Annotate the CommonJS export names for ESM import in node:
716
744
  0 && (module.exports = {