@osqd/bothandlerjs 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +516 -0
  2. package/LICENSE +102 -0
  3. package/README.md +308 -0
  4. package/bin/bothandlerjs.mjs +14 -0
  5. package/dist/actions/index.d.ts +22 -0
  6. package/dist/actions/types.d.ts +59 -0
  7. package/dist/adapters/fastify.d.ts +43 -0
  8. package/dist/adapters/fetch.d.ts +51 -0
  9. package/dist/adapters/index.cjs +723 -0
  10. package/dist/adapters/index.cjs.map +1 -0
  11. package/dist/adapters/index.d.ts +8 -0
  12. package/dist/adapters/index.js +692 -0
  13. package/dist/adapters/index.js.map +1 -0
  14. package/dist/adapters/koa.d.ts +27 -0
  15. package/dist/adapters/node.d.ts +29 -0
  16. package/dist/adapters/shared.d.ts +39 -0
  17. package/dist/audit.d.ts +172 -0
  18. package/dist/challenge/index.d.ts +176 -0
  19. package/dist/challenge/language.d.ts +57 -0
  20. package/dist/challenge/page.d.ts +42 -0
  21. package/dist/challenge/pow.d.ts +33 -0
  22. package/dist/challenge/token.d.ts +72 -0
  23. package/dist/cli.cjs +13388 -0
  24. package/dist/cli.cjs.map +1 -0
  25. package/dist/cli.d.ts +5 -0
  26. package/dist/cli.js +13358 -0
  27. package/dist/cli.js.map +1 -0
  28. package/dist/client/index.cjs +139 -0
  29. package/dist/client/index.cjs.map +1 -0
  30. package/dist/client/index.d.ts +63 -0
  31. package/dist/client/index.js +112 -0
  32. package/dist/client/index.js.map +1 -0
  33. package/dist/config.d.ts +316 -0
  34. package/dist/core.d.ts +443 -0
  35. package/dist/corpus/adversarial.d.ts +2 -0
  36. package/dist/corpus/advertising-email.d.ts +2 -0
  37. package/dist/corpus/ai-crawlers.d.ts +2 -0
  38. package/dist/corpus/benign-bots.d.ts +2 -0
  39. package/dist/corpus/cdn-gateways.d.ts +2 -0
  40. package/dist/corpus/cookies.d.ts +49 -0
  41. package/dist/corpus/crawlers-regional.d.ts +2 -0
  42. package/dist/corpus/crawlers-vertical.d.ts +2 -0
  43. package/dist/corpus/headers.d.ts +403 -0
  44. package/dist/corpus/humans-apps.d.ts +2 -0
  45. package/dist/corpus/humans-browsers.d.ts +2 -0
  46. package/dist/corpus/humans.d.ts +2 -0
  47. package/dist/corpus/index.cjs +4196 -0
  48. package/dist/corpus/index.cjs.map +1 -0
  49. package/dist/corpus/index.d.ts +37 -0
  50. package/dist/corpus/index.js +4132 -0
  51. package/dist/corpus/index.js.map +1 -0
  52. package/dist/corpus/infrastructure.d.ts +17 -0
  53. package/dist/corpus/libraries-extended.d.ts +2 -0
  54. package/dist/corpus/ranges.d.ts +31 -0
  55. package/dist/corpus/reputation.d.ts +21 -0
  56. package/dist/corpus/runner.d.ts +130 -0
  57. package/dist/corpus/schema.d.ts +185 -0
  58. package/dist/corpus/tooling.d.ts +2 -0
  59. package/dist/corpus/unwanted.d.ts +2 -0
  60. package/dist/crawler-ranges.d.ts +113 -0
  61. package/dist/dashboard/client.generated.d.ts +7 -0
  62. package/dist/dashboard/feed.d.ts +186 -0
  63. package/dist/dashboard/index.d.ts +5 -0
  64. package/dist/dashboard/page.d.ts +51 -0
  65. package/dist/dashboard/parse-request.d.ts +28 -0
  66. package/dist/dashboard/preview.d.ts +45 -0
  67. package/dist/dashboard/server.d.ts +70 -0
  68. package/dist/dashboard/types.d.ts +714 -0
  69. package/dist/detectors/accept-signature.d.ts +6 -0
  70. package/dist/detectors/browsing-coherence.d.ts +44 -0
  71. package/dist/detectors/cadence.d.ts +28 -0
  72. package/dist/detectors/clearance.d.ts +25 -0
  73. package/dist/detectors/client-hints.d.ts +19 -0
  74. package/dist/detectors/client-signals.d.ts +27 -0
  75. package/dist/detectors/crawl-breadth.d.ts +28 -0
  76. package/dist/detectors/crawler-verification.d.ts +39 -0
  77. package/dist/detectors/fetch-metadata.d.ts +20 -0
  78. package/dist/detectors/header-integrity.d.ts +26 -0
  79. package/dist/detectors/header-order.d.ts +27 -0
  80. package/dist/detectors/identity-rotation.d.ts +34 -0
  81. package/dist/detectors/index.d.ts +56 -0
  82. package/dist/detectors/ip-intelligence.d.ts +30 -0
  83. package/dist/detectors/known-bots.d.ts +94 -0
  84. package/dist/detectors/probe-signature.d.ts +50 -0
  85. package/dist/detectors/rate-anomaly.d.ts +36 -0
  86. package/dist/detectors/self-identified.d.ts +33 -0
  87. package/dist/detectors/session-integrity.d.ts +24 -0
  88. package/dist/detectors/tls-fingerprint.d.ts +34 -0
  89. package/dist/detectors/trap.d.ts +98 -0
  90. package/dist/detectors/types.d.ts +70 -0
  91. package/dist/detectors/ua-coherence.d.ts +32 -0
  92. package/dist/evidence.d.ts +45 -0
  93. package/dist/facts.d.ts +46 -0
  94. package/dist/index.cjs +9600 -0
  95. package/dist/index.cjs.map +1 -0
  96. package/dist/index.d.ts +63 -0
  97. package/dist/index.js +9444 -0
  98. package/dist/index.js.map +1 -0
  99. package/dist/internal/async.d.ts +16 -0
  100. package/dist/internal/clock.d.ts +17 -0
  101. package/dist/internal/crypto.d.ts +26 -0
  102. package/dist/internal/dns.d.ts +100 -0
  103. package/dist/internal/emitter.d.ts +18 -0
  104. package/dist/internal/http.d.ts +22 -0
  105. package/dist/internal/ip.d.ts +73 -0
  106. package/dist/internal/lru.d.ts +34 -0
  107. package/dist/internal/matcher.d.ts +41 -0
  108. package/dist/internal/pattern.d.ts +14 -0
  109. package/dist/internal/ua.d.ts +92 -0
  110. package/dist/metrics.d.ts +120 -0
  111. package/dist/notify/hub.d.ts +70 -0
  112. package/dist/notify/index.d.ts +7 -0
  113. package/dist/notify/redact.d.ts +48 -0
  114. package/dist/notify/sinks.d.ts +62 -0
  115. package/dist/notify/types.d.ts +56 -0
  116. package/dist/policy/index.d.ts +7 -0
  117. package/dist/policy/match.d.ts +20 -0
  118. package/dist/policy/policy.d.ts +123 -0
  119. package/dist/policy/presets.d.ts +161 -0
  120. package/dist/policy/types.d.ts +155 -0
  121. package/dist/robots.d.ts +83 -0
  122. package/dist/state.d.ts +161 -0
  123. package/dist/stores/index.d.ts +5 -0
  124. package/dist/stores/memory.d.ts +33 -0
  125. package/dist/stores/redis.d.ts +59 -0
  126. package/dist/stores/types.d.ts +37 -0
  127. package/dist/types.d.ts +250 -0
  128. package/docs/challenge/index.md +152 -0
  129. package/docs/challenge/localisation.md +105 -0
  130. package/docs/concepts/actors.md +113 -0
  131. package/docs/concepts/evidence.md +133 -0
  132. package/docs/concepts/the-guard.md +121 -0
  133. package/docs/concepts/threat-model.md +120 -0
  134. package/docs/concepts/verdicts.md +108 -0
  135. package/docs/course/01-first-assessment.md +178 -0
  136. package/docs/course/02-proof-and-suspicion.md +164 -0
  137. package/docs/course/03-verdicts-and-scores.md +160 -0
  138. package/docs/course/04-the-guard.md +200 -0
  139. package/docs/course/05-detectors.md +178 -0
  140. package/docs/course/06-identity.md +161 -0
  141. package/docs/course/07-actors.md +178 -0
  142. package/docs/course/08-traps.md +183 -0
  143. package/docs/course/09-rules.md +224 -0
  144. package/docs/course/10-actions-and-presets.md +216 -0
  145. package/docs/course/11-the-challenge.md +227 -0
  146. package/docs/course/12-going-live.md +211 -0
  147. package/docs/course/13-operating-it.md +238 -0
  148. package/docs/course/14-scaling.md +195 -0
  149. package/docs/course/15-extending.md +259 -0
  150. package/docs/course/16-proving-it.md +266 -0
  151. package/docs/course/index.md +112 -0
  152. package/docs/design/decisions.md +216 -0
  153. package/docs/detection/client-signals.md +96 -0
  154. package/docs/detection/detectors.md +376 -0
  155. package/docs/detection/index.md +134 -0
  156. package/docs/detection/signatures.md +111 -0
  157. package/docs/detection/verification.md +123 -0
  158. package/docs/detection/writing-a-detector.md +145 -0
  159. package/docs/index.md +100 -0
  160. package/docs/integration/adapters.md +138 -0
  161. package/docs/integration/client-ip.md +114 -0
  162. package/docs/integration/index.md +55 -0
  163. package/docs/integration/stores.md +76 -0
  164. package/docs/operations/audit.md +116 -0
  165. package/docs/operations/dashboard.md +725 -0
  166. package/docs/operations/index.md +101 -0
  167. package/docs/operations/metrics.md +74 -0
  168. package/docs/operations/notifications.md +72 -0
  169. package/docs/operations/runtime-changes.md +118 -0
  170. package/docs/policy/actions.md +196 -0
  171. package/docs/policy/index.md +84 -0
  172. package/docs/policy/presets.md +191 -0
  173. package/docs/policy/robots.md +128 -0
  174. package/docs/policy/rules.md +134 -0
  175. package/docs/reference/api.md +126 -0
  176. package/docs/reference/configuration.md +175 -0
  177. package/docs/start/choosing-a-policy.md +127 -0
  178. package/docs/start/first-integration.md +127 -0
  179. package/docs/start/installation.md +63 -0
  180. package/docs/start/upgrading.md +72 -0
  181. package/docs/testing/cli.md +119 -0
  182. package/docs/testing/corpus.md +129 -0
  183. package/docs/testing/index.md +61 -0
  184. package/docs/testing/replay.md +102 -0
  185. package/docs/testing/try-it.md +189 -0
  186. package/package.json +145 -0
@@ -0,0 +1,129 @@
1
+ # The traffic corpus
2
+
3
+ 526 cases, 739 requests, 6,002 header lines, 55 categories of real web traffic — and a
4
+ harness that runs them against *your* configuration.
5
+
6
+ ← [Documentation](../index.md) · [Testing](index.md)
7
+
8
+ ---
9
+
10
+ ## Why it exists
11
+
12
+ Every claim in this documentation about what detection catches, what a preset does, and who
13
+ a policy hurts is measured on this corpus rather than asserted. It is also the reason the
14
+ [guard](../concepts/the-guard.md) can be defended concretely: "no case marked as a person is
15
+ denied service" is a test that runs, not a design intention.
16
+
17
+ ```bash
18
+ npm run corpus # the protect-content preset
19
+ npm run corpus -- --preset protect-data
20
+ npm run corpus -- --audience human --verbose
21
+ ```
22
+
23
+ ```
24
+ monitor-only 526/526 cases pass · 0 false positives
25
+ protect-content 526/526 cases pass · 0 false positives
26
+ protect-data 525/526 cases pass · 0 false positives
27
+ protect-auth 511/526 cases pass · 0 false positives
28
+ ```
29
+
30
+ The two strict presets do not pass everything, and **the gap is the documentation rather
31
+ than a defect**: [`protect-auth`](../policy/presets.md#protect-auth) blocks proven
32
+ automation, and the fifteen cases it fails are your payment webhook, your own server-side
33
+ renderer, and your orchestrator's health probes — correct on a login route, an outage
34
+ anywhere else. That is why the preset says to mount it on those routes only, and why the
35
+ corpus keeps the receipts.
36
+
37
+ ## What is in it
38
+
39
+ | Audience | Cases | Examples |
40
+ | -------- | ----: | -------- |
41
+ | **human** | 182 | 30 browser profiles across Chromium, Gecko and WebKit · desktop, mobile, tablet, console, television · 40 in-app WebViews (Instagram, TikTok, WeChat, KakaoTalk, LINE, VK, banking and airline apps) · Electron desktop apps · Tor, `resistFingerprinting`, Sec-GPC · Lynx, w3m, screen readers · IE11, Android 4.4, a car's infotainment screen · corporate proxies, carrier transcoders, CGNAT, iCloud Private Relay · an author signing in at `/wp-login.php` and a developer searching a docs site for SQL syntax |
42
+ | **benign-bot** | 144 | Googlebot and Bingbot verified by DNS · 30 regional crawlers (Naver, Seznam, Coc Coc, Sogou, 360, Shenma, Qwant, Mojeek) · Google's and Microsoft's specialist fleets · 12 link unfurlers · monitoring · feed and podcast clients · academic and archival crawlers · ad verification · email link scanners |
43
+ | **declared-bot** | 32 | The AI fleet split by job — training, search, fetch-for-a-user — plus an agentic browser |
44
+ | **unwanted-bot** | 106 | SEO and market-intelligence crawlers · **50 HTTP clients in their real header orders**, across Python, Node, JVM, Go, Rust, PHP, Ruby, .NET, Perl · headless runtimes · fabricated User-Agents from a randomiser (Chrome on an iPhone, Windows and macOS at once, Firefox on WebKit) |
45
+ | **hostile** | 29 | Forged Googlebot four ways · scanners · credential stuffing · traps · protocol abuse and request-smuggling framing · forwarding-header injection · wordlist probes for `/.env`, `/.git`, JNDI and TRACE |
46
+ | **infrastructure** | 33 | CDN origin pulls (Cloudflare, Fastly, Akamai, CloudFront) · API gateways and service meshes · k8s and ELB probes · webhooks · browser prefetch |
47
+
48
+ Every request is built the way the client actually builds it — the Client Hints block, the
49
+ Fetch Metadata group, `Priority`, a cookie jar from somebody who has genuinely used the web
50
+ (`_ga`, `_fbp`, `__cf_bm`, a TCF consent string), cache validators on a revisit, and the
51
+ conditional long tail: `Sec-GPC`, `Save-Data`, `Sec-Purpose: prefetch`, `Early-Data`,
52
+ `Sec-CH-Prefers-Color-Scheme`, the Network Information hints. 155 requests carry cookies.
53
+ Every case records **provenance**, enforced by a test.
54
+
55
+ ## The guarantee, checked rather than argued
56
+
57
+ Every case marked `human` carries an automatic `neverAction: ["block", "drop", "redirect"]`,
58
+ enforced by the harness whatever the case's own expectations say, and asserted **for every
59
+ request of a sequence** — because a person denied on request seven is still a person denied.
60
+
61
+ There is exactly one documented exemption, and it is *counted rather than hidden*: a
62
+ listener tapping a link inside a podcast app, whose software sends the same
63
+ crawler-convention User-Agent it uses to fetch feeds. The guarantee is about guesses, and a
64
+ client that announces itself automated is not a guess. The scorecard lists every such case
65
+ with a written reason.
66
+
67
+ ## Two details that make it honest
68
+
69
+ **Header order is reproduced, not invented.** Chromium emits `Host, Connection, sec-ch-ua…,
70
+ User-Agent, Accept, Sec-Fetch-*, Accept-Encoding, Accept-Language, Priority`; Gecko leads
71
+ with identity and closes with Fetch Metadata and `TE`; WebKit interleaves them.
72
+ `python-requests` sending `Accept-Encoding` before `Accept` is only a signal if the corpus
73
+ gets this right.
74
+
75
+ **DNS is controlled, not mocked away.** Each case declares its answers, so *"the operator's
76
+ DNS disproves this claim"* and *"our resolver was briefly unhappy"* can be told apart. One is
77
+ an impersonator; the other is Googlebot during a blip. See
78
+ [verification](../detection/verification.md).
79
+
80
+ ## The evasion ladder
81
+
82
+ Five cases that run from crude to undetectable, and **the top rungs are expected to fail**:
83
+
84
+ | | | |
85
+ |-|-|-|
86
+ | 1 | Copied the User-Agent only | caught, four independent signals |
87
+ | 2 | Copied the header set, not the order | caught, weakly |
88
+ | 3 | Copied the order and the Client Hints | **not caught** from one request |
89
+ | 4 | …at a machine-perfect rhythm | caught by `cadence`, and only by `cadence` |
90
+ | 5 | …paced like a person, a few pages per address | **not caught at all** |
91
+
92
+ Level 5 is in the corpus so that nobody can claim otherwise. At that point the difference
93
+ from a person has stopped being technical, and what defeats it is cost — a
94
+ [proof of work](../challenge/index.md), or an account — not detection. See
95
+ [the threat model](../concepts/threat-model.md).
96
+
97
+ ## Running it against your own configuration
98
+
99
+ It is a published entry point, not a test fixture:
100
+
101
+ ```ts
102
+ import { runCorpus } from "@osqd/bothandlerjs/corpus";
103
+
104
+ const scorecard = await runCorpus({
105
+ create: ({ resolver, clock }) => new BotHandler({ ...myProductionConfig, resolver, clock }),
106
+ assertActions: false, // your actions are yours; the invariants are not
107
+ });
108
+
109
+ if (scorecard.falsePositives.length > 0) throw new Error("this policy turns people away");
110
+ ```
111
+
112
+ `assertActions: false` is the setting to understand. The corpus knows what each case *is*;
113
+ it does not know what your policy should do about it. The invariants — nothing marked as a
114
+ person is denied — hold regardless.
115
+
116
+ DNS is controlled and the clock is manual, so it runs offline, deterministically, in CI.
117
+
118
+ ## Adding cases
119
+
120
+ See [`src/corpus/README.md`](../../src/corpus/README.md). The bar is provenance: a case has
121
+ to say where its headers came from, and a test enforces it. A corpus of invented traffic
122
+ would make every claim in this documentation false in a way nobody could see.
123
+
124
+ ## Related
125
+
126
+ - [The CLI](cli.md#check) — the same corpus, one command
127
+ - [Log replay](replay.md) — the complement: traffic that is actually yours
128
+ - [The guard](../concepts/the-guard.md) — the invariant this enforces
129
+ - [Presets](../policy/presets.md) — the four the scorecard reports on
@@ -0,0 +1,61 @@
1
+ # Testing
2
+
3
+ Finding out what a policy does to your traffic before it does it.
4
+
5
+ ← [Documentation](../index.md)
6
+
7
+ ---
8
+
9
+ This library's central claim is that a bot policy is a claim about **your** traffic, and
10
+ that it should be checked rather than believed. Everything on these pages exists to make
11
+ that cheap.
12
+
13
+ | Page | Answers |
14
+ | ---- | ------- |
15
+ | [The CLI](cli.md) | "what is this one request?", "what does this policy do?" |
16
+ | [The corpus](corpus.md) | "who would this configuration hurt?" — 526 shapes of real traffic |
17
+ | [Log replay](replay.md) | "what would this have done to yesterday?" |
18
+ | [Try it](try-it.md) | "what does it look like running?" — the demo, the simulator, three dashboards |
19
+
20
+ ---
21
+
22
+ ## The order to use them in
23
+
24
+ 1. **[`explain`](cli.md#explain)** — one request. The ticket that says "why was I
25
+ challenged?".
26
+ 2. **[`check`](cli.md#check)** — your policy against the corpus. This is the CI step. It
27
+ exits non-zero if any case marked as a person is denied service.
28
+ 3. **[`replay`](replay.md)** — your policy against your own access log. The corpus knows
29
+ what the internet looks like; only your logs know what *your* visitors look like.
30
+ 4. **[`monitor-only`](../policy/presets.md#monitor-only) in production** — for a week,
31
+ watching [the dashboard](../operations/dashboard.md), before anything is enforced.
32
+
33
+ Step 4 is not optional and the other three do not replace it.
34
+
35
+ ## Testing your own configuration, not a preset
36
+
37
+ The corpus is a published entry point:
38
+
39
+ ```ts
40
+ import { runCorpus } from "@osqd/bothandlerjs/corpus";
41
+
42
+ const scorecard = await runCorpus({
43
+ create: ({ resolver, clock }) => new BotHandler({ ...myProductionConfig, resolver, clock }),
44
+ assertActions: false, // your actions are yours; the invariants are not
45
+ });
46
+ if (scorecard.falsePositives.length > 0) throw new Error("this policy turns people away");
47
+ ```
48
+
49
+ DNS is controlled rather than real and the clock is manual, so it is reproducible, offline,
50
+ and safe to run anywhere — including in CI, which is the point.
51
+
52
+ ## Testing a detector you wrote
53
+
54
+ [Writing a detector](../detection/writing-a-detector.md) covers the contract; the corpus is
55
+ how you find out whether your new signal fires on 182 kinds of person as well as on the
56
+ thing you built it for. That is usually the surprising part.
57
+
58
+ ## Related
59
+
60
+ - [Choosing a policy](../start/choosing-a-policy.md) — where these fit in the decision
61
+ - [The dashboard](../operations/dashboard.md) — the request tester, which is `explain` with a UI
@@ -0,0 +1,102 @@
1
+ # Log replay
2
+
3
+ What your policy would have done to yesterday.
4
+
5
+ ← [Documentation](../index.md) · [Testing](index.md)
6
+
7
+ ---
8
+
9
+ A bot policy is a claim about *your* traffic. The only honest way to check it before it
10
+ starts turning people away is to run it over traffic you already have.
11
+
12
+ ```bash
13
+ npx @osqd/bothandlerjs replay /var/log/nginx/access.log --preset protect-content
14
+ ```
15
+
16
+ ```
17
+ replay — 4,000 requests, preset "protect-content", format clf
18
+
19
+ verdicts
20
+ unknown 2906 72.7% ██████████████████
21
+ confirmed-bot 1094 27.4% ███████
22
+
23
+ what would have happened
24
+ allow 2906 72.7% ██████████████████
25
+ challenge 477 11.9% ███
26
+ tag 387 9.7% ██
27
+ block 116 2.9% █
28
+
29
+ 116 request(s) would have been DENIED, in 1 distinct kind(s). Read them.
30
+ ─────────────────────────────────────────────────────────────────────
31
+ 116x block by rule "scanner-block" — confirmed-bot, proven
32
+ sqlmap/1.7.2#stable (https://sqlmap.org)
33
+ [certain] self-identified: User-Agent identifies sqlmap
34
+ ```
35
+
36
+ **That last block is the point of the exercise.** Every request the policy would have
37
+ refused, grouped by kind, with the evidence. If any of them is a person, the policy is
38
+ wrong — and you found out from a log file rather than from a support ticket.
39
+
40
+ ---
41
+
42
+ ## Prefer JSON Lines, and here is why
43
+
44
+ It accepts Combined/Common Log Format or JSON Lines. **JSON Lines is much better**, and the
45
+ reason is a real distinction the library models:
46
+
47
+ > A header missing from a *record* is not a header missing from the *request*.
48
+
49
+ An nginx access line records the User-Agent and the Referer and nothing else. Several
50
+ [detectors](../detection/detectors.md) reason from absence — "claims to be a browser but
51
+ sent no `Accept-Language`" — and on a CLF line that reasoning is not merely weak, it is
52
+ meaningless.
53
+
54
+ So `RequestFacts.partialHeaders` marks a header-poor source, and every absence-based
55
+ detector stands down; the ones that reason from what *is* present carry on. Without it, a
56
+ replay over real nginx logs reports most of a site's human traffic as suspected bots —
57
+ worse than useless, because it is confidently wrong.
58
+
59
+ A CLF replay therefore **under-reports**, and its silence is not a clean bill of health.
60
+
61
+ ### The JSON Lines shape
62
+
63
+ One object per line:
64
+
65
+ ```json
66
+ { "ip": "203.0.113.5", "method": "GET", "url": "/products/12", "headers": { "user-agent": "...", "accept-language": "en-GB" }, "timestamp": 1757030400000 }
67
+ ```
68
+
69
+ Configure your access log to emit the header set you care about — at minimum `user-agent`,
70
+ `accept`, `accept-language`, `accept-encoding`, `referer` and the `sec-ch-ua`/`sec-fetch-*`
71
+ groups — and the replay sees what detection would have seen.
72
+
73
+ ## Setting `partialHeaders` yourself
74
+
75
+ Whenever you build facts from something that filters headers:
76
+
77
+ ```ts
78
+ createFacts({ method, url, headers, ip, partialHeaders: true });
79
+ ```
80
+
81
+ Any log pipeline, any CDN export, any sampled trace. It is the difference between a replay
82
+ that tells you something and a replay that tells you everything is a bot.
83
+
84
+ ## What it cannot tell you
85
+
86
+ **Behaviour it did not record.** `cadence` and `crawl-breadth` read a sequence, so they work
87
+ only if your log preserves ordering and timestamps — which JSON Lines does and a truncated
88
+ sample does not.
89
+
90
+ **DNS as it was.** [Verification](../detection/verification.md) runs against today's DNS,
91
+ not the DNS of the day the log was written. A crawler that has since changed hands verifies
92
+ differently.
93
+
94
+ **The effect of the mitigation.** A replay shows what the policy would have *asked for*, not
95
+ how the traffic would have responded to being challenged or slowed. Those are the same
96
+ number only if nobody adapts, and adapting is what the population in question does.
97
+
98
+ ## Related
99
+
100
+ - [The CLI](cli.md) — every flag
101
+ - [The corpus](corpus.md) — the complement: traffic you do not have yet
102
+ - [Detection](../detection/index.md) — the pipeline being replayed
@@ -0,0 +1,189 @@
1
+ # Try it
2
+
3
+ A demo site with a live dashboard, eighteen scripted clients, and three dashboards
4
+ showing three different amounts of the same traffic.
5
+
6
+ ← [Documentation](../index.md) · [Testing](index.md)
7
+
8
+ ---
9
+
10
+ ## The demo and the simulator
11
+
12
+ Two commands, two terminals:
13
+
14
+ ```bash
15
+ npm run demo # a protected site on :9673 + a live dashboard on :9674
16
+ npm run simulate # points eighteen kinds of client at it
17
+ ```
18
+
19
+ Open **http://localhost:9674/** and watch. Every request lands in the feed with its
20
+ verdict, its score, the action taken, and — click a row — the individual pieces of
21
+ evidence with their certainty tiers and, for proven ones, the written basis. The
22
+ second tab is the statistics panel: traffic over time, the latency histogram, and the
23
+ distributions behind the tiles.
24
+
25
+ That dashboard is not demo scaffolding. It is the one the library ships, started with
26
+ one call — `detector.serveDashboard({ port: 9674 })` — and you can put the same thing
27
+ in your own application. See [The dashboard](../operations/dashboard.md#the-dashboard).
28
+
29
+ The demo policy is deliberately overreaching: it asks to **block every suspected
30
+ bot**. Strict mode refuses every time. Those amber rows are the whole library in one
31
+ screen — requests scoring 93 and 98 that still do not get blocked, because a score is
32
+ not proof.
33
+
34
+ `npm run simulate` runs these, each from its own source address so they are distinct
35
+ actors, and prints what happened:
36
+
37
+ ```
38
+ human 7x 200 2ms unknown x7
39
+ curl 2x 429 1ms confirmed-bot x2
40
+ python 2x 429 1ms confirmed-bot x2
41
+ go 1x 429 1ms confirmed-bot
42
+ scanner 3x 403 1ms confirmed-bot x3
43
+ headless 1x 429 1ms confirmed-bot
44
+ spoofed-browser 3x 429 1ms suspected-bot x3
45
+ platform-mismatch 1x 200 1ms unknown
46
+ fake-googlebot 1x 403 39ms confirmed-bot
47
+ verified-crawler 4x 200 2ms verified-bot x4
48
+ declared-crawler 30x 200, 6x 429 1ms confirmed-bot x36
49
+ trap 1x 200, 1x 403 1ms unknown, confirmed-bot
50
+ trap-field 1x 403 2ms confirmed-bot
51
+ scraper 34x 429 1ms suspected-bot x34
52
+ metronome 14x 200 2ms unknown x14
53
+ burst 30x 200 1ms unknown x30
54
+ credential-stuffing 8x 401 252ms unknown x8
55
+ no-user-agent 1x 200 1ms unknown
56
+ ```
57
+
58
+ Four rows are worth reading twice:
59
+
60
+ - **`human`** — seven ordinary page views, all served, verdict `unknown`. This is the
61
+ row that matters. Every other row is only interesting if this one stays clean.
62
+ - **`verified-crawler`** — GPTBot arriving from the operator's published range is
63
+ *allowed*, ahead of every other rule. Bot detection that costs you your search
64
+ traffic has not helped.
65
+ - **`burst`** — thirty requests as fast as a socket allows, and all thirty are
66
+ served. Rate is capped at `moderate` and cannot block on its own, because behind a
67
+ corporate NAT that burst is a floor of people, not a bot — and the client's request
68
+ carries the marks of a real browsing session, which discounts it further.
69
+ - **`credential-stuffing`** — served, `401`, and **253 ms each**. No challenge, no
70
+ block, nothing excluded; the attack simply stops being economic.
71
+
72
+ The simulator speaks raw HTTP over a socket rather than using `fetch`, because
73
+ `fetch` normalises the header set and fixes the order — the exact properties several
74
+ detectors read. Run one scenario at a time with `npm run simulate:curl`,
75
+ `npm run simulate:trap`, and so on; `npm run simulate:list` prints them all.
76
+
77
+ ### Replaying the whole corpus over the wire
78
+
79
+ ```bash
80
+ npm run simulate:corpus # all 526 cases
81
+ npm run simulate:corpus:human # only the people
82
+ npm run simulate -- --corpus --tag known-cost # only the awkward ones
83
+ npm run simulate -- --corpus --case browse-chrome-windows --verbose
84
+ ```
85
+
86
+ ```
87
+ by audience
88
+ human 178/178 pass served 173, challenge 5
89
+ benign-bot 137/137 pass served 89, block 46, challenge 2
90
+ unwanted-bot 104/104 pass challenge 78, served 23, block 3
91
+ declared-bot 29/ 29 pass served 27, block 2
92
+ infrastructure 33/ 33 pass served 19, challenge 14
93
+ hostile 19/ 19 pass block 10, challenge 5, served 4
94
+
95
+ 500/500 replayed cases pass · 0 false positives
96
+ ```
97
+
98
+ This is the stronger of the two tests. The in-process corpus runner calls `assess()`
99
+ directly; replaying the same cases down a socket exercises the **whole stack** — the
100
+ adapter, Node's header parsing, whether wire order survives into `rawHeaders`, cookie
101
+ parsing, and client-address resolution through the forwarding headers. A case that
102
+ passes in-process and fails on the wire has found an adapter bug, and one already did:
103
+ `exposeVerdictHeaders` turned out to be honoured on every response *except* the
104
+ challenge interstitial.
105
+
106
+ It also tells you where a check *cannot* apply on this runtime. A request carrying
107
+ both `Content-Length` and `Transfer-Encoding` is proven automation by RFC 9112 — and
108
+ Node's own parser answers it 400 before any handler runs, so that case reaches the
109
+ engine only when the facts were built somewhere Node's parser is not in the path: an
110
+ edge worker, a WAF event, a log line. The replay found that, and the case now says so
111
+ in its skip line rather than passing by accident.
112
+
113
+ Cases with no wire equivalent are skipped and counted rather than quietly passing —
114
+ HTTP/2 fixtures, cases that declare their own DNS answers, cases needing a clearance
115
+ token only the server can mint, cases the runtime refuses before detection, and cases
116
+ needing ranges the demo does not load.
117
+
118
+ ---
119
+
120
+ **Start in monitor mode.** Run `preset: "monitor-only"` against real traffic for a
121
+ week before you enforce anything. It withholds nothing from anybody and shows you
122
+ exactly what your traffic looks like. Every bot policy that has caused an outage was
123
+ deployed straight to enforcement by someone confident they already knew.
124
+
125
+ ---
126
+
127
+ ## Role-gated dashboards
128
+
129
+ A second demo, for the question the first one does not answer: who is allowed to look?
130
+
131
+ ```bash
132
+ npm run demo:roles
133
+ ```
134
+
135
+ Four listeners this time — a protected site with an operator console on **:9683**, and
136
+ three dashboards over the same handler: an **analyst** view on **:9684**, an
137
+ **operator** view on **:9685** with the policy editor and the reset button, and an
138
+ **admin** view on **:9686** that can also change the guard.
139
+
140
+ Start at **http://localhost:9683/operator** and pick a role. There is no password, on
141
+ purpose: this demonstrates *authorisation* — what a proven identity may reach — and
142
+ the proving belongs to your identity provider. Signing in sets an HMAC-signed cookie
143
+ carrying the role, and `auth: { authorize }` reads it.
144
+
145
+ Then try the three dashboards as each role:
146
+
147
+ | | analyst `:9684` | operator `:9685` | admin `:9686` |
148
+ | --- | --- | --- | --- |
149
+ | anonymous | `401` | `401` | `401` |
150
+ | analyst | `200` | `401` | `401` |
151
+ | operator | `200` | `200` | `401` |
152
+ | admin | `200` | `200` | `200` |
153
+
154
+ As an analyst the other two answer `401` on *every* path — authentication runs before
155
+ routing, so a caller without the role cannot even map the endpoints. And on the analyst
156
+ dashboard the editor endpoint answers `403` even for an admin, because `controls`
157
+ belongs to the listener rather than to the visitor: a dashboard started without it has
158
+ no editor to reach, whoever is asking.
159
+
160
+ Each rung adds exactly one thing, and the last is the one to look at hardest. An
161
+ operator can write a rule that asks for more than the evidence supports, and the guard
162
+ stops it. The admin listener is the only one that can change *whether anything stops
163
+ it* — `controls.editGuard`, off by default and separate from `editPolicy` on purpose —
164
+ and the only one that can act on a single client: allowlist an address, forget an
165
+ actor's history, clear one as human (`controls.editRanges`).
166
+
167
+ Change any of it and watch the terminal. Every change prints with the name of whoever
168
+ made it, because this demo's `authorize` returns the signed-in email instead of `true`
169
+ — which is all the dashboard needs to attribute a change in the handler's warnings, in
170
+ the `guard-change`, `range-change` and `actor-change` events, and on the marker it
171
+ leaves on the traffic timeline.
172
+
173
+ The analyst listener also runs with `sections: { evidence: false, policy: false }` and
174
+ `redact: { maskIp: true }`. It has two tabs rather than three, its rows show what
175
+ happened without naming which detector fired, and the feed shows `203.0.113.0/24` where
176
+ the others show the address. Open devtools on it: the evidence is not hidden from the
177
+ page, it never reaches it. The dashboard people watch all day need not be the one that
178
+ names individuals, or the one that explains your detection to whoever is scraping you.
179
+
180
+ Read [the dashboard's Roles section](../operations/dashboard.md#roles) for what to change before
181
+ this goes anywhere real.
182
+
183
+ ---
184
+
185
+ ## Related
186
+
187
+ - [Testing](index.md) — the rest: the CLI, the corpus, log replay
188
+ - [The dashboard](../operations/dashboard.md) — every option the demo is showing you
189
+ - [The corpus](corpus.md) — the traffic the simulator is a live version of
package/package.json ADDED
@@ -0,0 +1,145 @@
1
+ {
2
+ "name": "@osqd/bothandlerjs",
3
+ "version": "0.3.0",
4
+ "publishConfig": {
5
+ "access": "public"
6
+ },
7
+ "description": "Bot traffic detection and handling for TypeScript — evidence-tiered detection that separates proof from suspicion, so blocking never rests on a guess.",
8
+ "keywords": [
9
+ "bot-detection",
10
+ "bot-management",
11
+ "crawler",
12
+ "scraper",
13
+ "anti-bot",
14
+ "user-agent",
15
+ "rate-limit",
16
+ "security",
17
+ "middleware",
18
+ "typescript"
19
+ ],
20
+ "license": "SEE LICENSE IN LICENSE",
21
+ "repository": {
22
+ "type": "git",
23
+ "url": "git+https://github.com/OSQD-ts/BothandlerJS.git"
24
+ },
25
+ "homepage": "https://github.com/OSQD-ts/BothandlerJS#readme",
26
+ "bugs": {
27
+ "url": "https://github.com/OSQD-ts/BothandlerJS/issues"
28
+ },
29
+ "author": "Michał Płatosz <platosz.michal@gmail.com>",
30
+ "type": "module",
31
+ "exports": {
32
+ ".": {
33
+ "types": "./dist/index.d.ts",
34
+ "import": "./dist/index.js",
35
+ "require": "./dist/index.cjs"
36
+ },
37
+ "./adapters": {
38
+ "types": "./dist/adapters/index.d.ts",
39
+ "import": "./dist/adapters/index.js",
40
+ "require": "./dist/adapters/index.cjs"
41
+ },
42
+ "./client": {
43
+ "types": "./dist/client/index.d.ts",
44
+ "import": "./dist/client/index.js",
45
+ "require": "./dist/client/index.cjs"
46
+ },
47
+ "./package.json": "./package.json",
48
+ "./cli": {
49
+ "types": "./dist/cli.d.ts",
50
+ "import": "./dist/cli.js",
51
+ "require": "./dist/cli.cjs"
52
+ },
53
+ "./corpus": {
54
+ "types": "./dist/corpus/index.d.ts",
55
+ "import": "./dist/corpus/index.js",
56
+ "require": "./dist/corpus/index.cjs"
57
+ }
58
+ },
59
+ "main": "./dist/index.cjs",
60
+ "types": "./dist/index.d.ts",
61
+ "bin": {
62
+ "bothandlerjs": "bin/bothandlerjs.mjs"
63
+ },
64
+ "directories": {
65
+ "example": "examples",
66
+ "test": "tests"
67
+ },
68
+ "files": [
69
+ "dist",
70
+ "bin",
71
+ "docs",
72
+ "README.md",
73
+ "CHANGELOG.md",
74
+ "LICENSE"
75
+ ],
76
+ "scripts": {
77
+ "build": "tsup && tsc --emitDeclarationOnly",
78
+ "build:watch": "tsup --watch",
79
+ "test": "vitest run",
80
+ "test:watch": "vitest",
81
+ "test:coverage": "vitest run --coverage",
82
+ "test:browser": "vitest run --config vitest.browser.config.ts",
83
+ "lint": "biome check .",
84
+ "docs:check": "node scripts/check-links.mjs",
85
+ "format": "biome check --write .",
86
+ "demo": "tsx demo/server.ts",
87
+ "demo:roles": "tsx demo/roles.ts",
88
+ "simulate": "tsx scripts/simulate.ts",
89
+ "simulate:corpus": "tsx scripts/simulate.ts --corpus",
90
+ "simulate:corpus:human": "tsx scripts/simulate.ts --corpus --audience human",
91
+ "simulate:list": "tsx scripts/simulate.ts --list",
92
+ "simulate:human": "tsx scripts/simulate.ts human",
93
+ "simulate:curl": "tsx scripts/simulate.ts curl",
94
+ "simulate:python": "tsx scripts/simulate.ts python",
95
+ "simulate:go": "tsx scripts/simulate.ts go",
96
+ "simulate:scanner": "tsx scripts/simulate.ts scanner",
97
+ "simulate:headless": "tsx scripts/simulate.ts headless",
98
+ "simulate:spoofed-browser": "tsx scripts/simulate.ts spoofed-browser",
99
+ "simulate:platform-mismatch": "tsx scripts/simulate.ts platform-mismatch",
100
+ "simulate:fake-googlebot": "tsx scripts/simulate.ts fake-googlebot",
101
+ "simulate:verified-crawler": "tsx scripts/simulate.ts verified-crawler",
102
+ "simulate:declared-crawler": "tsx scripts/simulate.ts declared-crawler",
103
+ "simulate:trap": "tsx scripts/simulate.ts trap",
104
+ "simulate:trap-field": "tsx scripts/simulate.ts trap-field",
105
+ "simulate:scraper": "tsx scripts/simulate.ts scraper",
106
+ "simulate:metronome": "tsx scripts/simulate.ts metronome",
107
+ "simulate:burst": "tsx scripts/simulate.ts burst",
108
+ "simulate:credential-stuffing": "tsx scripts/simulate.ts credential-stuffing",
109
+ "simulate:no-user-agent": "tsx scripts/simulate.ts no-user-agent",
110
+ "typecheck": "tsc -p tsconfig.typecheck.json && tsc -p tsconfig.browser.json",
111
+ "bench": "tsx --expose-gc scripts/bench.ts",
112
+ "corpus": "tsx scripts/corpus.ts",
113
+ "example": "tsx examples/express-basic.ts",
114
+ "example:fetch": "tsx examples/fetch-worker.ts",
115
+ "example:notify": "tsx examples/notifications.ts",
116
+ "prepublishOnly": "npm run build",
117
+ "prepack": "npm run build",
118
+ "client:build": "node scripts/build-client.mjs",
119
+ "client:check": "node scripts/build-client.mjs --check",
120
+ "prebuild": "npm run client:build",
121
+ "pretest": "npm run client:build",
122
+ "pretypecheck": "npm run client:build",
123
+ "pretest:browser": "npm run client:build",
124
+ "bench:guard": "tsx scripts/bench-guard.ts"
125
+ },
126
+ "devDependencies": {
127
+ "@biomejs/biome": "^2.4.0",
128
+ "@types/express": "^5.0.6",
129
+ "@types/node": "^26.3.0",
130
+ "@vitest/coverage-v8": "^4.1.11",
131
+ "axe-core": "^4.13.0",
132
+ "esbuild": "^0.27.7",
133
+ "express": "^5.2.1",
134
+ "playwright": "^1.62.1",
135
+ "tsup": "^8.5.1",
136
+ "tsx": "^4.23.12",
137
+ "typescript": "^5.9.3",
138
+ "vitest": "^4.1.11"
139
+ },
140
+ "engines": {
141
+ "node": ">=20"
142
+ },
143
+ "module": "./dist/index.js",
144
+ "sideEffects": false
145
+ }