@osqd/bothandlerjs 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +516 -0
  2. package/LICENSE +102 -0
  3. package/README.md +308 -0
  4. package/bin/bothandlerjs.mjs +14 -0
  5. package/dist/actions/index.d.ts +22 -0
  6. package/dist/actions/types.d.ts +59 -0
  7. package/dist/adapters/fastify.d.ts +43 -0
  8. package/dist/adapters/fetch.d.ts +51 -0
  9. package/dist/adapters/index.cjs +723 -0
  10. package/dist/adapters/index.cjs.map +1 -0
  11. package/dist/adapters/index.d.ts +8 -0
  12. package/dist/adapters/index.js +692 -0
  13. package/dist/adapters/index.js.map +1 -0
  14. package/dist/adapters/koa.d.ts +27 -0
  15. package/dist/adapters/node.d.ts +29 -0
  16. package/dist/adapters/shared.d.ts +39 -0
  17. package/dist/audit.d.ts +172 -0
  18. package/dist/challenge/index.d.ts +176 -0
  19. package/dist/challenge/language.d.ts +57 -0
  20. package/dist/challenge/page.d.ts +42 -0
  21. package/dist/challenge/pow.d.ts +33 -0
  22. package/dist/challenge/token.d.ts +72 -0
  23. package/dist/cli.cjs +13388 -0
  24. package/dist/cli.cjs.map +1 -0
  25. package/dist/cli.d.ts +5 -0
  26. package/dist/cli.js +13358 -0
  27. package/dist/cli.js.map +1 -0
  28. package/dist/client/index.cjs +139 -0
  29. package/dist/client/index.cjs.map +1 -0
  30. package/dist/client/index.d.ts +63 -0
  31. package/dist/client/index.js +112 -0
  32. package/dist/client/index.js.map +1 -0
  33. package/dist/config.d.ts +316 -0
  34. package/dist/core.d.ts +443 -0
  35. package/dist/corpus/adversarial.d.ts +2 -0
  36. package/dist/corpus/advertising-email.d.ts +2 -0
  37. package/dist/corpus/ai-crawlers.d.ts +2 -0
  38. package/dist/corpus/benign-bots.d.ts +2 -0
  39. package/dist/corpus/cdn-gateways.d.ts +2 -0
  40. package/dist/corpus/cookies.d.ts +49 -0
  41. package/dist/corpus/crawlers-regional.d.ts +2 -0
  42. package/dist/corpus/crawlers-vertical.d.ts +2 -0
  43. package/dist/corpus/headers.d.ts +403 -0
  44. package/dist/corpus/humans-apps.d.ts +2 -0
  45. package/dist/corpus/humans-browsers.d.ts +2 -0
  46. package/dist/corpus/humans.d.ts +2 -0
  47. package/dist/corpus/index.cjs +4196 -0
  48. package/dist/corpus/index.cjs.map +1 -0
  49. package/dist/corpus/index.d.ts +37 -0
  50. package/dist/corpus/index.js +4132 -0
  51. package/dist/corpus/index.js.map +1 -0
  52. package/dist/corpus/infrastructure.d.ts +17 -0
  53. package/dist/corpus/libraries-extended.d.ts +2 -0
  54. package/dist/corpus/ranges.d.ts +31 -0
  55. package/dist/corpus/reputation.d.ts +21 -0
  56. package/dist/corpus/runner.d.ts +130 -0
  57. package/dist/corpus/schema.d.ts +185 -0
  58. package/dist/corpus/tooling.d.ts +2 -0
  59. package/dist/corpus/unwanted.d.ts +2 -0
  60. package/dist/crawler-ranges.d.ts +113 -0
  61. package/dist/dashboard/client.generated.d.ts +7 -0
  62. package/dist/dashboard/feed.d.ts +186 -0
  63. package/dist/dashboard/index.d.ts +5 -0
  64. package/dist/dashboard/page.d.ts +51 -0
  65. package/dist/dashboard/parse-request.d.ts +28 -0
  66. package/dist/dashboard/preview.d.ts +45 -0
  67. package/dist/dashboard/server.d.ts +70 -0
  68. package/dist/dashboard/types.d.ts +714 -0
  69. package/dist/detectors/accept-signature.d.ts +6 -0
  70. package/dist/detectors/browsing-coherence.d.ts +44 -0
  71. package/dist/detectors/cadence.d.ts +28 -0
  72. package/dist/detectors/clearance.d.ts +25 -0
  73. package/dist/detectors/client-hints.d.ts +19 -0
  74. package/dist/detectors/client-signals.d.ts +27 -0
  75. package/dist/detectors/crawl-breadth.d.ts +28 -0
  76. package/dist/detectors/crawler-verification.d.ts +39 -0
  77. package/dist/detectors/fetch-metadata.d.ts +20 -0
  78. package/dist/detectors/header-integrity.d.ts +26 -0
  79. package/dist/detectors/header-order.d.ts +27 -0
  80. package/dist/detectors/identity-rotation.d.ts +34 -0
  81. package/dist/detectors/index.d.ts +56 -0
  82. package/dist/detectors/ip-intelligence.d.ts +30 -0
  83. package/dist/detectors/known-bots.d.ts +94 -0
  84. package/dist/detectors/probe-signature.d.ts +50 -0
  85. package/dist/detectors/rate-anomaly.d.ts +36 -0
  86. package/dist/detectors/self-identified.d.ts +33 -0
  87. package/dist/detectors/session-integrity.d.ts +24 -0
  88. package/dist/detectors/tls-fingerprint.d.ts +34 -0
  89. package/dist/detectors/trap.d.ts +98 -0
  90. package/dist/detectors/types.d.ts +70 -0
  91. package/dist/detectors/ua-coherence.d.ts +32 -0
  92. package/dist/evidence.d.ts +45 -0
  93. package/dist/facts.d.ts +46 -0
  94. package/dist/index.cjs +9600 -0
  95. package/dist/index.cjs.map +1 -0
  96. package/dist/index.d.ts +63 -0
  97. package/dist/index.js +9444 -0
  98. package/dist/index.js.map +1 -0
  99. package/dist/internal/async.d.ts +16 -0
  100. package/dist/internal/clock.d.ts +17 -0
  101. package/dist/internal/crypto.d.ts +26 -0
  102. package/dist/internal/dns.d.ts +100 -0
  103. package/dist/internal/emitter.d.ts +18 -0
  104. package/dist/internal/http.d.ts +22 -0
  105. package/dist/internal/ip.d.ts +73 -0
  106. package/dist/internal/lru.d.ts +34 -0
  107. package/dist/internal/matcher.d.ts +41 -0
  108. package/dist/internal/pattern.d.ts +14 -0
  109. package/dist/internal/ua.d.ts +92 -0
  110. package/dist/metrics.d.ts +120 -0
  111. package/dist/notify/hub.d.ts +70 -0
  112. package/dist/notify/index.d.ts +7 -0
  113. package/dist/notify/redact.d.ts +48 -0
  114. package/dist/notify/sinks.d.ts +62 -0
  115. package/dist/notify/types.d.ts +56 -0
  116. package/dist/policy/index.d.ts +7 -0
  117. package/dist/policy/match.d.ts +20 -0
  118. package/dist/policy/policy.d.ts +123 -0
  119. package/dist/policy/presets.d.ts +161 -0
  120. package/dist/policy/types.d.ts +155 -0
  121. package/dist/robots.d.ts +83 -0
  122. package/dist/state.d.ts +161 -0
  123. package/dist/stores/index.d.ts +5 -0
  124. package/dist/stores/memory.d.ts +33 -0
  125. package/dist/stores/redis.d.ts +59 -0
  126. package/dist/stores/types.d.ts +37 -0
  127. package/dist/types.d.ts +250 -0
  128. package/docs/challenge/index.md +152 -0
  129. package/docs/challenge/localisation.md +105 -0
  130. package/docs/concepts/actors.md +113 -0
  131. package/docs/concepts/evidence.md +133 -0
  132. package/docs/concepts/the-guard.md +121 -0
  133. package/docs/concepts/threat-model.md +120 -0
  134. package/docs/concepts/verdicts.md +108 -0
  135. package/docs/course/01-first-assessment.md +178 -0
  136. package/docs/course/02-proof-and-suspicion.md +164 -0
  137. package/docs/course/03-verdicts-and-scores.md +160 -0
  138. package/docs/course/04-the-guard.md +200 -0
  139. package/docs/course/05-detectors.md +178 -0
  140. package/docs/course/06-identity.md +161 -0
  141. package/docs/course/07-actors.md +178 -0
  142. package/docs/course/08-traps.md +183 -0
  143. package/docs/course/09-rules.md +224 -0
  144. package/docs/course/10-actions-and-presets.md +216 -0
  145. package/docs/course/11-the-challenge.md +227 -0
  146. package/docs/course/12-going-live.md +211 -0
  147. package/docs/course/13-operating-it.md +238 -0
  148. package/docs/course/14-scaling.md +195 -0
  149. package/docs/course/15-extending.md +259 -0
  150. package/docs/course/16-proving-it.md +266 -0
  151. package/docs/course/index.md +112 -0
  152. package/docs/design/decisions.md +216 -0
  153. package/docs/detection/client-signals.md +96 -0
  154. package/docs/detection/detectors.md +376 -0
  155. package/docs/detection/index.md +134 -0
  156. package/docs/detection/signatures.md +111 -0
  157. package/docs/detection/verification.md +123 -0
  158. package/docs/detection/writing-a-detector.md +145 -0
  159. package/docs/index.md +100 -0
  160. package/docs/integration/adapters.md +138 -0
  161. package/docs/integration/client-ip.md +114 -0
  162. package/docs/integration/index.md +55 -0
  163. package/docs/integration/stores.md +76 -0
  164. package/docs/operations/audit.md +116 -0
  165. package/docs/operations/dashboard.md +725 -0
  166. package/docs/operations/index.md +101 -0
  167. package/docs/operations/metrics.md +74 -0
  168. package/docs/operations/notifications.md +72 -0
  169. package/docs/operations/runtime-changes.md +118 -0
  170. package/docs/policy/actions.md +196 -0
  171. package/docs/policy/index.md +84 -0
  172. package/docs/policy/presets.md +191 -0
  173. package/docs/policy/robots.md +128 -0
  174. package/docs/policy/rules.md +134 -0
  175. package/docs/reference/api.md +126 -0
  176. package/docs/reference/configuration.md +175 -0
  177. package/docs/start/choosing-a-policy.md +127 -0
  178. package/docs/start/first-integration.md +127 -0
  179. package/docs/start/installation.md +63 -0
  180. package/docs/start/upgrading.md +72 -0
  181. package/docs/testing/cli.md +119 -0
  182. package/docs/testing/corpus.md +129 -0
  183. package/docs/testing/index.md +61 -0
  184. package/docs/testing/replay.md +102 -0
  185. package/docs/testing/try-it.md +189 -0
  186. package/package.json +145 -0
@@ -0,0 +1,224 @@
1
+ # Lesson 9 — Rules
2
+
3
+ **Goal:** write the policy you designed in lesson 3, and understand why order decides
4
+ everything.
5
+
6
+ ← [Course](index.md) · Prev: [Traps](08-traps.md) · Next: [Actions and presets](10-actions-and-presets.md)
7
+
8
+ ---
9
+
10
+ ## The shape
11
+
12
+ ```js
13
+ {
14
+ id: "scraper-ratelimit", // required, and it appears in every decision
15
+ match: { botClass: "scraper" }, // every field present must match
16
+ action: "rate-limit",
17
+ params: { limit: { max: 60, windowMs: 60_000 } },
18
+ reason: "Bulk extraction. Rate-limited rather than refused.",
19
+ }
20
+ ```
21
+
22
+ **First match wins.** Rules are evaluated in order and the first that matches decides;
23
+ nothing accumulates. If none matches, `defaultAction` applies — `allow` unless you change
24
+ it.
25
+
26
+ **`id` is not decoration.** It comes back in `decision.rule`, in the dashboard, in every
27
+ downgrade event and in the replay report. Name rules after what they do, not after what
28
+ they match.
29
+
30
+ **`reason` is read by people.** It appears in the dashboard and in `bothandlerjs replay`
31
+ output. Write it for whoever is looking at a refused request at 2am.
32
+
33
+ ## Every field you can match on
34
+
35
+ Grouped by the question each answers.
36
+
37
+ **What did we conclude?**
38
+
39
+ | Field | |
40
+ | ----- | - |
41
+ | `verdict` | one or several of `confirmed-bot`, `verified-bot`, `suspected-bot`, `human`, `unknown` |
42
+ | `certain` | `true` to require proof |
43
+ | `botClass` | one or several classes |
44
+ | `minScore` / `maxScore` | inclusive score band |
45
+
46
+ **Who is it?**
47
+
48
+ | Field | |
49
+ | ----- | - |
50
+ | `identity` | a named signature — `"gptbot"`, `"googlebot"` |
51
+ | `category` | `ai`, `search`, `seo`, `social`, `monitoring`, `feed`, `archive`, `security`, `library`, `headless`, `advertising` |
52
+ | `detector` | fired by a named detector — `"trap"`, `"probe-signature"` |
53
+
54
+ **What did they do?**
55
+
56
+ | Field | |
57
+ | ----- | - |
58
+ | `path` | string prefix, `RegExp`, or a list of either |
59
+ | `method` | `"POST"`, or a list |
60
+
61
+ **What have they done before?**
62
+
63
+ | Field | |
64
+ | ----- | - |
65
+ | `minPriorConfirmations` | proven a bot this many times in the window |
66
+ | `minUnsolvedChallenges` | issued this many challenges and finished none |
67
+
68
+ Every field present must match. An empty `match: {}` matches everything, which is how you
69
+ write a catch-all last rule.
70
+
71
+ ## Do this: Serif's policy
72
+
73
+ Take the table from [lesson 3](03-verdicts-and-scores.md) and write it out.
74
+
75
+ ```js
76
+ import { BotHandler, createFacts } from "@osqd/bothandlerjs";
77
+
78
+ const detector = new BotHandler({
79
+ rules: [
80
+ { id: "verified-crawler-allow", match: { verdict: "verified-bot" }, action: "allow",
81
+ reason: "Confirmed search or social crawler — the traffic Serif wants." },
82
+
83
+ { id: "impersonator-block", match: { botClass: "impersonator", certain: true }, action: "block",
84
+ reason: "Forged a verifiable crawler identity. Proven by DNS, not inferred." },
85
+
86
+ { id: "scanner-block", match: { botClass: "scanner", certain: true }, action: "block",
87
+ reason: "Self-identified security scanner." },
88
+
89
+ { id: "trap-block", match: { detector: "trap", certain: true }, action: "block",
90
+ reason: "Followed a link no person can reach." },
91
+
92
+ { id: "scraper-ratelimit", match: { botClass: "scraper" }, action: "rate-limit",
93
+ params: { limit: { max: 60, windowMs: 60_000 } },
94
+ reason: "Behaves like bulk extraction. Slowed rather than refused." },
95
+
96
+ { id: "http-client-challenge", match: { botClass: "http-client" }, action: "challenge",
97
+ reason: "Bare HTTP client. Challenged rather than blocked: it may be somebody's integration." },
98
+
99
+ { id: "declared-bot-tag", match: { botClass: "declared-bot" }, action: "tag",
100
+ reason: "Announced itself honestly. Tagged so the application can decide." },
101
+
102
+ { id: "suspected-tag", match: { verdict: "suspected-bot" }, action: "tag",
103
+ reason: "Some signal, not enough to act on." },
104
+ ],
105
+ });
106
+ ```
107
+
108
+ Run the lesson 3 cases through `decide` and check each lands on the rule you intended.
109
+
110
+ ## Order is the policy
111
+
112
+ Move `suspected-tag` to the top and every rule below it stops mattering for anything
113
+ suspected — including your scanner block, because a scanner is also suspected. Nothing
114
+ warns you; the policy simply does less than it reads.
115
+
116
+ Two habits that prevent it:
117
+
118
+ **Allows for traffic you want, first.** A verified crawler should be decided before
119
+ anything gets a chance to be clever about it.
120
+
121
+ **Specific before general.** `{ botClass: "scanner" }` before `{ verdict: "suspected-bot" }`,
122
+ always.
123
+
124
+ ## Your own rules run before a preset's
125
+
126
+ ```js
127
+ new BotHandler({
128
+ preset: "protect-content",
129
+ rules: [{ id: "partner-allow", match: { path: "/api/partner" }, action: "allow" }],
130
+ });
131
+ ```
132
+
133
+ `rules` are evaluated **before** the preset's, which makes this the clean way to add
134
+ exceptions without copying a preset. [Lesson 10](10-actions-and-presets.md) covers when to
135
+ adopt a preset wholesale instead.
136
+
137
+ ## Predicates, and what they cost
138
+
139
+ When the declarative fields cannot say it, `match` takes a function:
140
+
141
+ ```js
142
+ {
143
+ id: "checkout-under-attack",
144
+ match: (assessment, facts) => facts.path.startsWith("/checkout") && assessment.score > 50 && !assessment.certain,
145
+ action: "delay",
146
+ params: { delayMs: 500 },
147
+ }
148
+ ```
149
+
150
+ Two things it costs you.
151
+
152
+ **`robotsFromRules` cannot read it.** A predicate can be run but not asked which crawlers
153
+ it is about, so it is reported as `unreadable` rather than guessed at — see
154
+ [lesson 10](10-actions-and-presets.md).
155
+
156
+ **The dashboard cannot preview it usefully.** A declarative match can be explained; a
157
+ function can only be executed.
158
+
159
+ Reach for one when you need it, and prefer the fields when you do not.
160
+
161
+ ## Validate before you ship
162
+
163
+ ```js
164
+ import { validateRules } from "@osqd/bothandlerjs";
165
+ const problems = validateRules(myRules);
166
+ if (problems.length > 0) throw new Error(problems.join("\n"));
167
+ ```
168
+
169
+ It catches duplicate ids, unknown actions, a `redirect` with no `location`, a `custom` with
170
+ no registered handler, and score bands that can never match. `new BotHandler()` runs it for
171
+ you and reports through `onWarning`; calling it yourself turns a warning into a failing
172
+ test.
173
+
174
+ ## Exercise
175
+
176
+ Add two rules to Serif and place them correctly:
177
+
178
+ 1. `/checkout` should never be served to anything proven automated.
179
+ 2. A client that has been issued five challenges and finished none should be blocked.
180
+
181
+ <details>
182
+ <summary>Answer</summary>
183
+
184
+ ```js
185
+ rules: [
186
+ { id: "verified-crawler-allow", match: { verdict: "verified-bot" }, action: "allow" },
187
+
188
+ // Before the generic blocks: it is more specific, and it is about a path.
189
+ { id: "checkout-no-bots", match: { path: "/checkout", certain: true, botClass: ["http-client", "automation", "scanner", "impersonator", "declared-bot"] },
190
+ action: "block", reason: "Proven automation on the checkout." },
191
+
192
+ { id: "persistent-refusal", match: { minUnsolvedChallenges: 5 }, action: "block",
193
+ reason: "Issued five challenges, finished none." },
194
+
195
+ // …the rest as before
196
+ ]
197
+ ```
198
+
199
+ Two things to notice.
200
+
201
+ **`checkout-no-bots` lists classes rather than using bare `certain: true`** — which would
202
+ also match a proven *human*, and blocking a customer you just vouched for on your own
203
+ checkout is the worst possible outcome. This is the trap from lesson 3, in the place it
204
+ does most damage.
205
+
206
+ **`persistent-refusal` is a rule, not evidence.** One abandoned challenge is a person. What
207
+ five of them mean is a judgement about your traffic, so the library exposes the count and
208
+ lets you decide rather than deciding for you.
209
+ </details>
210
+
211
+ ## What you learned
212
+
213
+ - First match wins; order is the policy and nothing warns you when it is wrong
214
+ - `id` and `reason` are read by people and by tools — write them properly
215
+ - Twelve declarative fields, grouped by the question they answer
216
+ - Your `rules` run before a preset's, which is how to add exceptions cleanly
217
+ - A predicate costs you `robots.txt` generation and dashboard preview
218
+
219
+ ## Reference
220
+
221
+ - [Matching requests](../policy/rules.md) — every field in detail
222
+ - [Policy overview](../policy/index.md) — how a decision is reached
223
+
224
+ Next: [Actions and presets](10-actions-and-presets.md).
@@ -0,0 +1,216 @@
1
+ # Lesson 10 — Actions and presets
2
+
3
+ **Goal:** choose responses by what they cost somebody who did nothing wrong, then compare
4
+ your hand-written policy against the eight shipped ones.
5
+
6
+ ← [Course](index.md) · Prev: [Rules](09-rules.md) · Next: [The challenge](11-the-challenge.md)
7
+
8
+ ---
9
+
10
+ ## Ten actions, ranked by who they hurt
11
+
12
+ That ranking is the useful one. Every action here will eventually be applied to somebody it
13
+ should not have been, so the question that matters is what that costs them.
14
+
15
+ | Action | Terminal | Costs a person |
16
+ | ------ | -------- | -------------- |
17
+ | `allow` | | nothing |
18
+ | `log` | | nothing |
19
+ | `tag` | | nothing |
20
+ | `delay` | | a moment |
21
+ | `rate-limit` | | a retry, if they are fast |
22
+ | `challenge` | | seconds, and a working browser |
23
+ | `custom` | | whatever you wrote |
24
+ | `redirect` | ✓ | their destination |
25
+ | `block` | ✓ | the page |
26
+ | `drop` | ✓ | the page, with no explanation |
27
+
28
+ The three marked terminal are the ones [the guard](04-the-guard.md) will not let rest on a
29
+ guess.
30
+
31
+ ## The ones worth dwelling on
32
+
33
+ **`tag`** attaches the verdict to the **request** — `x-bot-verdict`, `x-bot-score`,
34
+ `x-bot-class`, `x-bot-certain`, `x-bot-reason` — so your own handlers can react without
35
+ re-running detection:
36
+
37
+ ```js
38
+ app.get("/search", (req, res) => {
39
+ const bot = req.headers["x-bot-verdict"] !== "unknown";
40
+ res.json(bot ? cachedResults() : personalisedResults(req.user));
41
+ });
42
+ ```
43
+
44
+ Response-side verdict headers are **off by default**, and should stay off: an `X-Bot-Score`
45
+ in the response is a live feedback signal for anyone tuning a scraper against you.
46
+
47
+ **`delay`** costs you almost nothing and costs a scraper everything, while a person notices
48
+ a pause and nothing else. Default 500 ms, capped at 10 s. It holds a connection open, so
49
+ prefer `rate-limit` under heavy concurrency.
50
+
51
+ **`rate-limit`** is the one action that **needs a shared store**. A limit of 100/minute
52
+ enforced independently by four replicas is a limit of 400/minute. [Lesson 14](14-scaling.md).
53
+
54
+ **`drop`** is the harshest thing here: the connection is destroyed with no response at all.
55
+ A person who hits it gets a browser error with nothing in it, cannot tell your site from an
56
+ outage, and has no way to contact you. **`block` with a `reason` is almost always better.**
57
+ Reserve `drop` for traffic you have already decided about.
58
+
59
+ ## Custom actions
60
+
61
+ ```js
62
+ import { BotHandler, defineHandler } from "@osqd/bothandlerjs";
63
+
64
+ const shadowBan = defineHandler({
65
+ id: "shadow-ban",
66
+ description: "Serves an empty result set rather than an error",
67
+ execute: ({ assessment }) => ({
68
+ kind: "respond",
69
+ status: 200,
70
+ headers: { "content-type": "application/json" },
71
+ body: JSON.stringify({ results: [] }),
72
+ }),
73
+ });
74
+
75
+ new BotHandler({
76
+ handlers: [shadowBan],
77
+ rules: [{ id: "shadow", match: { botClass: "scraper" }, action: "custom", params: { handler: "shadow-ban" } }],
78
+ });
79
+ ```
80
+
81
+ `execute` returns `{ kind: "continue" }` — optionally with `requestHeaders`,
82
+ `responseHeaders` or `delayMs` — or `{ kind: "respond", status, headers, body }`, or
83
+ `{ kind: "drop" }`. It may be async.
84
+
85
+ **The guard does not apply to custom handlers.** It cannot know what yours does, so a
86
+ handler that denies service is a decision you own entirely. That is worth reading twice: it
87
+ is the one door out of the guarantee, and it is unlocked from the inside.
88
+
89
+ ## The eight presets
90
+
91
+ A preset is not a black box. It is an array of the same rules you just wrote, exported and
92
+ readable:
93
+
94
+ ```js
95
+ import { protectContent } from "@osqd/bothandlerjs";
96
+ console.log(protectContent());
97
+ ```
98
+
99
+ | Preset | For | Terminal on |
100
+ | ------ | --- | ----------- |
101
+ | `monitor-only` | week one, always | nothing |
102
+ | `allow-crawlers` | publishers who want to be found | impersonators, scanners, traps |
103
+ | `protect-content` | a public content site | impersonators, scanners, traps |
104
+ | `decline-ai-training` | keeping search, declining trainers | + declared AI crawlers |
105
+ | `protect-data` | pricing, listings, inventory | + declared AI and SEO crawlers |
106
+ | `protect-api` | a JSON API | impersonators, scanners, traps |
107
+ | `protect-auth` | login, signup, checkout — **those routes only** | all proven automation |
108
+ | `under-attack` | during an incident, then off again | all proven automation |
109
+
110
+ ```js
111
+ new BotHandler({ preset: "protect-content" });
112
+ ```
113
+
114
+ ## Compare yours to theirs
115
+
116
+ Print `protectContent()` beside your lesson 9 rules. You will find they are close — and the
117
+ differences are worth understanding:
118
+
119
+ - It has a `cleared-human-allow` rule first, which yours lacks. You will add the `isHuman`
120
+ it depends on in [lesson 12](12-going-live.md).
121
+ - It challenges suspected traffic at `minScore: 70`, where yours only tags. That threshold
122
+ is the number to move first when tuning.
123
+
124
+ ## Three presets with a warning attached
125
+
126
+ **`protect-auth` must be mounted on auth routes only.** Site-wide it blocks your payment
127
+ webhooks, your own server-side renderer and every honest crawler — all proven automation,
128
+ which is exactly what it refuses. Correct on a login form, an outage anywhere else.
129
+
130
+ ```js
131
+ app.use(botHandler(siteDetector));
132
+ app.use("/login", botHandler(authDetector));
133
+ app.use("/checkout", botHandler(authDetector));
134
+ ```
135
+
136
+ **`protect-api` deliberately never challenges.** A proof of work is solved by a browser
137
+ running JavaScript, and an API client is not one — challenging your customers' integrations
138
+ breaks them while an attacker solves it once in headless Chrome. The escalation ladder there
139
+ is rate limiting.
140
+
141
+ **`under-attack` is temporary.** It challenges at `minScore: 40` and rate-limits
142
+ *everybody*, people included, because a uniform ceiling is the one mitigation that cannot
143
+ single anybody out. It is not DDoS protection — it runs after the connection is accepted —
144
+ and like `protect-auth` it refuses proven automation, so allowlist your webhooks *before*
145
+ you switch it on rather than during the incident.
146
+
147
+ ## robots.txt, generated from the policy
148
+
149
+ Declining a crawler and not saying so is the worst of both worlds: it keeps coming, and you
150
+ get no credit for having a policy.
151
+
152
+ ```js
153
+ import { declineAiTraining, robotsFromRules } from "@osqd/bothandlerjs";
154
+
155
+ const { robotsTxt, declined, served, unreadable } = robotsFromRules(declineAiTraining(), {
156
+ disallowPaths: ["/internal/export.csv"], // your trap paths belong here
157
+ sitemap: "https://serif.example/sitemap.xml",
158
+ });
159
+ ```
160
+
161
+ **Read `unreadable` before you publish.** It lists rules that could not be reflected — a
162
+ predicate match, or a rule scoped to a path (which a named `robots.txt` group cannot
163
+ express without turning the crawler away from the whole site). A generated file that
164
+ silently omits something you block is worse than none at all: it tells crawlers they are
165
+ welcome where they are not.
166
+
167
+ `served` is the other one to read: crawlers a later rule would decline but an earlier rule
168
+ serves. That is why `decline-ai-training` blocks the `ai` category yet leaves
169
+ `ChatGPT-User` out of the file — it is served by an earlier rule, and the file agrees with
170
+ the policy rather than contradicting it.
171
+
172
+ ## Exercise
173
+
174
+ Serif's business decision: keep search engines, decline model trainers, and say so.
175
+
176
+ <details>
177
+ <summary>Answer</summary>
178
+
179
+ ```js
180
+ import { BotHandler, declineAiTraining, robotsFromRules } from "@osqd/bothandlerjs";
181
+
182
+ const detector = new BotHandler({ preset: "decline-ai-training" });
183
+
184
+ const { robotsTxt, unreadable } = robotsFromRules(declineAiTraining(), {
185
+ disallowPaths: ["/internal/export.csv"],
186
+ sitemap: "https://serif.example/sitemap.xml",
187
+ });
188
+ if (unreadable.length > 0) console.warn("not reflected in robots.txt:", unreadable);
189
+
190
+ app.get("/robots.txt", (_req, res) => res.type("text/plain").send(robotsTxt));
191
+ ```
192
+
193
+ The preset serves `ChatGPT-User`, `PerplexityBot`, `OAI-SearchBot`, `ClaudeBot`,
194
+ `Mistral-AI` and `DuckDuckBot` — a crawler fetching one page because a person asked about
195
+ it is a citation, not a corpus — and blocks the rest of the `ai` category with a body that
196
+ says why.
197
+
198
+ **`robots.txt` is the primary mechanism, not the rules.** The crawlers named here honour
199
+ it; the rules are what happens to the ones that do not.
200
+ </details>
201
+
202
+ ## What you learned
203
+
204
+ - Rank actions by what they cost a person who did nothing wrong
205
+ - `tag` is the workhorse; `drop` is almost never right
206
+ - Custom handlers are outside the guard, deliberately
207
+ - Presets are readable arrays of rules, not black boxes
208
+ - Three presets carry warnings; read them before mounting
209
+ - Generate `robots.txt` from the policy, and read `unreadable` and `served`
210
+
211
+ ## Reference
212
+
213
+ - [Actions](../policy/actions.md) · [Presets](../policy/presets.md) · [robots.txt](../policy/robots.md)
214
+ - [Choosing a policy](../start/choosing-a-policy.md)
215
+
216
+ Next: [The challenge](11-the-challenge.md).
@@ -0,0 +1,227 @@
1
+ # Lesson 11 — The challenge
2
+
3
+ **Goal:** turn on proof of work, know exactly what it buys, and make sure it cannot become
4
+ a wall for somebody who did nothing wrong.
5
+
6
+ ← [Course](index.md) · Prev: [Actions and presets](10-actions-and-presets.md) · Next: [Going live](12-going-live.md)
7
+
8
+ ---
9
+
10
+ ## What it actually buys
11
+
12
+ The client must find a counter such that `SHA-256(nonce + ":" + counter)` begins with
13
+ `difficulty` zero bits. Verification is one hash; solving takes about `2^difficulty` of
14
+ them.
15
+
16
+ **It does not identify anyone and it does not prove a human is present.** A headless Chrome
17
+ solves it as readily as a person's phone, just paying for the CPU.
18
+
19
+ What it does is convert a scrape from free into merely cheap, and change the *shape* of the
20
+ attack: a stateless scraper pulling a million pages must now run a JavaScript engine and
21
+ burn CPU on every one. Often that is enough to make bulk extraction not worth doing, and it
22
+ costs a real visitor a fraction of a second, once.
23
+
24
+ That is precisely why it sits where it does on the [action ladder](10-actions-and-presets.md).
25
+ It is the heaviest thing the [guard](04-the-guard.md) will let a *probabilistic* verdict ask
26
+ for, because a client wrongly suspected can pass it on its own and carry on.
27
+
28
+ ## Do this
29
+
30
+ Remember lesson 4, where a rule asking to block got `tag` instead? That was because no
31
+ challenge was configured. Fix it:
32
+
33
+ ```js
34
+ import { BotHandler, createFacts } from "@osqd/bothandlerjs";
35
+
36
+ const detector = new BotHandler({
37
+ challenge: {
38
+ secrets: [process.env.SERIF_CHALLENGE_SECRET], // at least 32 characters
39
+ contactHtml: '<p>Locked out? Email <a href="mailto:help@serif.example">help@serif.example</a>.</p>',
40
+ },
41
+ rules: [
42
+ { id: "suspected-challenge", match: { verdict: "suspected-bot", minScore: 40 }, action: "challenge" },
43
+ ],
44
+ suspectThreshold: 40,
45
+ });
46
+
47
+ const spoof = createFacts({
48
+ method: "GET", url: "/books", ip: "203.0.113.55",
49
+ headers: { host: "serif.example", "user-agent": CHROME["user-agent"], accept: "*/*" },
50
+ });
51
+
52
+ const result = await detector.handle(spoof);
53
+ console.log(result.outcome.kind, result.outcome.status);
54
+ ```
55
+
56
+ ### Checkpoint
57
+
58
+ ```
59
+ respond 429
60
+ ```
61
+
62
+ `handle` does assess, decide and act in one call, and returns an `ActionOutcome` for an
63
+ adapter to apply. The interstitial is a `429` with a locked-down CSP — `default-src 'none'`
64
+ and a nonce for its own script. No external resource of any kind appears on it.
65
+
66
+ ## The secret has no default, on purpose
67
+
68
+ A library-supplied fallback secret is a library-supplied forgery key, and it would end up in
69
+ production somewhere. Without `secrets`, a rule asking for a challenge degrades to `tag` and
70
+ says so through `onWarning`. Secrets must be at least 32 characters; shorter throws at
71
+ construction.
72
+
73
+ Rotating: the **first** secret signs, **all** of them verify. Prepend a new one, keep the
74
+ old for a token lifetime, and nobody is logged out.
75
+
76
+ ```js
77
+ challenge: { secrets: [NEW_SECRET, PREVIOUS_SECRET] }
78
+ ```
79
+
80
+ ## `contactHtml` is the most valuable line here
81
+
82
+ Everyone who sees the no-JavaScript fallback is a person your site just turned away: no
83
+ JavaScript, no WebCrypto, or a device too slow to finish. Put a real support address, a
84
+ phone number, or a link to a form there.
85
+
86
+ The default text is honest but generic. Yours can name a human.
87
+
88
+ ## Difficulty
89
+
90
+ In *bits*, so each step doubles the work. Default 16 — about 65,000 hashes, tens of
91
+ milliseconds in a modern browser. Refused above 24.
92
+
93
+ Past about 20 you are charging real people a visible delay, and **the oldest and slowest
94
+ devices pay the most** — which disproportionately means the users least able to replace
95
+ them. Raise it during an incident, not as a posture.
96
+
97
+ ## What a clearance proves
98
+
99
+ Solving grants a signed cookie that the `clearance` detector reads afterwards. It carries a
100
+ **level**, and the level is what a rule should key on:
101
+
102
+ | Level | Demonstrated | Certainty |
103
+ | ----- | ------------ | --------- |
104
+ | `pow` | ran JavaScript, has WebCrypto, spent CPU | not conclusive |
105
+ | `interaction` | a trusted input event was observed | stronger, still forgeable by a driven browser |
106
+ | `operator` | **your application** said this is a person | `certain` |
107
+
108
+ Only `operator` is conclusive, because that assertion comes from you rather than from the
109
+ client. You grant it yourself:
110
+
111
+ ```js
112
+ detector.grantClearance(facts, "operator"); // e.g. just after a successful sign-in
113
+ ```
114
+
115
+ That is the only conclusive human signal that exists anywhere in this library.
116
+
117
+ ## The one piece of state
118
+
119
+ The lifecycle is deliberately stateless until the moment of success: a challenge is a signed
120
+ blob the client carries, so a flood of unsolved challenges costs nothing but the bytes to
121
+ send them.
122
+
123
+ Exactly one thing is written, at the one moment it is indispensable: the solved nonce is
124
+ **claimed atomically**, so a solution cannot be replayed. That claim lives in the store —
125
+ with the default in-memory one and several replicas, a scraper retries a solved nonce
126
+ against other instances until one has not seen it. [Lesson 14](14-scaling.md).
127
+
128
+ ## Two refusals
129
+
130
+ **Challenging an actor that already holds valid clearance** is refused and warned about.
131
+ Passing a challenge cannot change a proven verdict, so re-issuing would loop for ever.
132
+
133
+ **Challenging on an API** breaks your customers' integrations and stops nobody — see
134
+ `protect-api` in [lesson 10](10-actions-and-presets.md).
135
+
136
+ ## Unsolved challenges as a signal
137
+
138
+ Every issued-and-never-solved challenge is counted on the [actor](07-actors.md):
139
+
140
+ ```js
141
+ { id: "persistent-refusal", match: { minUnsolvedChallenges: 5 }, action: "block" }
142
+ ```
143
+
144
+ It is **not evidence**, deliberately. One abandoned challenge is a person who changed their
145
+ mind; what five of them mean is a judgement about your traffic that only you can make.
146
+
147
+ ## Speaking the visitor's language
148
+
149
+ The interstitial is the only page this library shows to a member of the public, and they
150
+ are seeing it because a *probabilistic* verdict went against them. Somebody who cannot read
151
+ it cannot find the contact link on it either — which turns a check into a wall.
152
+
153
+ ```js
154
+ challenge: {
155
+ secrets: [SECRET],
156
+ contactHtml: "<p>…</p>",
157
+ translations: {
158
+ ja: { title: "ブラウザーを確認しています", message: "数秒で完了します。" },
159
+ de: { title: "Browser wird überprüft" },
160
+ "pt-BR": { title: "Verificando seu navegador" },
161
+ },
162
+ }
163
+ ```
164
+
165
+ The library ships **no translations and will not**: a machine-translated apology on a page
166
+ that just turned somebody away is worse than an honest English one, and only you know which
167
+ languages your audience reads. Anything a translation omits falls back to the default, so a
168
+ `title`-only entry is a fine first step.
169
+
170
+ Matching is exact tag first, then primary subtag — and it stops there. `pt-PT` is **not**
171
+ handed `pt-BR`. That looks unhelpful until you consider the case it protects: serving
172
+ Simplified Chinese to somebody who asked for Traditional is a worse failure than serving
173
+ English, and no rule can tell the two apart. Whether one regional variant stands in for
174
+ another is decided by which keys you write.
175
+
176
+ Each translation may set `lang`, which decides the voice a screen reader uses. Japanese
177
+ announced as `lang="en"` is unintelligible; getting the copy right and the attribute wrong
178
+ helps nobody.
179
+
180
+ ## Exercise
181
+
182
+ Serif serves readers in Britain, Brazil and Japan. Configure the challenge so that none of
183
+ them hits a wall, and say what happens to a Portuguese reader in Lisbon.
184
+
185
+ <details>
186
+ <summary>Answer</summary>
187
+
188
+ ```js
189
+ challenge: {
190
+ secrets: [process.env.SERIF_CHALLENGE_SECRET],
191
+ contactHtml: '<p>Locked out? Email <a href="mailto:help@serif.example">help@serif.example</a>.</p>',
192
+ translations: {
193
+ "pt-BR": { lang: "pt-BR", title: "Verificando seu navegador", message: "Isso leva alguns segundos.",
194
+ contactHtml: '<p>Problemas? <a href="mailto:help@serif.example">Fale conosco</a>.</p>' },
195
+ ja: { lang: "ja", title: "ブラウザーを確認しています", message: "数秒で完了します。",
196
+ contactHtml: '<p>お困りですか?<a href="mailto:help@serif.example">サポート</a>へご連絡ください。</p>' },
197
+ },
198
+ }
199
+ ```
200
+
201
+ **The Lisbon reader gets English.** `pt-PT` matches no exact key, and the primary-subtag
202
+ step looks for `pt`, which you did not supply — `pt-BR` is a regional variant and will not
203
+ stand in for another.
204
+
205
+ If Serif wants European Portuguese readers covered by the Brazilian copy, that is a
206
+ judgement about the audience, and it is made by **filing the copy under `pt`** rather than
207
+ `pt-BR`. The library will not make it for you.
208
+
209
+ Note that every translation carries its own `contactHtml`. A translated "checking your
210
+ browser" over an English "email us" is half a fix.
211
+ </details>
212
+
213
+ ## What you learned
214
+
215
+ - Proof of work imposes cost; it proves neither identity nor humanity
216
+ - The secret has no default because a default secret is a forgery key
217
+ - Difficulty is in bits, and the slowest devices pay most
218
+ - Only `operator` clearance is conclusive, and only you can grant it
219
+ - `minUnsolvedChallenges` is a rule, not evidence
220
+ - The interstitial is public-facing: give it a real contact and a language people read
221
+
222
+ ## Reference
223
+
224
+ - [The challenge](../challenge/index.md) · [Localisation](../challenge/localisation.md)
225
+ - [Actions](../policy/actions.md) — where `challenge` sits
226
+
227
+ Next: [Going live](12-going-live.md).