@osqd/bothandlerjs 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +516 -0
  2. package/LICENSE +102 -0
  3. package/README.md +308 -0
  4. package/bin/bothandlerjs.mjs +14 -0
  5. package/dist/actions/index.d.ts +22 -0
  6. package/dist/actions/types.d.ts +59 -0
  7. package/dist/adapters/fastify.d.ts +43 -0
  8. package/dist/adapters/fetch.d.ts +51 -0
  9. package/dist/adapters/index.cjs +723 -0
  10. package/dist/adapters/index.cjs.map +1 -0
  11. package/dist/adapters/index.d.ts +8 -0
  12. package/dist/adapters/index.js +692 -0
  13. package/dist/adapters/index.js.map +1 -0
  14. package/dist/adapters/koa.d.ts +27 -0
  15. package/dist/adapters/node.d.ts +29 -0
  16. package/dist/adapters/shared.d.ts +39 -0
  17. package/dist/audit.d.ts +172 -0
  18. package/dist/challenge/index.d.ts +176 -0
  19. package/dist/challenge/language.d.ts +57 -0
  20. package/dist/challenge/page.d.ts +42 -0
  21. package/dist/challenge/pow.d.ts +33 -0
  22. package/dist/challenge/token.d.ts +72 -0
  23. package/dist/cli.cjs +13388 -0
  24. package/dist/cli.cjs.map +1 -0
  25. package/dist/cli.d.ts +5 -0
  26. package/dist/cli.js +13358 -0
  27. package/dist/cli.js.map +1 -0
  28. package/dist/client/index.cjs +139 -0
  29. package/dist/client/index.cjs.map +1 -0
  30. package/dist/client/index.d.ts +63 -0
  31. package/dist/client/index.js +112 -0
  32. package/dist/client/index.js.map +1 -0
  33. package/dist/config.d.ts +316 -0
  34. package/dist/core.d.ts +443 -0
  35. package/dist/corpus/adversarial.d.ts +2 -0
  36. package/dist/corpus/advertising-email.d.ts +2 -0
  37. package/dist/corpus/ai-crawlers.d.ts +2 -0
  38. package/dist/corpus/benign-bots.d.ts +2 -0
  39. package/dist/corpus/cdn-gateways.d.ts +2 -0
  40. package/dist/corpus/cookies.d.ts +49 -0
  41. package/dist/corpus/crawlers-regional.d.ts +2 -0
  42. package/dist/corpus/crawlers-vertical.d.ts +2 -0
  43. package/dist/corpus/headers.d.ts +403 -0
  44. package/dist/corpus/humans-apps.d.ts +2 -0
  45. package/dist/corpus/humans-browsers.d.ts +2 -0
  46. package/dist/corpus/humans.d.ts +2 -0
  47. package/dist/corpus/index.cjs +4196 -0
  48. package/dist/corpus/index.cjs.map +1 -0
  49. package/dist/corpus/index.d.ts +37 -0
  50. package/dist/corpus/index.js +4132 -0
  51. package/dist/corpus/index.js.map +1 -0
  52. package/dist/corpus/infrastructure.d.ts +17 -0
  53. package/dist/corpus/libraries-extended.d.ts +2 -0
  54. package/dist/corpus/ranges.d.ts +31 -0
  55. package/dist/corpus/reputation.d.ts +21 -0
  56. package/dist/corpus/runner.d.ts +130 -0
  57. package/dist/corpus/schema.d.ts +185 -0
  58. package/dist/corpus/tooling.d.ts +2 -0
  59. package/dist/corpus/unwanted.d.ts +2 -0
  60. package/dist/crawler-ranges.d.ts +113 -0
  61. package/dist/dashboard/client.generated.d.ts +7 -0
  62. package/dist/dashboard/feed.d.ts +186 -0
  63. package/dist/dashboard/index.d.ts +5 -0
  64. package/dist/dashboard/page.d.ts +51 -0
  65. package/dist/dashboard/parse-request.d.ts +28 -0
  66. package/dist/dashboard/preview.d.ts +45 -0
  67. package/dist/dashboard/server.d.ts +70 -0
  68. package/dist/dashboard/types.d.ts +714 -0
  69. package/dist/detectors/accept-signature.d.ts +6 -0
  70. package/dist/detectors/browsing-coherence.d.ts +44 -0
  71. package/dist/detectors/cadence.d.ts +28 -0
  72. package/dist/detectors/clearance.d.ts +25 -0
  73. package/dist/detectors/client-hints.d.ts +19 -0
  74. package/dist/detectors/client-signals.d.ts +27 -0
  75. package/dist/detectors/crawl-breadth.d.ts +28 -0
  76. package/dist/detectors/crawler-verification.d.ts +39 -0
  77. package/dist/detectors/fetch-metadata.d.ts +20 -0
  78. package/dist/detectors/header-integrity.d.ts +26 -0
  79. package/dist/detectors/header-order.d.ts +27 -0
  80. package/dist/detectors/identity-rotation.d.ts +34 -0
  81. package/dist/detectors/index.d.ts +56 -0
  82. package/dist/detectors/ip-intelligence.d.ts +30 -0
  83. package/dist/detectors/known-bots.d.ts +94 -0
  84. package/dist/detectors/probe-signature.d.ts +50 -0
  85. package/dist/detectors/rate-anomaly.d.ts +36 -0
  86. package/dist/detectors/self-identified.d.ts +33 -0
  87. package/dist/detectors/session-integrity.d.ts +24 -0
  88. package/dist/detectors/tls-fingerprint.d.ts +34 -0
  89. package/dist/detectors/trap.d.ts +98 -0
  90. package/dist/detectors/types.d.ts +70 -0
  91. package/dist/detectors/ua-coherence.d.ts +32 -0
  92. package/dist/evidence.d.ts +45 -0
  93. package/dist/facts.d.ts +46 -0
  94. package/dist/index.cjs +9600 -0
  95. package/dist/index.cjs.map +1 -0
  96. package/dist/index.d.ts +63 -0
  97. package/dist/index.js +9444 -0
  98. package/dist/index.js.map +1 -0
  99. package/dist/internal/async.d.ts +16 -0
  100. package/dist/internal/clock.d.ts +17 -0
  101. package/dist/internal/crypto.d.ts +26 -0
  102. package/dist/internal/dns.d.ts +100 -0
  103. package/dist/internal/emitter.d.ts +18 -0
  104. package/dist/internal/http.d.ts +22 -0
  105. package/dist/internal/ip.d.ts +73 -0
  106. package/dist/internal/lru.d.ts +34 -0
  107. package/dist/internal/matcher.d.ts +41 -0
  108. package/dist/internal/pattern.d.ts +14 -0
  109. package/dist/internal/ua.d.ts +92 -0
  110. package/dist/metrics.d.ts +120 -0
  111. package/dist/notify/hub.d.ts +70 -0
  112. package/dist/notify/index.d.ts +7 -0
  113. package/dist/notify/redact.d.ts +48 -0
  114. package/dist/notify/sinks.d.ts +62 -0
  115. package/dist/notify/types.d.ts +56 -0
  116. package/dist/policy/index.d.ts +7 -0
  117. package/dist/policy/match.d.ts +20 -0
  118. package/dist/policy/policy.d.ts +123 -0
  119. package/dist/policy/presets.d.ts +161 -0
  120. package/dist/policy/types.d.ts +155 -0
  121. package/dist/robots.d.ts +83 -0
  122. package/dist/state.d.ts +161 -0
  123. package/dist/stores/index.d.ts +5 -0
  124. package/dist/stores/memory.d.ts +33 -0
  125. package/dist/stores/redis.d.ts +59 -0
  126. package/dist/stores/types.d.ts +37 -0
  127. package/dist/types.d.ts +250 -0
  128. package/docs/challenge/index.md +152 -0
  129. package/docs/challenge/localisation.md +105 -0
  130. package/docs/concepts/actors.md +113 -0
  131. package/docs/concepts/evidence.md +133 -0
  132. package/docs/concepts/the-guard.md +121 -0
  133. package/docs/concepts/threat-model.md +120 -0
  134. package/docs/concepts/verdicts.md +108 -0
  135. package/docs/course/01-first-assessment.md +178 -0
  136. package/docs/course/02-proof-and-suspicion.md +164 -0
  137. package/docs/course/03-verdicts-and-scores.md +160 -0
  138. package/docs/course/04-the-guard.md +200 -0
  139. package/docs/course/05-detectors.md +178 -0
  140. package/docs/course/06-identity.md +161 -0
  141. package/docs/course/07-actors.md +178 -0
  142. package/docs/course/08-traps.md +183 -0
  143. package/docs/course/09-rules.md +224 -0
  144. package/docs/course/10-actions-and-presets.md +216 -0
  145. package/docs/course/11-the-challenge.md +227 -0
  146. package/docs/course/12-going-live.md +211 -0
  147. package/docs/course/13-operating-it.md +238 -0
  148. package/docs/course/14-scaling.md +195 -0
  149. package/docs/course/15-extending.md +259 -0
  150. package/docs/course/16-proving-it.md +266 -0
  151. package/docs/course/index.md +112 -0
  152. package/docs/design/decisions.md +216 -0
  153. package/docs/detection/client-signals.md +96 -0
  154. package/docs/detection/detectors.md +376 -0
  155. package/docs/detection/index.md +134 -0
  156. package/docs/detection/signatures.md +111 -0
  157. package/docs/detection/verification.md +123 -0
  158. package/docs/detection/writing-a-detector.md +145 -0
  159. package/docs/index.md +100 -0
  160. package/docs/integration/adapters.md +138 -0
  161. package/docs/integration/client-ip.md +114 -0
  162. package/docs/integration/index.md +55 -0
  163. package/docs/integration/stores.md +76 -0
  164. package/docs/operations/audit.md +116 -0
  165. package/docs/operations/dashboard.md +725 -0
  166. package/docs/operations/index.md +101 -0
  167. package/docs/operations/metrics.md +74 -0
  168. package/docs/operations/notifications.md +72 -0
  169. package/docs/operations/runtime-changes.md +118 -0
  170. package/docs/policy/actions.md +196 -0
  171. package/docs/policy/index.md +84 -0
  172. package/docs/policy/presets.md +191 -0
  173. package/docs/policy/robots.md +128 -0
  174. package/docs/policy/rules.md +134 -0
  175. package/docs/reference/api.md +126 -0
  176. package/docs/reference/configuration.md +175 -0
  177. package/docs/start/choosing-a-policy.md +127 -0
  178. package/docs/start/first-integration.md +127 -0
  179. package/docs/start/installation.md +63 -0
  180. package/docs/start/upgrading.md +72 -0
  181. package/docs/testing/cli.md +119 -0
  182. package/docs/testing/corpus.md +129 -0
  183. package/docs/testing/index.md +61 -0
  184. package/docs/testing/replay.md +102 -0
  185. package/docs/testing/try-it.md +189 -0
  186. package/package.json +145 -0
@@ -0,0 +1,101 @@
1
+ # Operations
2
+
3
+ Knowing what the library is doing to your traffic, and changing it without a deploy.
4
+
5
+ ← [Documentation](../index.md)
6
+
7
+ ---
8
+
9
+ | Page | For |
10
+ | ---- | --- |
11
+ | **This page** — events and hooks | wiring the engine into your own logging, paging and queues |
12
+ | [The dashboard](dashboard.md) | seeing individual requests and the evidence behind them |
13
+ | [Metrics](metrics.md) | counters, histograms, Prometheus |
14
+ | [The audit](audit.md) | noticing that the traffic changed *shape* |
15
+ | [Notifications](notifications.md) | webhooks, Slack, batching and redaction |
16
+ | [Runtime changes](runtime-changes.md) | policy, guard, ranges and actors, live |
17
+
18
+ ---
19
+
20
+ ## Events and hooks
21
+
22
+ Everything the engine concludes is available as an event, in two forms that are the same
23
+ mechanism: a callback in the config, or a subscription you can add and remove.
24
+
25
+ ```ts
26
+ const detector = new BotHandler({
27
+ onDenial: ({ assessment, decision }) => audit.record(assessment.actor.key, decision.rule),
28
+ onDowngrade: ({ decision }) => pager.warn(`rule ${decision.rule} asked for more than its evidence`),
29
+ onAnomaly: (anomaly) => yourAlerting.send(anomaly.severity, anomaly.summary),
30
+ });
31
+
32
+ // Or later, and more than once, and removable:
33
+ const stop = detector.on("denial", ({ assessment }) => log.info({ actor: assessment.actor.key }));
34
+ stop();
35
+ ```
36
+
37
+ | Event | `onX` | Fires when |
38
+ | ----- | ----- | ---------- |
39
+ | `assessment` | `onAssessment` | Every assessment, including the ones that concluded nothing. The firehose. |
40
+ | `decision` | `onDecision` | Every decision, with the assessment behind it. |
41
+ | `denial` | `onDenial` | A request was actually denied — `block`, `drop` or `redirect`. |
42
+ | `downgrade` | `onDowngrade` | The [guard](../concepts/the-guard.md) replaced a terminal action. **The one worth paging on a rise in.** |
43
+ | `challenge` | `onChallenge` | A [challenge](../challenge/index.md) was issued, solved or rejected. |
44
+ | `detector-failure` | `onDetectorFailure` | A detector threw or timed out. Operational, not about traffic. |
45
+ | `policy-change` | `onPolicyChange` | The rule set was replaced at runtime. Your audit trail. |
46
+ | `guard-change` | `onGuardChange` | The guard settings changed at runtime, with the before and the after. **The other one worth paging on.** |
47
+ | `range-change` | `onRangeChange` | A range set was replaced — an allowlist entry added, a crawler's ranges refreshed. |
48
+ | `actor-change` | `onActorChange` | One [actor's](../concepts/actors.md) memory was forgotten, or it was cleared as human, by hand. |
49
+ | `anomaly` | `onAnomaly` | The [audit](audit.md) noticed the traffic change shape. |
50
+ | `warning` | `onWarning` | A misconfiguration, at startup or since. |
51
+ | `error` | `onError` | A detector, sink or store failed. |
52
+
53
+ ### Why `denial` and `downgrade` are separate events
54
+
55
+ They are *derived* rather than left for you to compute from `decision`, and that is
56
+ deliberate: "was this request refused" and "did the guard stop a rule" are the two
57
+ questions every integration asks, and three integrations deriving them separately is three
58
+ chances to disagree about the answer.
59
+
60
+ ### Why `guard-change` is separate from `policy-change`
61
+
62
+ For the same reason the dashboard gates them separately: "which rules exist" and "how far a
63
+ rule may go" are different powers with different consequences. A rule that overreaches is
64
+ stopped by the guard; a change to the guard is what decides whether anything stops it.
65
+
66
+ If you alert on one runtime change, alert on this one — and on `range-change`, because the
67
+ allowlist is the one list that stops detection *running*: an address on it is not judged
68
+ leniently, it is not judged at all.
69
+
70
+ ### `by` — who asked
71
+
72
+ All four runtime-change events carry **`by`**, when whatever made the change could say who
73
+ asked for it. Every mutating method takes a `{ by }` alongside its arguments, and the
74
+ dashboard fills it in from its own `auth`.
75
+
76
+ ```ts
77
+ detector.updateGuard({ falsePositivePolicy: "balanced" }, { by: "ada@example.com" });
78
+ // warning: Guard settings changed at runtime by ada@example.com: falsePositivePolicy strict → balanced.
79
+ ```
80
+
81
+ The library has no user model and does not want one; it carries the name it was given into
82
+ the warning and the event, so an audit trail can say *who* rather than only *what*. See
83
+ [runtime changes](runtime-changes.md).
84
+
85
+ ### None of them can hurt a request
86
+
87
+ A handler that throws is caught, reported once through `onError`, and the remaining
88
+ handlers still run — the same isolation the [detector pipeline](../detection/index.md)
89
+ gets. None is awaited: returning a promise is fine and its rejection is reported, but the
90
+ response never waits for your webhook.
91
+
92
+ That is the whole integration surface. A logger, a pager, a queue, a metrics client, a
93
+ webhook of your own — each is a function you pass in. Where you want batching, deduplication
94
+ and redaction as well, [the notification hub](notifications.md) already has them; where you
95
+ want the raw event, take it here.
96
+
97
+ ## Related
98
+
99
+ - [Metrics](metrics.md) · [The audit](audit.md) · [Notifications](notifications.md)
100
+ - [The dashboard](dashboard.md) — all of the above, rendered
101
+ - [Configuration reference](../reference/configuration.md)
@@ -0,0 +1,74 @@
1
+ # Metrics
2
+
3
+ Counters, histograms, and the two series worth alerting on.
4
+
5
+ ← [Documentation](../index.md) · [Operations](index.md)
6
+
7
+ ---
8
+
9
+ On by default; a handful of integer increments per request.
10
+
11
+ ```ts
12
+ detector.metrics(); // structured snapshot
13
+ detector.prometheus(); // Prometheus text exposition format
14
+ ```
15
+
16
+ ```ts
17
+ app.get("/internal/metrics", (_req, res) => res.type("text/plain").send(detector.prometheus()));
18
+ ```
19
+
20
+ **Serve it somewhere only your scraper can reach.** The detector-firing series describe how
21
+ detection behaves, which is exactly what someone tuning a scraper would like to read.
22
+
23
+ ## What is counted
24
+
25
+ Requests, verdicts, bot classes, actions, bypasses, detector firings and failures, the
26
+ challenge lifecycle, a latency histogram and a score histogram; plus a gauge for tracked
27
+ actors.
28
+
29
+ ## The two worth alerting on
30
+
31
+ **`bothandler_downgrades_total`** — rules that asked to deny service and were refused for
32
+ lack of proof. A rising count means your policy is asking for something the evidence does
33
+ not support. It is the [guard's](../concepts/the-guard.md) own report card, and the single
34
+ most informative number here.
35
+
36
+ **`bothandler_verdicts_total{verdict="unknown"}`** — ordinary traffic. If this falls,
37
+ either your traffic changed or your detection did, and you want to know which.
38
+
39
+ ## The one worth looking at *before* you move a threshold
40
+
41
+ **`bothandler_score_bucket`** — how suspicion is distributed across everything that was
42
+ scored, in ten buckets of ten points.
43
+
44
+ Proven assessments are counted by `bothandler_proven_total` instead: their score is 100 by
45
+ definition and decides nothing, so including them would put a spike at the top of the range
46
+ that means nothing. This is what the [dashboard's](dashboard.md) score distribution is drawn
47
+ from, which is why that chart and your alerting agree about how close ordinary traffic runs
48
+ to the line.
49
+
50
+ Moving `suspectThreshold` or a rule's `minScore` without looking at this is guessing. The
51
+ histogram tells you how many requests sit in the ten points you are about to cross.
52
+
53
+ ## Per-detector timing is available and off
54
+
55
+ Total assessment duration is always measured. Timing each detector separately costs two
56
+ clock reads per detector per request, which on a twenty-detector set is forty reads to
57
+ measure work usually counted in microseconds. Worth paying while you tune, not forever:
58
+
59
+ ```ts
60
+ new BotHandler({ metrics: { perDetectorTiming: true } });
61
+ ```
62
+
63
+ It fills `detectorTimings` in the snapshot, adds `bothandler_detector_duration_ms_sum` and
64
+ `_count` to the Prometheus output, and puts an average beside each detector on the
65
+ dashboard — which is how you find out that one `io` detector costs more than the other
66
+ nineteen together.
67
+
68
+ Switch metrics off entirely with `metrics: false`.
69
+
70
+ ## Related
71
+
72
+ - [The audit](audit.md) — because a counter cannot tell you a number is *unusual*
73
+ - [The guard](../concepts/the-guard.md) — what `bothandler_downgrades_total` is counting
74
+ - [The dashboard](dashboard.md) — these numbers, drawn
@@ -0,0 +1,72 @@
1
+ # Notifications
2
+
3
+ Webhooks, Slack and your own sinks — with a ceiling, because this fires under bot load.
4
+
5
+ ← [Documentation](../index.md) · [Operations](index.md)
6
+
7
+ ---
8
+
9
+ ```ts
10
+ notifications: {
11
+ sinks: [consoleNotifier(), slackNotifier({ url }), webhookNotifier({ url, secret })],
12
+ filter: { types: ["action", "downgrade", "error"], minScore: 70 },
13
+ redaction: { maskIp: true },
14
+ dedupeWindowMs: 60_000,
15
+ maxPerWindow: 200,
16
+ }
17
+ ```
18
+
19
+ ## The sinks that ship
20
+
21
+ | Sink | Notes |
22
+ | ---- | ----- |
23
+ | `consoleNotifier()` | development, and a reasonable default in a container |
24
+ | `webhookNotifier({ url, secret })` | HMAC-signed, with the timestamp *inside* the signed payload; retries with backoff; per-attempt timeout |
25
+ | `slackNotifier({ url })` | an incoming webhook |
26
+ | `notifyJsNotifier({ ... })` | the sibling [NotifyJS](https://github.com/OSQD-ts) hub |
27
+
28
+ Your own is one method — implement `Notifier`. If you want the raw event with no batching
29
+ at all, take it from [the events](index.md#events-and-hooks) instead.
30
+
31
+ ## The two properties that matter under load
32
+
33
+ Alerting is an asset or a liability depending on exactly these.
34
+
35
+ **It never blocks a request.** `emit` returns immediately and a wedged webhook slows
36
+ nothing. This is the same guarantee the event hooks give, for the same reason: nothing in
37
+ this library may put an unbounded await on the request path.
38
+
39
+ **It has a ceiling.** Repeats from one actor collapse within `dedupeWindowMs`; a global
40
+ `maxPerWindow` catches distributed traffic where every event is genuinely distinct; and the
41
+ **suppressed count is reported when the window rolls**, so a quiet channel is never mistaken
42
+ for quiet traffic.
43
+
44
+ Without both of those, the first real scrape either takes your site down or drowns the
45
+ channel you would have used to notice it.
46
+
47
+ ## Redaction
48
+
49
+ Addresses are masked to a `/24` or `/64` before an event leaves the process, and `Cookie`,
50
+ `Authorization` and friends are stripped unconditionally.
51
+
52
+ **Redaction runs on the way out**, so detection still sees everything. The engine needs the
53
+ full address to key an [actor](../concepts/actors.md) and match a range; your Slack channel
54
+ does not.
55
+
56
+ ## Filtering
57
+
58
+ `filter` decides what is worth sending at all — by event type, and by a score floor:
59
+
60
+ ```ts
61
+ filter: { types: ["denial", "downgrade", "anomaly"], minScore: 70 }
62
+ ```
63
+
64
+ A good starting set is exactly those three: something was refused, the guard stopped a rule,
65
+ or the [audit](audit.md) noticed a change. `assessment` is a firehose and belongs in a log,
66
+ not a chat channel.
67
+
68
+ ## Related
69
+
70
+ - [Events and hooks](index.md#events-and-hooks) — the raw, unbatched surface
71
+ - [The audit](audit.md) — the source of `anomaly` events
72
+ - [The dashboard](dashboard.md) — the notices panel these also land in
@@ -0,0 +1,118 @@
1
+ # Runtime changes
2
+
3
+ Policy, guard, ranges and actors — changed without a deploy, and recorded.
4
+
5
+ ← [Documentation](../index.md) · [Operations](index.md)
6
+
7
+ ---
8
+
9
+ ## Why any of this is mutable
10
+
11
+ Two things go stale between deploys, and both of them hurt.
12
+
13
+ **Published crawler ranges.** A stale list turns a verified crawler into an accused
14
+ impersonator — the address moved, the operator's file says so, and your copy does not.
15
+
16
+ **A rule set.** Sometimes a rule is wrong in a way you want to fix *now*, not at the next
17
+ deploy. That is exactly when the deploy pipeline feels longest.
18
+
19
+ Everything below is available as a method, as a [dashboard](dashboard.md) control behind its
20
+ own flag, and as an [event](index.md#events-and-hooks) so you have a trail.
21
+
22
+ ---
23
+
24
+ ## Rules
25
+
26
+ ```ts
27
+ detector.updatePolicy(
28
+ [...detector.policy.rules, { id: "allow-healthz", match: { path: "/healthz" }, action: "allow" }],
29
+ { by: "ada@example.com" },
30
+ );
31
+ ```
32
+
33
+ Validated first, swapped **atomically**, announced through `onWarning` and `policy-change`.
34
+ Invalid input throws and leaves the previous set standing.
35
+
36
+ **What it cannot change:** `falsePositivePolicy`, `fallbackAction` and
37
+ `terminalScoreThreshold`. No runtime edit can relax [the guard](../concepts/the-guard.md)
38
+ through this door.
39
+
40
+ ## The guard
41
+
42
+ Its own method, its own event, its own dashboard flag:
43
+
44
+ ```ts
45
+ detector.updateGuard({ falsePositivePolicy: "balanced" }, { by: "ada@example.com" });
46
+ // warning: Guard settings changed at runtime by ada@example.com: falsePositivePolicy strict → balanced.
47
+ ```
48
+
49
+ Separate from `updatePolicy` because "which rules exist" and "how far a rule may go" are
50
+ different powers. A rule that overreaches is stopped by the guard; a change to the guard is
51
+ what decides whether anything stops it. `onGuardChange` carries the before and the after.
52
+
53
+ If you page on one thing in this document, page on this.
54
+
55
+ ## Ranges
56
+
57
+ ```ts
58
+ detector.updateCrawlerRanges("gptbot", await fetchOpenAiRanges());
59
+ detector.updateRanges("datacenter", ranges);
60
+ detector.listRanges(); // [{ name: "crawler:gptbot", size: 42 }, ...]
61
+ ```
62
+
63
+ Parsed and validated before replacing the old set; invalid input throws while the previous
64
+ set stands, because a range set that silently matches nothing is worse than a stale one.
65
+
66
+ Two refusals specific to *crawler* ranges — a block wider than any crawler owns, and an
67
+ empty list — are covered in [verification](../detection/verification.md), because those
68
+ ranges do not merely describe a crawler, they verify one.
69
+
70
+ `range-change` is worth alerting on: the allowlist is the one list that stops detection
71
+ *running*. An address on it is not judged leniently, it is not judged at all.
72
+
73
+ ### Keeping them fresh automatically
74
+
75
+ ```ts
76
+ import { startCrawlerRangeRefresh } from "@osqd/bothandlerjs";
77
+ const stop = startCrawlerRangeRefresh(botHandler); // twice a day, by default
78
+ ```
79
+
80
+ Opt-in, because it makes outbound requests and a dependency-free package quietly fetching
81
+ URLs on a timer is not something to inherit by accident. It fails open, per source.
82
+
83
+ ## Actors
84
+
85
+ ```ts
86
+ detector.forgetActor(key, { by: "ada@example.com" });
87
+ detector.clearActor(key, 60 * 60_000, { by: "ada@example.com" });
88
+ ```
89
+
90
+ The support-ticket path: somebody is being challenged, you have looked at their requests,
91
+ and you are satisfied. `forgetActor` drops what is remembered about them; `clearActor`
92
+ exempts them for a stated number of milliseconds, so the exemption expires on its own rather
93
+ than becoming a permanent hole nobody remembers opening. Both emit `actor-change`. See [actors](../concepts/actors.md).
94
+
95
+ ---
96
+
97
+ ## `by`, and the audit trail
98
+
99
+ Every mutating method takes `{ by }`. The dashboard fills it in from its own `auth` — a
100
+ basic credential names itself, and a custom `authorize` can return an identity instead of
101
+ `true`.
102
+
103
+ The library has no user model and does not want one. It carries the name it was given into
104
+ the warning and the event, so your trail can say *who* rather than only *what*:
105
+
106
+ ```ts
107
+ detector.on("policy-change", ({ by, rules }) => auditLog.write({ who: by, count: rules.length }));
108
+ ```
109
+
110
+ Nothing verifies the name. It is as trustworthy as whatever supplied it, which is your
111
+ authentication — the same thing that decides whether the change is allowed at all.
112
+
113
+ ## Related
114
+
115
+ - [The dashboard](dashboard.md) — every one of these as a control, each behind its own flag
116
+ - [The guard](../concepts/the-guard.md) — the settings `updatePolicy` deliberately cannot reach
117
+ - [Verification](../detection/verification.md) — why crawler ranges are validated harder
118
+ - [Actors](../concepts/actors.md) — what `forgetActor` forgets
@@ -0,0 +1,196 @@
1
+ # Actions
2
+
3
+ All ten, ordered by what each costs a client that turns out to be a person.
4
+
5
+ ← [Documentation](../index.md) · [Policy](index.md)
6
+
7
+ ---
8
+
9
+ That ordering is the useful one. Every action here will eventually be applied to somebody
10
+ it should not have been applied to, so the question that matters is what that costs them.
11
+
12
+ | Action | Terminal | Costs a person |
13
+ | ------ | -------- | -------------- |
14
+ | [`allow`](#allow) | | nothing |
15
+ | [`log`](#log) | | nothing |
16
+ | [`tag`](#tag) | | nothing |
17
+ | [`delay`](#delay) | | a moment |
18
+ | [`rate-limit`](#rate-limit) | | a retry, if they are fast |
19
+ | [`challenge`](#challenge) | | seconds, and a working browser |
20
+ | [`custom`](#custom) | | whatever you wrote |
21
+ | [`redirect`](#redirect) | ✓ | their destination |
22
+ | [`block`](#block) | ✓ | the page |
23
+ | [`drop`](#drop) | ✓ | the page, with no explanation |
24
+
25
+ The three marked terminal are the ones [the guard](../concepts/the-guard.md) will not let
26
+ rest on a guess.
27
+
28
+ The engine never touches a response object. It returns an `ActionOutcome` and the
29
+ [adapter](../integration/adapters.md) applies it — which is why the same policy works
30
+ identically on Express, Fastify, Koa and a Fetch runtime.
31
+
32
+ ---
33
+
34
+ ## `allow`
35
+
36
+ Serve it. Explicitly, and that is not the same as having no rule: an `allow` above your
37
+ other rules is how you say "this is fine, stop asking".
38
+
39
+ ```ts
40
+ { id: "verified-crawler-allow", match: { verdict: "verified-bot" }, action: "allow",
41
+ reason: "Crawler identity confirmed against its operator's DNS or published ranges." }
42
+ ```
43
+
44
+ `params.headers` is honoured, so an allow can still carry something to the response.
45
+
46
+ ## `log`
47
+
48
+ Serve it, and record it. The whole of `monitor-only` is built from this.
49
+
50
+ ```ts
51
+ { id: "observe-suspected", match: { verdict: "suspected-bot" }, action: "log" }
52
+ ```
53
+
54
+ ## `tag`
55
+
56
+ Serve it, and attach the verdict to the **request** so your own handlers can react without
57
+ re-running detection: `x-bot-verdict`, `x-bot-score`, `x-bot-class`, `x-bot-certain`,
58
+ `x-bot-reason`.
59
+
60
+ ```ts
61
+ app.get("/search", (req, res) => {
62
+ const bot = req.headers["x-bot-verdict"] !== "unknown";
63
+ res.json(bot ? cachedResults() : personalisedResults(req.user));
64
+ });
65
+ ```
66
+
67
+ Response-side verdict headers are **off by default** (`exposeVerdictHeaders`). An
68
+ `X-Bot-Score` in the response is a live feedback signal for anyone tuning a scraper
69
+ against you.
70
+
71
+ ## `delay`
72
+
73
+ Serve it, slowly. `params.delayMs`, default 500 ms, capped at 10 s.
74
+
75
+ ```ts
76
+ { id: "slow-suspects", match: { verdict: "suspected-bot" }, action: "delay",
77
+ params: { delayMs: 1500 } }
78
+ ```
79
+
80
+ Cheap for you and expensive for anything making thousands of requests, while a person
81
+ notices a pause and nothing else. It holds a connection open, so it is a poor choice under
82
+ heavy concurrency — prefer `rate-limit` there.
83
+
84
+ ## `rate-limit`
85
+
86
+ A fixed window per [actor](../concepts/actors.md). Over the limit, the request is refused
87
+ with `429` and a `Retry-After`.
88
+
89
+ ```ts
90
+ { id: "scraper-ratelimit", match: { botClass: "scraper" }, action: "rate-limit",
91
+ params: { limit: { max: 60, windowMs: 60_000 } } }
92
+ ```
93
+
94
+ **This is the one action that needs a shared [store](../integration/stores.md).** A limit
95
+ of 100/minute enforced independently by four replicas is a limit of 400/minute. With the
96
+ default in-memory store it is per-instance, and the library will not pretend otherwise.
97
+
98
+ A fixed window rather than a sliding one: one round trip instead of several, and the extra
99
+ precision buys nothing for a mechanism whose job is to bound abuse rather than measure it.
100
+
101
+ ## `challenge`
102
+
103
+ Serve a proof-of-work interstitial. Solving it grants a signed clearance cookie; the
104
+ `clearance` detector reads it on subsequent requests.
105
+
106
+ ```ts
107
+ { id: "suspected-challenge", match: { verdict: "suspected-bot", minScore: 70 },
108
+ action: "challenge" }
109
+ ```
110
+
111
+ Needs `challenge.secrets` configured — without it, a rule asking for one degrades to a
112
+ `tag` and says so through `onWarning`. See [the challenge](../challenge/index.md) for what
113
+ it does and does not buy.
114
+
115
+ Challenging an actor that already holds valid clearance is refused and warned about:
116
+ passing a challenge cannot change a proven verdict, so re-issuing would loop for ever.
117
+
118
+ ## `custom`
119
+
120
+ Your own handler, registered by name.
121
+
122
+ ```ts
123
+ import { defineHandler } from "@osqd/bothandlerjs";
124
+
125
+ const shadowBan = defineHandler({
126
+ id: "shadow-ban",
127
+ description: "Serves an empty result set rather than an error",
128
+ execute: ({ assessment }) => ({
129
+ kind: "respond",
130
+ status: 200,
131
+ headers: { "content-type": "application/json" },
132
+ body: JSON.stringify({ results: [] }),
133
+ }),
134
+ });
135
+
136
+ new BotHandler({
137
+ handlers: [shadowBan],
138
+ rules: [{ id: "shadow", match: { botClass: "scraper" }, action: "custom",
139
+ params: { handler: "shadow-ban" } }],
140
+ });
141
+ ```
142
+
143
+ An `execute` returns one of three things: `{ kind: "continue" }` — optionally with
144
+ `requestHeaders`, `responseHeaders` or a `delayMs` — `{ kind: "respond", status, headers,
145
+ body }`, or `{ kind: "drop" }`. It may be async.
146
+
147
+ A handler named in a rule but not registered serves the request and warns. **The guard
148
+ does not apply to custom handlers** — it cannot know what yours does — so a handler that
149
+ denies service is a decision you own entirely.
150
+
151
+ ## `redirect`
152
+
153
+ Terminal. `params.location` is required; without it the request is served and a warning is
154
+ raised.
155
+
156
+ ```ts
157
+ { id: "bots-to-api", match: { certain: true, botClass: "scraper" }, action: "redirect",
158
+ params: { location: "/api/docs" } }
159
+ ```
160
+
161
+ Useful for sending automation somewhere it can be served cheaply and correctly — a data
162
+ endpoint, a sitemap, a licensing page — rather than simply refusing it.
163
+
164
+ ## `block`
165
+
166
+ Terminal. `403` by default with a plain-text body; `params.status` and `params.body`
167
+ override both.
168
+
169
+ ```ts
170
+ { id: "scanner-block", match: { botClass: "scanner", certain: true }, action: "block",
171
+ params: { status: 403, body: "Automated scanning is not served here." },
172
+ reason: "Proven scanner. Refused on proof, not on suspicion." }
173
+ ```
174
+
175
+ The body reaches a person often enough to be worth writing. Say what happened and how to
176
+ reach you.
177
+
178
+ ## `drop`
179
+
180
+ Terminal, and the harshest thing here: the connection is destroyed with no response at
181
+ all. What a port behind a dropping firewall looks like.
182
+
183
+ ```ts
184
+ { id: "denylisted", match: { detector: ["ip-intelligence"], certain: true }, action: "drop" }
185
+ ```
186
+
187
+ Reserve it for traffic you have decided about — a denylist entry, a confirmed attack. A
188
+ person who hits this gets a browser error with nothing in it, cannot tell your site from an
189
+ outage, and has no way to contact you. `block` with a `reason` is almost always the better
190
+ answer.
191
+
192
+ ## Related
193
+
194
+ - [The safety guard](../concepts/the-guard.md) — what happens when a guess asks for a terminal action
195
+ - [Presets](presets.md) — these actions, assembled
196
+ - [Adapters](../integration/adapters.md) — how an outcome becomes a response
@@ -0,0 +1,84 @@
1
+ # Policy
2
+
3
+ How an assessment becomes a decision.
4
+
5
+ ← [Documentation](../index.md)
6
+
7
+ ---
8
+
9
+ The engine's two halves are deliberately separate. `assess` says what a request *is*;
10
+ policy says what to *do* about it. Nothing in detection knows about actions, and nothing
11
+ in policy re-examines a request.
12
+
13
+ ```ts
14
+ const assessment = await detector.assess(facts); // detection
15
+ const decision = detector.decide(assessment); // policy — pure, no state, no I/O
16
+ ```
17
+
18
+ `decide` being pure is what makes the [replay](../testing/replay.md), the
19
+ [corpus](../testing/corpus.md) and the dashboard's policy preview possible: a candidate
20
+ policy can be run over recorded assessments as many times as you like, and nothing about
21
+ the running system moves.
22
+
23
+ ## Rules, in order
24
+
25
+ ```ts
26
+ new BotHandler({
27
+ rules: [
28
+ { id: "allow-verified", match: { verdict: "verified-bot" }, action: "allow" },
29
+ { id: "no-ai", match: { category: "ai" }, action: "block" },
30
+ { id: "slow-scrapers", match: { botClass: "scraper" }, action: "rate-limit",
31
+ params: { limit: { max: 60, windowMs: 60_000 } } },
32
+ ],
33
+ defaultAction: "allow",
34
+ });
35
+ ```
36
+
37
+ **First match wins.** Order is the whole semantics — a broad rule above a narrow one makes
38
+ the narrow one dead configuration, which is why the dashboard lists rule hit counts
39
+ *including the zeros* and why the rules in the editor are numbered and movable.
40
+
41
+ If nothing matches, `defaultAction` applies (default `allow`). Every decision names the
42
+ rule that produced it, including that one:
43
+
44
+ ```ts
45
+ {
46
+ action: "challenge",
47
+ rule: "suspected-challenge",
48
+ reason: "Several probabilistic signals agree. A challenge the client can pass on its own.",
49
+ params: {},
50
+ }
51
+ ```
52
+
53
+ ## Then the guard
54
+
55
+ Between the rule and the decision sits [the safety guard](../concepts/the-guard.md), which
56
+ refuses to let a terminal action rest on a guess. This is the mechanism the library is
57
+ built around and it is worth reading that page before writing rules that block anything.
58
+
59
+ A stopped decision carries what it was:
60
+
61
+ ```ts
62
+ { action: "challenge", downgradedFrom: "block", downgradeReason: "Strict mode permits …" }
63
+ ```
64
+
65
+ ## Reasons are part of the rule
66
+
67
+ ```ts
68
+ { id: "no-ai", match: { category: "ai" }, action: "block",
69
+ reason: "Not for model training. mailto:licensing@example.com" }
70
+ ```
71
+
72
+ `reason` appears in the decision, in your logs, on the dashboard, and — for a `block` —
73
+ can be served to the client. A refusal that explains itself and names a way to ask is the
74
+ difference between a policy and a wall. Somebody on the other end is often a person.
75
+
76
+ ## Where to go next
77
+
78
+ | | |
79
+ | --- | --- |
80
+ | [Matching requests](rules.md) | Every field a rule can match on. |
81
+ | [Actions](actions.md) | All ten, ordered by what each costs a person. |
82
+ | [Presets](presets.md) | Eight shipped policies; start from one. |
83
+ | [robots.txt](robots.md) | The file your policy implies. |
84
+ | [Runtime changes](../operations/runtime-changes.md) | Replacing rules without a deploy. |