@osqd/bothandlerjs 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +516 -0
  2. package/LICENSE +102 -0
  3. package/README.md +308 -0
  4. package/bin/bothandlerjs.mjs +14 -0
  5. package/dist/actions/index.d.ts +22 -0
  6. package/dist/actions/types.d.ts +59 -0
  7. package/dist/adapters/fastify.d.ts +43 -0
  8. package/dist/adapters/fetch.d.ts +51 -0
  9. package/dist/adapters/index.cjs +723 -0
  10. package/dist/adapters/index.cjs.map +1 -0
  11. package/dist/adapters/index.d.ts +8 -0
  12. package/dist/adapters/index.js +692 -0
  13. package/dist/adapters/index.js.map +1 -0
  14. package/dist/adapters/koa.d.ts +27 -0
  15. package/dist/adapters/node.d.ts +29 -0
  16. package/dist/adapters/shared.d.ts +39 -0
  17. package/dist/audit.d.ts +172 -0
  18. package/dist/challenge/index.d.ts +176 -0
  19. package/dist/challenge/language.d.ts +57 -0
  20. package/dist/challenge/page.d.ts +42 -0
  21. package/dist/challenge/pow.d.ts +33 -0
  22. package/dist/challenge/token.d.ts +72 -0
  23. package/dist/cli.cjs +13388 -0
  24. package/dist/cli.cjs.map +1 -0
  25. package/dist/cli.d.ts +5 -0
  26. package/dist/cli.js +13358 -0
  27. package/dist/cli.js.map +1 -0
  28. package/dist/client/index.cjs +139 -0
  29. package/dist/client/index.cjs.map +1 -0
  30. package/dist/client/index.d.ts +63 -0
  31. package/dist/client/index.js +112 -0
  32. package/dist/client/index.js.map +1 -0
  33. package/dist/config.d.ts +316 -0
  34. package/dist/core.d.ts +443 -0
  35. package/dist/corpus/adversarial.d.ts +2 -0
  36. package/dist/corpus/advertising-email.d.ts +2 -0
  37. package/dist/corpus/ai-crawlers.d.ts +2 -0
  38. package/dist/corpus/benign-bots.d.ts +2 -0
  39. package/dist/corpus/cdn-gateways.d.ts +2 -0
  40. package/dist/corpus/cookies.d.ts +49 -0
  41. package/dist/corpus/crawlers-regional.d.ts +2 -0
  42. package/dist/corpus/crawlers-vertical.d.ts +2 -0
  43. package/dist/corpus/headers.d.ts +403 -0
  44. package/dist/corpus/humans-apps.d.ts +2 -0
  45. package/dist/corpus/humans-browsers.d.ts +2 -0
  46. package/dist/corpus/humans.d.ts +2 -0
  47. package/dist/corpus/index.cjs +4196 -0
  48. package/dist/corpus/index.cjs.map +1 -0
  49. package/dist/corpus/index.d.ts +37 -0
  50. package/dist/corpus/index.js +4132 -0
  51. package/dist/corpus/index.js.map +1 -0
  52. package/dist/corpus/infrastructure.d.ts +17 -0
  53. package/dist/corpus/libraries-extended.d.ts +2 -0
  54. package/dist/corpus/ranges.d.ts +31 -0
  55. package/dist/corpus/reputation.d.ts +21 -0
  56. package/dist/corpus/runner.d.ts +130 -0
  57. package/dist/corpus/schema.d.ts +185 -0
  58. package/dist/corpus/tooling.d.ts +2 -0
  59. package/dist/corpus/unwanted.d.ts +2 -0
  60. package/dist/crawler-ranges.d.ts +113 -0
  61. package/dist/dashboard/client.generated.d.ts +7 -0
  62. package/dist/dashboard/feed.d.ts +186 -0
  63. package/dist/dashboard/index.d.ts +5 -0
  64. package/dist/dashboard/page.d.ts +51 -0
  65. package/dist/dashboard/parse-request.d.ts +28 -0
  66. package/dist/dashboard/preview.d.ts +45 -0
  67. package/dist/dashboard/server.d.ts +70 -0
  68. package/dist/dashboard/types.d.ts +714 -0
  69. package/dist/detectors/accept-signature.d.ts +6 -0
  70. package/dist/detectors/browsing-coherence.d.ts +44 -0
  71. package/dist/detectors/cadence.d.ts +28 -0
  72. package/dist/detectors/clearance.d.ts +25 -0
  73. package/dist/detectors/client-hints.d.ts +19 -0
  74. package/dist/detectors/client-signals.d.ts +27 -0
  75. package/dist/detectors/crawl-breadth.d.ts +28 -0
  76. package/dist/detectors/crawler-verification.d.ts +39 -0
  77. package/dist/detectors/fetch-metadata.d.ts +20 -0
  78. package/dist/detectors/header-integrity.d.ts +26 -0
  79. package/dist/detectors/header-order.d.ts +27 -0
  80. package/dist/detectors/identity-rotation.d.ts +34 -0
  81. package/dist/detectors/index.d.ts +56 -0
  82. package/dist/detectors/ip-intelligence.d.ts +30 -0
  83. package/dist/detectors/known-bots.d.ts +94 -0
  84. package/dist/detectors/probe-signature.d.ts +50 -0
  85. package/dist/detectors/rate-anomaly.d.ts +36 -0
  86. package/dist/detectors/self-identified.d.ts +33 -0
  87. package/dist/detectors/session-integrity.d.ts +24 -0
  88. package/dist/detectors/tls-fingerprint.d.ts +34 -0
  89. package/dist/detectors/trap.d.ts +98 -0
  90. package/dist/detectors/types.d.ts +70 -0
  91. package/dist/detectors/ua-coherence.d.ts +32 -0
  92. package/dist/evidence.d.ts +45 -0
  93. package/dist/facts.d.ts +46 -0
  94. package/dist/index.cjs +9600 -0
  95. package/dist/index.cjs.map +1 -0
  96. package/dist/index.d.ts +63 -0
  97. package/dist/index.js +9444 -0
  98. package/dist/index.js.map +1 -0
  99. package/dist/internal/async.d.ts +16 -0
  100. package/dist/internal/clock.d.ts +17 -0
  101. package/dist/internal/crypto.d.ts +26 -0
  102. package/dist/internal/dns.d.ts +100 -0
  103. package/dist/internal/emitter.d.ts +18 -0
  104. package/dist/internal/http.d.ts +22 -0
  105. package/dist/internal/ip.d.ts +73 -0
  106. package/dist/internal/lru.d.ts +34 -0
  107. package/dist/internal/matcher.d.ts +41 -0
  108. package/dist/internal/pattern.d.ts +14 -0
  109. package/dist/internal/ua.d.ts +92 -0
  110. package/dist/metrics.d.ts +120 -0
  111. package/dist/notify/hub.d.ts +70 -0
  112. package/dist/notify/index.d.ts +7 -0
  113. package/dist/notify/redact.d.ts +48 -0
  114. package/dist/notify/sinks.d.ts +62 -0
  115. package/dist/notify/types.d.ts +56 -0
  116. package/dist/policy/index.d.ts +7 -0
  117. package/dist/policy/match.d.ts +20 -0
  118. package/dist/policy/policy.d.ts +123 -0
  119. package/dist/policy/presets.d.ts +161 -0
  120. package/dist/policy/types.d.ts +155 -0
  121. package/dist/robots.d.ts +83 -0
  122. package/dist/state.d.ts +161 -0
  123. package/dist/stores/index.d.ts +5 -0
  124. package/dist/stores/memory.d.ts +33 -0
  125. package/dist/stores/redis.d.ts +59 -0
  126. package/dist/stores/types.d.ts +37 -0
  127. package/dist/types.d.ts +250 -0
  128. package/docs/challenge/index.md +152 -0
  129. package/docs/challenge/localisation.md +105 -0
  130. package/docs/concepts/actors.md +113 -0
  131. package/docs/concepts/evidence.md +133 -0
  132. package/docs/concepts/the-guard.md +121 -0
  133. package/docs/concepts/threat-model.md +120 -0
  134. package/docs/concepts/verdicts.md +108 -0
  135. package/docs/course/01-first-assessment.md +178 -0
  136. package/docs/course/02-proof-and-suspicion.md +164 -0
  137. package/docs/course/03-verdicts-and-scores.md +160 -0
  138. package/docs/course/04-the-guard.md +200 -0
  139. package/docs/course/05-detectors.md +178 -0
  140. package/docs/course/06-identity.md +161 -0
  141. package/docs/course/07-actors.md +178 -0
  142. package/docs/course/08-traps.md +183 -0
  143. package/docs/course/09-rules.md +224 -0
  144. package/docs/course/10-actions-and-presets.md +216 -0
  145. package/docs/course/11-the-challenge.md +227 -0
  146. package/docs/course/12-going-live.md +211 -0
  147. package/docs/course/13-operating-it.md +238 -0
  148. package/docs/course/14-scaling.md +195 -0
  149. package/docs/course/15-extending.md +259 -0
  150. package/docs/course/16-proving-it.md +266 -0
  151. package/docs/course/index.md +112 -0
  152. package/docs/design/decisions.md +216 -0
  153. package/docs/detection/client-signals.md +96 -0
  154. package/docs/detection/detectors.md +376 -0
  155. package/docs/detection/index.md +134 -0
  156. package/docs/detection/signatures.md +111 -0
  157. package/docs/detection/verification.md +123 -0
  158. package/docs/detection/writing-a-detector.md +145 -0
  159. package/docs/index.md +100 -0
  160. package/docs/integration/adapters.md +138 -0
  161. package/docs/integration/client-ip.md +114 -0
  162. package/docs/integration/index.md +55 -0
  163. package/docs/integration/stores.md +76 -0
  164. package/docs/operations/audit.md +116 -0
  165. package/docs/operations/dashboard.md +725 -0
  166. package/docs/operations/index.md +101 -0
  167. package/docs/operations/metrics.md +74 -0
  168. package/docs/operations/notifications.md +72 -0
  169. package/docs/operations/runtime-changes.md +118 -0
  170. package/docs/policy/actions.md +196 -0
  171. package/docs/policy/index.md +84 -0
  172. package/docs/policy/presets.md +191 -0
  173. package/docs/policy/robots.md +128 -0
  174. package/docs/policy/rules.md +134 -0
  175. package/docs/reference/api.md +126 -0
  176. package/docs/reference/configuration.md +175 -0
  177. package/docs/start/choosing-a-policy.md +127 -0
  178. package/docs/start/first-integration.md +127 -0
  179. package/docs/start/installation.md +63 -0
  180. package/docs/start/upgrading.md +72 -0
  181. package/docs/testing/cli.md +119 -0
  182. package/docs/testing/corpus.md +129 -0
  183. package/docs/testing/index.md +61 -0
  184. package/docs/testing/replay.md +102 -0
  185. package/docs/testing/try-it.md +189 -0
  186. package/package.json +145 -0
@@ -0,0 +1,83 @@
1
+ import type { BotCategory, BotSignature } from "./detectors/known-bots.js";
2
+ import type { Rule } from "./policy/types.js";
3
+ /**
4
+ * Generating `robots.txt` from what your policy actually does.
5
+ *
6
+ * Declining a crawler and not saying so is the worst of both worlds: the crawler
7
+ * keeps coming, wastes your bandwidth discovering it is unwelcome on every request,
8
+ * and you get no credit for having a policy. `robots.txt` is where you say it — and
9
+ * for the well-behaved crawlers, saying it is the *only* thing you need to do, since
10
+ * they will simply stop.
11
+ *
12
+ * Two things this file will not pretend. `robots.txt` is a request, not enforcement;
13
+ * everything that ignores it is exactly the population this library exists for. And a
14
+ * generated file can only reflect rules it can *read* — see {@link robotsFromRules}.
15
+ */
16
+ export interface RobotsOptions {
17
+ /** Signature ids to disallow entirely, e.g. `["gptbot", "ccbot"]`. */
18
+ disallowBots?: readonly string[];
19
+ /** Whole categories to disallow, e.g. `["ai", "seo"]`. */
20
+ disallowCategories?: readonly BotCategory[];
21
+ /** Paths disallowed for every crawler. Put your trap paths here. */
22
+ disallowPaths?: readonly string[];
23
+ /** Paths explicitly allowed for every crawler, evaluated ahead of the disallows. */
24
+ allowPaths?: readonly string[];
25
+ /** `Sitemap:` lines. Absolute URLs, per the specification. */
26
+ sitemap?: string | readonly string[];
27
+ /** `Crawl-delay:` in seconds for the wildcard group. Not honoured by every crawler. */
28
+ crawlDelay?: number;
29
+ /** Signature database to resolve ids and categories against. */
30
+ signatures?: readonly BotSignature[];
31
+ /** Lines added verbatim at the top, each already comment-prefixed if you want comments. */
32
+ header?: readonly string[];
33
+ }
34
+ /**
35
+ * Renders a `robots.txt`.
36
+ *
37
+ * Group ordering follows RFC 9309: a crawler obeys the most specific group that names
38
+ * it and ignores every other, so the per-agent groups must repeat any global path
39
+ * rules that should still apply to them. That is handled here — forgetting it is the
40
+ * classic way a `robots.txt` accidentally *un*-blocks a trap path for exactly the
41
+ * crawlers you were most careful about.
42
+ */
43
+ export declare function generateRobotsTxt(options?: RobotsOptions): string;
44
+ /** The `User-agent:` token for a signature. Matching is case-insensitive, so the token works either way. */
45
+ export declare function agentFor(signature: BotSignature): string;
46
+ export interface RobotsFromRulesResult {
47
+ robotsTxt: string;
48
+ /**
49
+ * Every signature id the generated file declines, with categories expanded to the
50
+ * crawlers they cover. Reporting only the explicitly-named ids would say "0
51
+ * declined" for a policy that turns away an entire category.
52
+ */
53
+ declined: string[];
54
+ /**
55
+ * Signature ids a later rule would have declined, but an earlier rule serves.
56
+ *
57
+ * First match wins, so these are *not* in the file. Worth reporting rather than
58
+ * silently omitting: "why is GPTBot in my robots.txt but ChatGPT-User is not" has an
59
+ * answer, and it is your own rule order.
60
+ */
61
+ served: string[];
62
+ /**
63
+ * Rules that could not be read, with the reason.
64
+ *
65
+ * A rule whose `match` is a predicate function is opaque — we can run it, but we
66
+ * cannot ask it which crawlers it is about. Those are listed rather than guessed
67
+ * at, because a `robots.txt` that silently omits something you block is worse than
68
+ * no generated file at all: it tells crawlers they are welcome where they are not.
69
+ */
70
+ unreadable: Array<{
71
+ rule: string;
72
+ reason: string;
73
+ }>;
74
+ }
75
+ /**
76
+ * Derives a `robots.txt` from the declarative rules in a policy.
77
+ *
78
+ * Reads every rule whose action denies service and whose `match` names an `identity`
79
+ * or a `category`, and declines exactly those crawlers. Rules expressed as predicate
80
+ * functions cannot be introspected and are reported in `unreadable` — check that list
81
+ * before publishing, and add anything it names by hand.
82
+ */
83
+ export declare function robotsFromRules(rules: readonly Rule[], options?: RobotsOptions): RobotsFromRulesResult;
@@ -0,0 +1,161 @@
1
+ import type { Clock } from "./internal/clock.js";
2
+ import type { ActorSnapshot, RequestFacts } from "./types.js";
3
+ /**
4
+ * The ceilings a detector's thresholds have to live under.
5
+ *
6
+ * Exported because they are not an implementation detail to anyone configuring a
7
+ * threshold: an actor's arrival count and distinct-User-Agent count both saturate
8
+ * here, so a threshold above them can never be reached and the detector is silently
9
+ * switched off. That is the failure this library refuses to ship elsewhere — a
10
+ * control that looks configured and is not — so the detectors check against these and
11
+ * refuse to be built that way.
12
+ */
13
+ export declare const MAX_TRACKED_ARRIVALS = 32;
14
+ export declare const MAX_TRACKED_PATHS = 64;
15
+ export declare const MAX_TRACKED_USER_AGENTS = 4;
16
+ export declare class ActorState {
17
+ readonly key: string;
18
+ readonly firstSeen: number;
19
+ lastSeen: number;
20
+ /** Total requests seen in this actor's lifetime within the registry window. */
21
+ total: number;
22
+ /** Assessments *this process* concluded were `confirmed-bot`. */
23
+ confirmations: number;
24
+ /**
25
+ * Confirmations other replicas had already recorded when this instance first saw the
26
+ * actor, or `undefined` when nothing shared them.
27
+ *
28
+ * Kept apart from the local count rather than added into it, so that the two facts
29
+ * stay distinguishable: what we saw, and what we were told. `priorConfirmations` is
30
+ * their sum, which is the number a rule means when it says
31
+ * `minPriorConfirmations: 1`.
32
+ */
33
+ sharedConfirmations: number | undefined;
34
+ /** Epoch ms until which a valid human clearance token stands. */
35
+ clearedUntil: number;
36
+ /**
37
+ * Challenges issued to this actor that no solution ever came back for.
38
+ *
39
+ * Incremented when one is issued and cleared when one is solved, so it is the number
40
+ * *outstanding* rather than a lifetime total: an actor challenged five times that
41
+ * solved the fifth reads zero, and one challenged five times that solved none reads
42
+ * five.
43
+ *
44
+ * The distinction is the whole signal. A person who abandons one challenge is
45
+ * ordinary — a slow phone, a lost tab, a change of mind. A client that has been asked
46
+ * five times and never once come back is not abandoning; it is unable or unwilling,
47
+ * and both of those are facts about software.
48
+ */
49
+ unsolvedChallenges: number;
50
+ private readonly timestamps;
51
+ private ringLength;
52
+ private ringNext;
53
+ private readonly paths;
54
+ private pathsOverflowed;
55
+ private pathsSaturatedAtTotal;
56
+ private readonly userAgents;
57
+ constructor(key: string, now: number);
58
+ /** Records an arrival. Called exactly once per request, by the engine. */
59
+ record(facts: RequestFacts): void;
60
+ /** Distinct paths seen, by hash. Saturates at {@link PATH_CAP}; `pathsSaturated` says whether it did. */
61
+ get distinctPaths(): number;
62
+ get pathsSaturated(): boolean;
63
+ /**
64
+ * Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
65
+ * still can. Over that many requests the distinct count is exact, so it is the only
66
+ * window in which a novelty ratio means anything.
67
+ */
68
+ get requestsWhenPathsSaturated(): number;
69
+ /** Distinct User-Agent strings. More than one from a single actor is unusual for a person. */
70
+ get distinctUserAgents(): number;
71
+ /** Milliseconds since the previous request, or `undefined` when this is the first. */
72
+ sinceLast(now: number): number | undefined;
73
+ /**
74
+ * Requests received in the last `windowMs`.
75
+ *
76
+ * Saturates at {@link TIMESTAMP_RING}: a very heavy actor is reported as exactly
77
+ * that number, not its true rate, and `ringSaturated` says so. That ceiling is
78
+ * deliberate. This series exists to *describe* an actor cheaply, and precise
79
+ * counting belongs to the rate-limit action, which uses the store and is exact.
80
+ */
81
+ requestsWithin(windowMs: number, now: number): number;
82
+ /** True when the ring is full, i.e. `requestsWithin` may be undercounting a heavy actor. */
83
+ get ringSaturated(): boolean;
84
+ /**
85
+ * Gaps between consecutive arrivals, oldest first. The cadence detector looks at
86
+ * the *variance* of these: a person's gaps are ragged, a `setInterval` loop's are
87
+ * not.
88
+ */
89
+ intervals(): number[];
90
+ /**
91
+ * Mean and dispersion of the arrival gaps, in one pass and without allocating.
92
+ *
93
+ * `intervals()` builds two arrays to answer the same question, and the cadence
94
+ * detector asks it on every request from every actor with enough history. Welford's
95
+ * algorithm gives the same numbers in a single walk of the ring with no garbage —
96
+ * and, incidentally, without the catastrophic cancellation that the naive
97
+ * sum-of-squares form suffers when gaps are large and their variance is small,
98
+ * which is precisely the machine-regular case this statistic exists to detect.
99
+ */
100
+ intervalStats(): {
101
+ count: number;
102
+ mean: number;
103
+ coefficientOfVariation: number;
104
+ };
105
+ snapshot(now: number): ActorSnapshot;
106
+ }
107
+ export interface ActorRegistryOptions {
108
+ /** How long an idle actor is remembered. Default 900000 (15 min). */
109
+ windowMs?: number;
110
+ /** Maximum actors tracked concurrently. Default 20000. See the cap arithmetic above. */
111
+ maxActors?: number;
112
+ }
113
+ /**
114
+ * One actor as a list can show it: the snapshot every rule reads, plus the two
115
+ * behavioural measures a person scanning for the worst offender actually wants.
116
+ */
117
+ export interface ActorSummary extends ActorSnapshot {
118
+ /** Requests in the last minute — a rate, where `requests` is a total. */
119
+ recentRate: number;
120
+ distinctUserAgents: number;
121
+ /** Regularity of the gaps between requests. Near zero is metronomic; `undefined` is too few gaps to say. */
122
+ cadenceCv: number | undefined;
123
+ }
124
+ /** Bounded LRU of {@link ActorState}, keyed by whatever `actorKey` produced. */
125
+ export declare class ActorRegistry {
126
+ private readonly clock;
127
+ private readonly actors;
128
+ constructor(clock: Clock, options?: ActorRegistryOptions);
129
+ get size(): number;
130
+ /** Fetches or creates the state for `key` and records this request against it. */
131
+ /**
132
+ * Called once for each actor this registry has not seen before.
133
+ *
134
+ * The hook exists so `BotHandler` can go and ask the store what other replicas know
135
+ * about a newcomer, without the registry itself learning what a store is. It fires
136
+ * exactly once per actor per instance — not once per request — which is the whole
137
+ * reason this is affordable: the thing `state.ts` refuses to do is a round trip on
138
+ * the request path, not a round trip ever.
139
+ */
140
+ onFirstSight?: (state: ActorState) => void;
141
+ observe(key: string, facts: RequestFacts): ActorState;
142
+ /** Reads without recording. Used by the clearance path, which must not inflate rates. */
143
+ peek(key: string): ActorState | undefined;
144
+ /**
145
+ * The busiest actors the registry is currently holding, most requests first.
146
+ *
147
+ * The registry knows about far more actors than any feed does — up to `maxActors`,
148
+ * each with its rate series, its path breadth and its confirmations — while a
149
+ * dashboard's ring holds a few hundred *requests*, which on a busy origin is a few
150
+ * seconds. "Who is hitting me hardest right now" is a question only this can answer,
151
+ * and until this method existed nothing could ask it.
152
+ *
153
+ * A read, and only a read: it neither records a request against an actor nor moves
154
+ * one up the LRU, so watching the list cannot change what it lists.
155
+ */
156
+ top(limit: number, now: number): ActorSummary[];
157
+ forget(key: string): void;
158
+ clear(): void;
159
+ /** Marks an actor as holding valid human clearance until `until`. */
160
+ clearUntil(key: string, until: number): void;
161
+ }
@@ -0,0 +1,5 @@
1
+ export type { BotHandlerStore } from "./types.js";
2
+ export { MemoryStore } from "./memory.js";
3
+ export type { MemoryStoreOptions } from "./memory.js";
4
+ export { RedisStore } from "./redis.js";
5
+ export type { RedisLike, RedisStoreOptions } from "./redis.js";
@@ -0,0 +1,33 @@
1
+ import type { Clock } from "../internal/clock.js";
2
+ import type { BotHandlerStore } from "./types.js";
3
+ export interface MemoryStoreOptions {
4
+ /** Maximum distinct counter keys held. Default 100000. */
5
+ maxCounters?: number;
6
+ /** Maximum single-use token keys held. Default 100000. */
7
+ maxTokens?: number;
8
+ /** Maximum general key/value entries held. Default 20000. */
9
+ maxValues?: number;
10
+ clock?: Clock;
11
+ }
12
+ /**
13
+ * In-process store. The default, and correct for a single instance.
14
+ *
15
+ * Every structure is a bounded LRU, so this cannot be grown without limit by a
16
+ * client sending unbounded distinct keys. That bounding has a consequence worth
17
+ * knowing: under heavy key churn an old single-use token can be evicted before it
18
+ * expires, and a replay of *that* token would then succeed. If replay resistance
19
+ * matters to you — and on more than one replica it definitely does — use
20
+ * {@link RedisStore}.
21
+ */
22
+ export declare class MemoryStore implements BotHandlerStore {
23
+ private readonly counters;
24
+ private readonly tokens;
25
+ private readonly values;
26
+ private readonly clock;
27
+ constructor(options?: MemoryStoreOptions);
28
+ increment(key: string, windowMs: number): Promise<number>;
29
+ consumeOnce(key: string, ttlMs: number): Promise<boolean>;
30
+ get(key: string): Promise<string | undefined>;
31
+ set(key: string, value: string, ttlMs: number): Promise<void>;
32
+ delete(key: string): Promise<void>;
33
+ }
@@ -0,0 +1,59 @@
1
+ import type { BotHandlerStore } from "./types.js";
2
+ import type { Clock } from "../internal/clock.js";
3
+ /**
4
+ * The handful of commands this store needs, described structurally.
5
+ *
6
+ * Declaring the shape rather than importing a client keeps `ioredis` and `node-redis`
7
+ * out of this library's dependency tree — you pass whichever you already run. Both
8
+ * satisfy this interface as-is.
9
+ */
10
+ export interface RedisLike {
11
+ incr(key: string): Promise<number>;
12
+ pexpire(key: string, ms: number): Promise<unknown>;
13
+ /**
14
+ * `SET key value PX ttl [NX]`.
15
+ *
16
+ * Both forms are needed and they are not interchangeable: `consumeOnce` requires
17
+ * `NX` (the atomic claim that makes replay protection work), while `set` must
18
+ * overwrite or it silently keeps the first value written under a key forever.
19
+ */
20
+ set(key: string, value: string, mode: "PX", ttl: number, condition?: "NX"): Promise<unknown>;
21
+ get(key: string): Promise<string | null>;
22
+ del(key: string): Promise<unknown>;
23
+ }
24
+ export interface RedisStoreOptions {
25
+ /** Prefix for every key. Default `"bh:"`. Change it if the instance is shared. */
26
+ prefix?: string;
27
+ /**
28
+ * Where the window boundary comes from. Defaults to the system clock.
29
+ *
30
+ * Present for the same reason every other stateful piece of this library takes one:
31
+ * a fixed window derived from `Date.now()` cannot be tested without sleeping, and a
32
+ * rate limit is exactly the thing you want to test at the boundary rather than
33
+ * near it. Pass the handler's own clock to keep one notion of time throughout.
34
+ */
35
+ clock?: Clock;
36
+ }
37
+ /**
38
+ * Redis-backed shared state, for deployments with more than one replica.
39
+ *
40
+ * This is the configuration in which single-use tokens and rate limits actually mean
41
+ * what they say — see {@link BotHandlerStore} for why per-instance memory silently
42
+ * weakens both.
43
+ *
44
+ * Note what is deliberately *not* here: nothing writes per-request behavioural data.
45
+ * Detection stays local and synchronous. A Redis round-trip happens only when a
46
+ * challenge is solved or a rate limit is consulted, so a slow or unavailable Redis
47
+ * costs you those two features and leaves everything else untouched.
48
+ */
49
+ export declare class RedisStore implements BotHandlerStore {
50
+ private readonly client;
51
+ private readonly prefix;
52
+ private readonly clock;
53
+ constructor(client: RedisLike, options?: RedisStoreOptions);
54
+ increment(key: string, windowMs: number): Promise<number>;
55
+ consumeOnce(key: string, ttlMs: number): Promise<boolean>;
56
+ get(key: string): Promise<string | undefined>;
57
+ set(key: string, value: string, ttlMs: number): Promise<void>;
58
+ delete(key: string): Promise<void>;
59
+ }
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Shared state, for the two things that genuinely need it across replicas.
3
+ *
4
+ * The library keeps its behavioural series in-process on purpose (see `state.ts`),
5
+ * so this interface is small by design. It exists for the cases where a per-instance
6
+ * answer is actually *wrong* rather than merely less precise:
7
+ *
8
+ * - **Single-use tokens.** A challenge solution accepted twice is a replay. With
9
+ * per-instance memory a scraper simply retries against another replica until it
10
+ * lands on one that has not seen the nonce, which defeats the mechanism entirely.
11
+ * - **Rate limiting.** A limit of 100/minute enforced independently by four replicas
12
+ * is a limit of 400/minute.
13
+ *
14
+ * Every method may reject. Callers treat a rejection as "no answer" and fail *open* —
15
+ * a store outage must degrade detection, never take down the site it protects.
16
+ */
17
+ export interface BotHandlerStore {
18
+ /**
19
+ * Adds one to a counter that expires `windowMs` after its first increment, and
20
+ * returns the new value. A fixed window, not a sliding one: it is one round trip
21
+ * instead of several, and the extra precision buys nothing for a mechanism whose
22
+ * job is to bound abuse rather than to measure it.
23
+ */
24
+ increment(key: string, windowMs: number): Promise<number>;
25
+ /**
26
+ * Atomically claims `key`. Returns `true` the first time and `false` for every
27
+ * subsequent call within `ttlMs`.
28
+ *
29
+ * Atomicity is the whole point — a `get` followed by a `set` has a window in which
30
+ * two concurrent replays both succeed. Implementations must use a single
31
+ * check-and-set operation.
32
+ */
33
+ consumeOnce(key: string, ttlMs: number): Promise<boolean>;
34
+ get(key: string): Promise<string | undefined>;
35
+ set(key: string, value: string, ttlMs: number): Promise<void>;
36
+ delete(key: string): Promise<void>;
37
+ }
@@ -0,0 +1,250 @@
1
+ /**
2
+ * Core vocabulary. Everything else in the library is written against these types.
3
+ *
4
+ * The single most important idea here is the split between **proof** and
5
+ * **suspicion**, expressed as {@link Certainty}. It is what lets the engine offer a
6
+ * real no-false-positive guarantee on its blocking path without pretending that bot
7
+ * detection is a solved problem: probabilistic signals exist, they are useful, and
8
+ * they are structurally barred from reaching a terminal action.
9
+ */
10
+ /** What kind of client we believe we are talking to. */
11
+ export type BotClass =
12
+ /** Positive evidence of a person driving a real browser. */
13
+ "human"
14
+ /** A crawler whose *identity* was confirmed, not merely claimed. See `verified-crawler`. */
15
+ | "verified-bot"
16
+ /** Announces itself as a bot in its User-Agent and does not pretend otherwise. */
17
+ | "declared-bot"
18
+ /** A real browser engine driven by automation (Puppeteer, Playwright, Selenium). */
19
+ | "automation"
20
+ /** A bare HTTP library or CLI: curl, wget, requests, axios, Go's http client. */
21
+ | "http-client"
22
+ /** Security/vulnerability scanning and fuzzing tooling. */
23
+ | "scanner"
24
+ /** Behaves like bulk content extraction regardless of what it claims to be. */
25
+ | "scraper"
26
+ /**
27
+ * Claims an identity it provably does not have — a forged `Googlebot`, a
28
+ * `Sec-CH-UA` that contradicts its own User-Agent. The most actionable class we
29
+ * produce, because the contradiction is self-evident from the request.
30
+ */
31
+ | "impersonator"
32
+ /** Nothing pointed either way with enough weight to name a class. */
33
+ | "unknown";
34
+ /**
35
+ * How much a piece of evidence is worth — and, critically, whether it is allowed to
36
+ * get a request blocked.
37
+ *
38
+ * - `certain` — **deterministic**. The request is self-contradictory, self-declaring,
39
+ * or its identity was verified against an authority outside the request itself.
40
+ * No legitimate human client produces it. Only this tier may drive a terminal
41
+ * action under the default policy, and every `certain` evidence must carry a
42
+ * `deterministicBasis` explaining why it cannot be wrong.
43
+ * - `strong` / `moderate` / `weak` — **probabilistic**. Real signals, genuinely
44
+ * useful in aggregate, but each has a population of legitimate clients that trips
45
+ * it: privacy browsers, corporate proxies, accessibility tooling, someone on a
46
+ * train tunnel. These accumulate into a score and can drive a challenge, a tag, or
47
+ * an alert. They cannot, by default, drive a block.
48
+ */
49
+ export type Certainty = "certain" | "strong" | "moderate" | "weak";
50
+ /** Which way a piece of evidence points. Human evidence actively rebuts bot evidence. */
51
+ export type EvidenceDirection = "bot" | "human";
52
+ /** One observation from one detector. Detectors return these; they never return verdicts. */
53
+ export interface Evidence {
54
+ /** Id of the detector that produced this. */
55
+ detector: string;
56
+ /** Human-readable, log-safe explanation of what was observed. */
57
+ summary: string;
58
+ direction: EvidenceDirection;
59
+ certainty: Certainty;
60
+ /**
61
+ * Contribution to the suspicion score, 0–1. Defaults to the weight implied by
62
+ * `certainty` ({@link CERTAINTY_WEIGHT}); override only to express that a signal is
63
+ * unusually weak or strong *for its tier*.
64
+ */
65
+ weight?: number | undefined;
66
+ /**
67
+ * A shared *root cause*, when this observation has one.
68
+ *
69
+ * Two pieces of evidence in the same family are not two reasons to be suspicious —
70
+ * they are one circumstance seen twice. A corporate proxy that strips
71
+ * `Sec-Fetch-*` also strips the Client Hints and the `Accept-Language`, and every
72
+ * detector that reasons from absence then fires at once about a single person
73
+ * behind a single appliance. Noisy-OR assumes independence; these are not
74
+ * independent, and treating them as if they were is how an ordinary employee
75
+ * accumulates a score of 95.
76
+ *
77
+ * {@link CERTAINTY_WEIGHT} scoring therefore takes the **strongest** observation in
78
+ * each family rather than compounding them. Leave it undefined when a signal really
79
+ * does stand on its own — every unfamilied piece of evidence is its own family, so
80
+ * the default is the old behaviour.
81
+ */
82
+ family?: string | undefined;
83
+ /** The class this evidence argues for, if it argues for one. */
84
+ botClass?: BotClass | undefined;
85
+ /** A concrete identity this evidence establishes or claims, e.g. `"googlebot"`. */
86
+ identity?: string | undefined;
87
+ /**
88
+ * Required on `certain` evidence: why this observation admits no benign
89
+ * explanation. Enforced at runtime in development (see {@link BotHandlerConfig.strictEvidence}).
90
+ * Writing one is a useful forcing function — if you cannot, your evidence is `strong`.
91
+ */
92
+ deterministicBasis?: string | undefined;
93
+ /** Structured detail for logs and dashboards. Must be JSON-serialisable. */
94
+ metadata?: Record<string, unknown> | undefined;
95
+ }
96
+ /**
97
+ * Every verdict and class, in a fixed order.
98
+ *
99
+ * Exported because three places need to enumerate them — the metrics counters, the
100
+ * dashboard's rule editor, and anything generating documentation — and three hand-kept
101
+ * copies is how a new verdict ends up missing from one of them.
102
+ */
103
+ export declare const VERDICTS: readonly Verdict[];
104
+ export declare const BOT_CLASSES: readonly BotClass[];
105
+ /** Score contribution implied by each certainty tier. */
106
+ export declare const CERTAINTY_WEIGHT: Record<Certainty, number>;
107
+ /**
108
+ * The engine's conclusion about a request.
109
+ *
110
+ * `confirmed-bot` and `verified-bot` are *proven*: they rest on at least one
111
+ * `certain` evidence. `suspected-bot` is a judgement call from accumulated
112
+ * probabilistic signal — treat it as "worth a challenge", never as "worth a 403".
113
+ */
114
+ export type Verdict =
115
+ /** Proven automated: self-declared, self-contradictory, or caught in a trap. */
116
+ "confirmed-bot"
117
+ /** Proven automated *and* proven to be who it says it is — Googlebot, Bingbot. */
118
+ | "verified-bot"
119
+ /** Probabilistic. Enough signal to act gently; not enough to be sure. */
120
+ | "suspected-bot"
121
+ /** Positive evidence of a person (valid attestation, real interaction, clearance token). */
122
+ | "human"
123
+ /** Nothing conclusive either way. The default resting state of ordinary traffic. */
124
+ | "unknown";
125
+ /** A request reduced to exactly the facts detectors are allowed to see. */
126
+ export interface RequestFacts {
127
+ /** Uppercase HTTP method. */
128
+ method: string;
129
+ /** Path only, no query string. Always begins with `/`. */
130
+ path: string;
131
+ /** Decoded query parameters. Null-prototype so `?__proto__=x` is visible, not swallowed. */
132
+ query: Record<string, string>;
133
+ /** Lowercased header names to values. Multi-value headers are joined with `, `. */
134
+ headers: Record<string, string | undefined>;
135
+ /**
136
+ * Header names in the order the client actually sent them, lowercased. Browsers
137
+ * emit a stable, engine-specific order; most tooling does not. Empty when the
138
+ * transport does not expose ordering (HTTP/2 pseudo-headers, some proxies).
139
+ */
140
+ headerOrder: readonly string[];
141
+ /** Resolved client IP. See {@link ProxyConfig} for how this is derived. */
142
+ ip: string;
143
+ /** Wall-clock arrival time, ms since epoch. Injected so tests can control it. */
144
+ timestamp: number;
145
+ /** Parsed request cookies, if the adapter supplied them. */
146
+ cookies?: Record<string, string> | undefined;
147
+ /** `https` when the connection is TLS-terminated at or before this server. */
148
+ protocol?: "http" | "https" | undefined;
149
+ /** Negotiated HTTP version, e.g. `"1.1"`, `"2.0"`. Used for header-order sanity. */
150
+ httpVersion?: string | undefined;
151
+ /**
152
+ * TLS client fingerprint (JA3/JA4), if your edge computes one and forwards it.
153
+ * Never computed here — Node does not expose the ClientHello.
154
+ */
155
+ tlsFingerprint?: string | undefined;
156
+ /**
157
+ * Set when the header set is known to be **incomplete** — the source could not
158
+ * supply everything the client actually sent.
159
+ *
160
+ * This is a statement about the observer, not the client, and it changes what an
161
+ * absent header means. Several detectors treat "no `Accept-Language`" as evidence,
162
+ * which is sound for a live request and nonsense for a line of an access log that
163
+ * only ever records two headers. With this set, every detector that reasons from
164
+ * absence stands down; detectors that reason from what *is* present carry on.
165
+ *
166
+ * Adapters leave it unset. Set it yourself when replaying logs, reading from an
167
+ * analytics pipeline, or on any transport that filters headers before you see them.
168
+ */
169
+ partialHeaders?: boolean | undefined;
170
+ /** Opaque per-request extras an adapter or your own code attaches. */
171
+ extra?: Record<string, unknown> | undefined;
172
+ }
173
+ /** What the engine knows about the actor behind a request, accumulated over time. */
174
+ export interface ActorSnapshot {
175
+ /** Stable key this actor is tracked under. See {@link BotHandlerConfig.actorKey}. */
176
+ key: string;
177
+ /** Requests seen from this actor inside the behavioural window. */
178
+ requests: number;
179
+ /** Distinct paths seen inside the window — breadth of crawl. */
180
+ distinctPaths: number;
181
+ /** First and last sighting, ms since epoch. */
182
+ firstSeen: number;
183
+ lastSeen: number;
184
+ /** Milliseconds since the previous request from this actor, or `undefined` if first. */
185
+ sinceLastMs?: number | undefined;
186
+ /** Assessments in the window that concluded `confirmed-bot`. */
187
+ priorConfirmations: number;
188
+ /**
189
+ * Challenges issued to this actor that were never solved.
190
+ *
191
+ * Outstanding rather than cumulative: solving one clears the count. See
192
+ * `MatchSpec.minUnsolvedChallenges` for the rule that reads it, and note that it is
193
+ * deliberately **not** evidence — a person who gives up on a challenge is a person,
194
+ * and what repeated abandonment means is a judgement about your traffic that only you
195
+ * can make.
196
+ */
197
+ unsolvedChallenges: number;
198
+ /** True when this actor holds a currently-valid human clearance token. */
199
+ cleared: boolean;
200
+ }
201
+ /** The engine's full, explainable output for one request. */
202
+ export interface Assessment {
203
+ /** Random per-request id, safe to log and to echo in a response header. */
204
+ requestId: string;
205
+ verdict: Verdict;
206
+ botClass: BotClass;
207
+ /** Established identity when there is one, e.g. `"googlebot"`. */
208
+ identity?: string | undefined;
209
+ /** Suspicion, 0–100. Derived only from probabilistic evidence. */
210
+ score: number;
211
+ /** How much to trust the verdict, 0–1. Exactly 1 when `certain` is true. */
212
+ confidence: number;
213
+ /**
214
+ * True when at least one unrebutted `certain` evidence fired. This flag — not the
215
+ * score — is what gates terminal actions in the default policy.
216
+ */
217
+ certain: boolean;
218
+ /** Every bot-pointing observation, strongest first. */
219
+ evidence: Evidence[];
220
+ /** Every human-pointing observation. These rebut and dampen the score. */
221
+ humanEvidence: Evidence[];
222
+ actor: ActorSnapshot;
223
+ /**
224
+ * Set when detection was skipped rather than performed. `undefined` means every
225
+ * configured detector actually ran, which is what distinguishes "we looked and
226
+ * found nothing" from "we never looked" — a distinction that matters enormously
227
+ * when reading a dashboard.
228
+ */
229
+ bypass?: BypassReason | undefined;
230
+ /** Total time spent in detection, ms. */
231
+ durationMs: number;
232
+ /** Detectors that threw or timed out. Detection continues without them. */
233
+ failures: DetectorFailure[];
234
+ facts: RequestFacts;
235
+ }
236
+ /** Why an assessment skipped detection. */
237
+ export type BypassReason =
238
+ /** The address matched the configured allowlist. */
239
+ "allowlist"
240
+ /** The path matched `ignorePaths`. */
241
+ | "ignored-path";
242
+ export interface DetectorFailure {
243
+ detector: string;
244
+ reason: "error" | "timeout";
245
+ message: string;
246
+ }
247
+ /** JSON-serialisable value, for metadata and notification payloads. */
248
+ export type Json = string | number | boolean | null | Json[] | {
249
+ [key: string]: Json;
250
+ };