@osqd/bothandlerjs 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +516 -0
  2. package/LICENSE +102 -0
  3. package/README.md +308 -0
  4. package/bin/bothandlerjs.mjs +14 -0
  5. package/dist/actions/index.d.ts +22 -0
  6. package/dist/actions/types.d.ts +59 -0
  7. package/dist/adapters/fastify.d.ts +43 -0
  8. package/dist/adapters/fetch.d.ts +51 -0
  9. package/dist/adapters/index.cjs +723 -0
  10. package/dist/adapters/index.cjs.map +1 -0
  11. package/dist/adapters/index.d.ts +8 -0
  12. package/dist/adapters/index.js +692 -0
  13. package/dist/adapters/index.js.map +1 -0
  14. package/dist/adapters/koa.d.ts +27 -0
  15. package/dist/adapters/node.d.ts +29 -0
  16. package/dist/adapters/shared.d.ts +39 -0
  17. package/dist/audit.d.ts +172 -0
  18. package/dist/challenge/index.d.ts +176 -0
  19. package/dist/challenge/language.d.ts +57 -0
  20. package/dist/challenge/page.d.ts +42 -0
  21. package/dist/challenge/pow.d.ts +33 -0
  22. package/dist/challenge/token.d.ts +72 -0
  23. package/dist/cli.cjs +13388 -0
  24. package/dist/cli.cjs.map +1 -0
  25. package/dist/cli.d.ts +5 -0
  26. package/dist/cli.js +13358 -0
  27. package/dist/cli.js.map +1 -0
  28. package/dist/client/index.cjs +139 -0
  29. package/dist/client/index.cjs.map +1 -0
  30. package/dist/client/index.d.ts +63 -0
  31. package/dist/client/index.js +112 -0
  32. package/dist/client/index.js.map +1 -0
  33. package/dist/config.d.ts +316 -0
  34. package/dist/core.d.ts +443 -0
  35. package/dist/corpus/adversarial.d.ts +2 -0
  36. package/dist/corpus/advertising-email.d.ts +2 -0
  37. package/dist/corpus/ai-crawlers.d.ts +2 -0
  38. package/dist/corpus/benign-bots.d.ts +2 -0
  39. package/dist/corpus/cdn-gateways.d.ts +2 -0
  40. package/dist/corpus/cookies.d.ts +49 -0
  41. package/dist/corpus/crawlers-regional.d.ts +2 -0
  42. package/dist/corpus/crawlers-vertical.d.ts +2 -0
  43. package/dist/corpus/headers.d.ts +403 -0
  44. package/dist/corpus/humans-apps.d.ts +2 -0
  45. package/dist/corpus/humans-browsers.d.ts +2 -0
  46. package/dist/corpus/humans.d.ts +2 -0
  47. package/dist/corpus/index.cjs +4196 -0
  48. package/dist/corpus/index.cjs.map +1 -0
  49. package/dist/corpus/index.d.ts +37 -0
  50. package/dist/corpus/index.js +4132 -0
  51. package/dist/corpus/index.js.map +1 -0
  52. package/dist/corpus/infrastructure.d.ts +17 -0
  53. package/dist/corpus/libraries-extended.d.ts +2 -0
  54. package/dist/corpus/ranges.d.ts +31 -0
  55. package/dist/corpus/reputation.d.ts +21 -0
  56. package/dist/corpus/runner.d.ts +130 -0
  57. package/dist/corpus/schema.d.ts +185 -0
  58. package/dist/corpus/tooling.d.ts +2 -0
  59. package/dist/corpus/unwanted.d.ts +2 -0
  60. package/dist/crawler-ranges.d.ts +113 -0
  61. package/dist/dashboard/client.generated.d.ts +7 -0
  62. package/dist/dashboard/feed.d.ts +186 -0
  63. package/dist/dashboard/index.d.ts +5 -0
  64. package/dist/dashboard/page.d.ts +51 -0
  65. package/dist/dashboard/parse-request.d.ts +28 -0
  66. package/dist/dashboard/preview.d.ts +45 -0
  67. package/dist/dashboard/server.d.ts +70 -0
  68. package/dist/dashboard/types.d.ts +714 -0
  69. package/dist/detectors/accept-signature.d.ts +6 -0
  70. package/dist/detectors/browsing-coherence.d.ts +44 -0
  71. package/dist/detectors/cadence.d.ts +28 -0
  72. package/dist/detectors/clearance.d.ts +25 -0
  73. package/dist/detectors/client-hints.d.ts +19 -0
  74. package/dist/detectors/client-signals.d.ts +27 -0
  75. package/dist/detectors/crawl-breadth.d.ts +28 -0
  76. package/dist/detectors/crawler-verification.d.ts +39 -0
  77. package/dist/detectors/fetch-metadata.d.ts +20 -0
  78. package/dist/detectors/header-integrity.d.ts +26 -0
  79. package/dist/detectors/header-order.d.ts +27 -0
  80. package/dist/detectors/identity-rotation.d.ts +34 -0
  81. package/dist/detectors/index.d.ts +56 -0
  82. package/dist/detectors/ip-intelligence.d.ts +30 -0
  83. package/dist/detectors/known-bots.d.ts +94 -0
  84. package/dist/detectors/probe-signature.d.ts +50 -0
  85. package/dist/detectors/rate-anomaly.d.ts +36 -0
  86. package/dist/detectors/self-identified.d.ts +33 -0
  87. package/dist/detectors/session-integrity.d.ts +24 -0
  88. package/dist/detectors/tls-fingerprint.d.ts +34 -0
  89. package/dist/detectors/trap.d.ts +98 -0
  90. package/dist/detectors/types.d.ts +70 -0
  91. package/dist/detectors/ua-coherence.d.ts +32 -0
  92. package/dist/evidence.d.ts +45 -0
  93. package/dist/facts.d.ts +46 -0
  94. package/dist/index.cjs +9600 -0
  95. package/dist/index.cjs.map +1 -0
  96. package/dist/index.d.ts +63 -0
  97. package/dist/index.js +9444 -0
  98. package/dist/index.js.map +1 -0
  99. package/dist/internal/async.d.ts +16 -0
  100. package/dist/internal/clock.d.ts +17 -0
  101. package/dist/internal/crypto.d.ts +26 -0
  102. package/dist/internal/dns.d.ts +100 -0
  103. package/dist/internal/emitter.d.ts +18 -0
  104. package/dist/internal/http.d.ts +22 -0
  105. package/dist/internal/ip.d.ts +73 -0
  106. package/dist/internal/lru.d.ts +34 -0
  107. package/dist/internal/matcher.d.ts +41 -0
  108. package/dist/internal/pattern.d.ts +14 -0
  109. package/dist/internal/ua.d.ts +92 -0
  110. package/dist/metrics.d.ts +120 -0
  111. package/dist/notify/hub.d.ts +70 -0
  112. package/dist/notify/index.d.ts +7 -0
  113. package/dist/notify/redact.d.ts +48 -0
  114. package/dist/notify/sinks.d.ts +62 -0
  115. package/dist/notify/types.d.ts +56 -0
  116. package/dist/policy/index.d.ts +7 -0
  117. package/dist/policy/match.d.ts +20 -0
  118. package/dist/policy/policy.d.ts +123 -0
  119. package/dist/policy/presets.d.ts +161 -0
  120. package/dist/policy/types.d.ts +155 -0
  121. package/dist/robots.d.ts +83 -0
  122. package/dist/state.d.ts +161 -0
  123. package/dist/stores/index.d.ts +5 -0
  124. package/dist/stores/memory.d.ts +33 -0
  125. package/dist/stores/redis.d.ts +59 -0
  126. package/dist/stores/types.d.ts +37 -0
  127. package/dist/types.d.ts +250 -0
  128. package/docs/challenge/index.md +152 -0
  129. package/docs/challenge/localisation.md +105 -0
  130. package/docs/concepts/actors.md +113 -0
  131. package/docs/concepts/evidence.md +133 -0
  132. package/docs/concepts/the-guard.md +121 -0
  133. package/docs/concepts/threat-model.md +120 -0
  134. package/docs/concepts/verdicts.md +108 -0
  135. package/docs/course/01-first-assessment.md +178 -0
  136. package/docs/course/02-proof-and-suspicion.md +164 -0
  137. package/docs/course/03-verdicts-and-scores.md +160 -0
  138. package/docs/course/04-the-guard.md +200 -0
  139. package/docs/course/05-detectors.md +178 -0
  140. package/docs/course/06-identity.md +161 -0
  141. package/docs/course/07-actors.md +178 -0
  142. package/docs/course/08-traps.md +183 -0
  143. package/docs/course/09-rules.md +224 -0
  144. package/docs/course/10-actions-and-presets.md +216 -0
  145. package/docs/course/11-the-challenge.md +227 -0
  146. package/docs/course/12-going-live.md +211 -0
  147. package/docs/course/13-operating-it.md +238 -0
  148. package/docs/course/14-scaling.md +195 -0
  149. package/docs/course/15-extending.md +259 -0
  150. package/docs/course/16-proving-it.md +266 -0
  151. package/docs/course/index.md +112 -0
  152. package/docs/design/decisions.md +216 -0
  153. package/docs/detection/client-signals.md +96 -0
  154. package/docs/detection/detectors.md +376 -0
  155. package/docs/detection/index.md +134 -0
  156. package/docs/detection/signatures.md +111 -0
  157. package/docs/detection/verification.md +123 -0
  158. package/docs/detection/writing-a-detector.md +145 -0
  159. package/docs/index.md +100 -0
  160. package/docs/integration/adapters.md +138 -0
  161. package/docs/integration/client-ip.md +114 -0
  162. package/docs/integration/index.md +55 -0
  163. package/docs/integration/stores.md +76 -0
  164. package/docs/operations/audit.md +116 -0
  165. package/docs/operations/dashboard.md +725 -0
  166. package/docs/operations/index.md +101 -0
  167. package/docs/operations/metrics.md +74 -0
  168. package/docs/operations/notifications.md +72 -0
  169. package/docs/operations/runtime-changes.md +118 -0
  170. package/docs/policy/actions.md +196 -0
  171. package/docs/policy/index.md +84 -0
  172. package/docs/policy/presets.md +191 -0
  173. package/docs/policy/robots.md +128 -0
  174. package/docs/policy/rules.md +134 -0
  175. package/docs/reference/api.md +126 -0
  176. package/docs/reference/configuration.md +175 -0
  177. package/docs/start/choosing-a-policy.md +127 -0
  178. package/docs/start/first-integration.md +127 -0
  179. package/docs/start/installation.md +63 -0
  180. package/docs/start/upgrading.md +72 -0
  181. package/docs/testing/cli.md +119 -0
  182. package/docs/testing/corpus.md +129 -0
  183. package/docs/testing/index.md +61 -0
  184. package/docs/testing/replay.md +102 -0
  185. package/docs/testing/try-it.md +189 -0
  186. package/package.json +145 -0
@@ -0,0 +1,6 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Reads the content-negotiation headers as a signature of the client.
4
+ *
5
+ * A browser asking for a page sends a long, specific `Accept` describing the
6
+ * document formats it renders. A scraper asks for `*/ export declare function acceptSignatureDetector(): Detector;
@@ -0,0 +1,44 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Evidence that this request came out of a browsing session.
4
+ *
5
+ * Almost every detector in this library argues in one direction. That asymmetry is a
6
+ * problem the scoring model makes visible: a person reading forty pages of
7
+ * documentation from a university's shared address accumulates `rate-anomaly`,
8
+ * `cadence`, `crawl-breadth` and `ip-intelligence` and can cross the suspicion
9
+ * threshold without a single thing being wrong with their request. The engine
10
+ * discounts a bot score by whatever human evidence it has — `pBot × (1 − pHuman)` —
11
+ * and until now the only things that produced any were your own application's
12
+ * assertion, a clearance token, a TLS profile table most people do not maintain, and
13
+ * a page script most people do not embed. For an ordinary request from an ordinary
14
+ * browser, `pHuman` was zero.
15
+ *
16
+ * So this detector reads the properties that come from a client having *state and
17
+ * history* rather than from a client being well-formed:
18
+ *
19
+ * - **A cache to revalidate.** `If-None-Match` and `If-Modified-Since` mean this
20
+ * client has been here before and kept what it was given. A stateless fetch loop
21
+ * has nothing to revalidate against and asks for the resource fresh every time.
22
+ * - **A cookie jar.** Something set state on this client and the client sent it back.
23
+ * - **A user gesture.** `Sec-Fetch-User: ?1` is a forbidden header the browser sets
24
+ * only when a *person* activated the navigation — a link click, a typed URL, a
25
+ * bookmark. Page script cannot set it, and a driver navigating a page
26
+ * programmatically does not produce it.
27
+ * - **A coherent modern fingerprint.** The Fetch Metadata group, the Client Hints and
28
+ * the negotiation headers all present and agreeing with each other.
29
+ *
30
+ * **None of this is proof and none of it ever will be.** Every one of these
31
+ * properties is copyable: a scraper that keeps a cookie jar, replays an ETag and
32
+ * copies a header set produces all four. That is why the ceiling here is `moderate`,
33
+ * why the whole set shares one {@link Evidence.family} so it can never stack into a
34
+ * large discount, and why it can only ever *reduce* a suspicion score rather than
35
+ * establish a `human` verdict on its own. The only conclusive human evidence in this
36
+ * library comes from you — `isHuman`, or an operator clearance token — because you
37
+ * are the one party in the exchange whose word the client cannot forge.
38
+ *
39
+ * What it buys is the honest half of a two-sided argument: a request that shows the
40
+ * marks of a real session is less suspicious than an identical one that does not,
41
+ * and the population that benefits is precisely the one this library exists to
42
+ * protect.
43
+ */
44
+ export declare function browsingCoherenceDetector(): Detector;
@@ -0,0 +1,28 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface CadenceOptions {
3
+ /** Minimum gaps needed before the statistic means anything. Default 6. */
4
+ minSamples?: number;
5
+ /** Coefficient of variation at or below which the rhythm is machine-regular. Default 0.15. */
6
+ regularityThreshold?: number;
7
+ /** Ignore actors whose mean gap exceeds this, ms. Default 120000 (2 min). */
8
+ maxMeanIntervalMs?: number;
9
+ }
10
+ /**
11
+ * Is this actor's *rhythm* human?
12
+ *
13
+ * People generate ragged inter-arrival times. They read, scroll, get distracted, open
14
+ * three tabs at once, then nothing for four minutes. A loop calling `setInterval` or
15
+ * awaiting a fixed delay produces gaps clustered tightly around one value, and the
16
+ * coefficient of variation makes that visible in a single number.
17
+ *
18
+ * The check is on *regularity*, not speed, which is what makes it complementary to
19
+ * `rate-anomaly` — a slow, polite scraper deliberately pacing itself at one request
20
+ * every two seconds to stay under a rate limit is invisible to rate counting and
21
+ * extremely visible here.
22
+ *
23
+ * It stays `moderate` because a real page can produce regular traffic too: a polling
24
+ * XHR, a video player fetching segments, an SSE reconnect loop. Those are your own
25
+ * frontend, and the fix is to exclude their paths (see `ignorePaths` in the engine
26
+ * config) rather than to weaken the statistic.
27
+ */
28
+ export declare function cadenceDetector(options?: CadenceOptions): Detector;
@@ -0,0 +1,25 @@
1
+ import type { Detector } from "./types.js";
2
+ import type { ChallengeService } from "../challenge/index.js";
3
+ /**
4
+ * Reads a clearance token the client already holds.
5
+ *
6
+ * This is the library's only source of *human*-pointing evidence that is not a
7
+ * guess about headers, and it is worth being exact about what each level earns,
8
+ * because overstating any of them would undo the whole design.
9
+ *
10
+ * - **`operator`** is `certain`. Your application told us this is a person — an
11
+ * authenticated session, a completed purchase, whatever your own bar is. We are
12
+ * believing you, not deducing anything, which is the same reasoning that makes a
13
+ * client's self-declaration `certain` in the other direction.
14
+ * - **`interaction`** is `strong`. A trusted input event was observed. Automation
15
+ * driving a real browser can synthesise something close, so this is very good
16
+ * evidence and not proof.
17
+ * - **`pow`** is only `moderate`, and this is the number people are most tempted to
18
+ * inflate. A solved proof of work shows a JavaScript engine ran and CPU was spent.
19
+ * A headless Chrome does both, happily and at scale. It raises the cost of a scrape
20
+ * substantially; it says nothing whatsoever about whether a human is present.
21
+ *
22
+ * The token is bound to the actor and signed, so it cannot be lifted from one client
23
+ * and replayed by another under a different actor key.
24
+ */
25
+ export declare function clearanceDetector(service: ChallengeService): Detector;
@@ -0,0 +1,19 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Cross-checks User-Agent Client Hints against the legacy User-Agent string.
4
+ *
5
+ * A real Chromium browser generates both from the same internal state, so they
6
+ * always agree. A client that rewrites one and forgets the other contradicts itself,
7
+ * and a contradiction is visible from a single request with no history, no state and
8
+ * no network call — which makes this one of the cheapest high-value checks there is.
9
+ *
10
+ * **Why none of this is `certain`, despite being a genuine contradiction.** The
11
+ * population that rewrites a User-Agent without touching Client Hints is not only
12
+ * scrapers: it is also every person running a UA-spoofing privacy extension, every
13
+ * enterprise browser with a rewritten UA policy, and every developer with device
14
+ * emulation open. Those are real people, and a `certain` tier that swept them up
15
+ * would make the word meaningless. So these observations score — sometimes heavily —
16
+ * and never block on their own. The lone exception is a hint that *self-declares*
17
+ * headless operation, which is a statement rather than an inference.
18
+ */
19
+ export declare function clientHintsDetector(): Detector;
@@ -0,0 +1,27 @@
1
+ import type { Detector, DetectionContext } from "./types.js";
2
+ export interface ClientSignalsOptions {
3
+ /**
4
+ * Where to find the signals on the request. Default: `facts.extra.clientSignals`.
5
+ *
6
+ * You are responsible for putting them there — typically by storing what your
7
+ * signal endpoint received against the session and attaching it in the adapter's
8
+ * `enrich` hook. The library does not invent a storage mechanism for you, because
9
+ * where per-session data lives is a decision only your application can make.
10
+ */
11
+ read?: (facts: DetectionContext["facts"]) => unknown;
12
+ }
13
+ /**
14
+ * Reads signals reported by the page script.
15
+ *
16
+ * The ceiling here is `moderate`, and it is a hard ceiling for a reason worth
17
+ * restating: every one of these values was produced by JavaScript running inside the
18
+ * client, which is the one place an adversary has complete control. A framework that
19
+ * wants `navigator.webdriver` to read `false` sets it to `false`, and everything this
20
+ * detector sees afterwards is whatever that framework decided to say.
21
+ *
22
+ * What it genuinely catches is automation that never bothered to hide — Selenium out
23
+ * of the box, a scripted Chrome someone pointed at your site this afternoon — which
24
+ * is a large share of real bot traffic. What it must never do is convince you that a
25
+ * clean report means a person.
26
+ */
27
+ export declare function clientSignalsDetector(options?: ClientSignalsOptions): Detector;
@@ -0,0 +1,28 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface CrawlBreadthOptions {
3
+ /**
4
+ * Distinct paths from one actor at or above which breadth is worth reporting.
5
+ * Default 30. Cannot exceed {@link MAX_TRACKED_PATHS}, which is where the count
6
+ * saturates; asking for more throws rather than never firing.
7
+ */
8
+ threshold?: number;
9
+ /** Fraction of requests that must be to a path not seen before. Default 0.85. */
10
+ noveltyRatio?: number;
11
+ /** Minimum requests before the ratio is meaningful. Default 20. */
12
+ minRequests?: number;
13
+ }
14
+ /**
15
+ * Is this actor *reading* the site or *enumerating* it?
16
+ *
17
+ * A person revisits. They land on an article, go back to the index, follow a related
18
+ * link, return to the article. Their ratio of distinct paths to total requests
19
+ * settles well below one. A crawler walking a sitemap almost never revisits, so its
20
+ * ratio sits near one and its distinct-path count climbs steadily.
21
+ *
22
+ * Worth being clear about what this cannot distinguish: a *welcome* crawler produces
23
+ * exactly this shape, and so does a person on a first visit to a documentation site
24
+ * clicking through the sidebar. It is a shape, not a motive — which is why it stays
25
+ * `weak` and why the interesting use is combining it with an actor that has already
26
+ * failed a header check.
27
+ */
28
+ export declare function crawlBreadthDetector(options?: CrawlBreadthOptions): Detector;
@@ -0,0 +1,39 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface CrawlerVerificationOptions {
3
+ /**
4
+ * Treat an address with no PTR record as a forged claim. Default true.
5
+ *
6
+ * Every operator whose crawler this applies to publishes PTR records precisely so
7
+ * that servers can check them; a claimed Googlebot with no reverse DNS at all is
8
+ * not a configuration accident, it is the cheapest possible forgery. Set false if
9
+ * you have a resolver that cannot be trusted to distinguish NXDOMAIN from failure.
10
+ */
11
+ treatMissingPtrAsForgery?: boolean;
12
+ /**
13
+ * Also assert forgery for `ip-ranges` crawlers when ranges *are* configured and the
14
+ * client is outside them. Default true. Has no effect for a crawler whose ranges
15
+ * you have not supplied — an unverifiable claim stays unverified, never accused.
16
+ */
17
+ useConfiguredRanges?: boolean;
18
+ }
19
+ /**
20
+ * Confirms or refutes a claimed crawler identity against an authority outside the
21
+ * request.
22
+ *
23
+ * This is the only detector that can produce `verified-bot`, and the only one that
24
+ * can produce a `certain` `impersonator`. Both directions matter:
25
+ *
26
+ * - **Confirmed** — the operator's own DNS vouches for this address. Forging it needs
27
+ * control of `googlebot.com`'s DNS, so a pass here is proof, and the default policy
28
+ * uses it to *allow*: your SEO does not deserve to be collateral damage from a bot
29
+ * rule.
30
+ * - **Refuted** — the client specifically claimed to be a named third party and the
31
+ * claim is false. Note how narrow that is. A privacy extension rewriting a UA to a
32
+ * generic browser string never lands here, because it never claims to be Googlebot.
33
+ * Only a deliberate forgery of a *verifiable* identity does, which is why this one
34
+ * is allowed to block.
35
+ *
36
+ * Anything short of a clear answer — a timeout, SERVFAIL, an unconfigured range list
37
+ * — yields nothing at all. Silence is never treated as an accusation.
38
+ */
39
+ export declare function crawlerVerificationDetector(options?: CrawlerVerificationOptions): Detector;
@@ -0,0 +1,20 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Checks the Fetch Metadata headers (`Sec-Fetch-Site`, `-Mode`, `-Dest`, `-User`).
4
+ *
5
+ * These are *forbidden* headers: page JavaScript cannot set or alter them, so in a
6
+ * real browser they are generated by the network stack from the actual context of
7
+ * the request. That makes them unusually hard to fake convincingly — a client
8
+ * assembling requests by hand either omits them or picks a combination the browser
9
+ * would never produce.
10
+ *
11
+ * Two failure modes are checked. **Absence** on an engine known to send them, over
12
+ * HTTPS, where they should be present. And **incoherence** — a `Sec-Fetch-Dest:
13
+ * document` that is not a navigation describes a request no browser makes.
14
+ *
15
+ * Neither is `certain`, for the same reason as elsewhere: a stripping proxy in front
16
+ * of your server produces the first, and an unusual-but-real client integration can
17
+ * produce the second. Together with a matching header set, though, they are among
18
+ * the sharpest probabilistic signals available.
19
+ */
20
+ export declare function fetchMetadataDetector(): Detector;
@@ -0,0 +1,26 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface HeaderIntegrityOptions {
3
+ /** Report a browser-claiming client that omits `Accept-Language`. Default true. */
4
+ checkAcceptLanguage?: boolean;
5
+ }
6
+ /**
7
+ * Does this request's header set match the client it claims to be?
8
+ *
9
+ * Browsers are extremely consistent about which headers they attach; HTTP libraries
10
+ * attach the minimum that gets a response. The gap between the two is one of the
11
+ * most reliable probabilistic signals available from a single request.
12
+ *
13
+ * Almost everything here is deliberately kept below `certain`, because headers pass
14
+ * through corporate proxies, CDNs, privacy extensions and mobile carrier
15
+ * transcoders, any of which will strip or rewrite them for an entirely real person.
16
+ *
17
+ * The exceptions are the three genuine protocol violations: a connection-specific
18
+ * header on HTTP/2 or HTTP/3, a message carrying both `Content-Length` and
19
+ * `Transfer-Encoding`, and a repeated `Host` or `Content-Length`. Each is a rule the
20
+ * specification requires a recipient to *enforce* rather than merely recommends, so a
21
+ * client that breaks it cannot get a response through any compliant proxy — and each
22
+ * is read from what the request *contains*, never from what it lacks. That second
23
+ * property is the one doing the work: an absence is indistinguishable from a facts
24
+ * source that dropped the header, and no absence in this file may reach `certain`.
25
+ */
26
+ export declare function headerIntegrityDetector(options?: HeaderIntegrityOptions): Detector;
@@ -0,0 +1,27 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface HeaderOrderOptions {
3
+ /** Include the order fingerprint in evidence metadata for cross-actor correlation. Default true. */
4
+ emitFingerprint?: boolean;
5
+ }
6
+ /**
7
+ * Reads the *order* in which headers arrived.
8
+ *
9
+ * Header order is a genuine fingerprint: a browser's network stack emits a fixed
10
+ * sequence that has nothing to do with what the page requested, and it is one of the
11
+ * few properties a scraper cannot fix by copying a User-Agent string. HTTP client
12
+ * libraries have their own, very different, and equally fixed orders.
13
+ *
14
+ * The reason this detector stays weak is structural rather than a lack of
15
+ * confidence: **anything between the client and this process may reorder headers.**
16
+ * HTTP/2 and HTTP/3 do not preserve a meaningful order at all, some CDNs normalise
17
+ * it, and some proxies rebuild the request wholesale. So it is checked only on
18
+ * HTTP/1.x, only when the transport actually exposed an order, and it contributes a
19
+ * nudge rather than a conclusion.
20
+ */
21
+ export declare function headerOrderDetector(options?: HeaderOrderOptions): Detector;
22
+ /**
23
+ * Stable fingerprint of a header order, for correlating one actor across rotating
24
+ * addresses. Exported because it is useful well outside this detector — an actor
25
+ * that changes IP every request but keeps the same order fingerprint is one actor.
26
+ */
27
+ export declare function headerOrderFingerprint(order: readonly string[]): string;
@@ -0,0 +1,34 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface IdentityRotationOptions {
3
+ /**
4
+ * Distinct User-Agents from one actor at or above which rotation is reported.
5
+ * Default 3.
6
+ *
7
+ * An actor remembers at most {@link MAX_TRACKED_USER_AGENTS} of them, so a threshold
8
+ * above that can never be reached — an actor cycling through thirty spoofed strings
9
+ * still reports four. Asking for more throws rather than producing a detector that
10
+ * runs on every request and can never fire.
11
+ */
12
+ threshold?: number;
13
+ /** Minimum requests before the count is meaningful. Default 10. */
14
+ minRequests?: number;
15
+ }
16
+ /**
17
+ * One actor, several identities.
18
+ *
19
+ * A single client does not change its User-Agent mid-session. Something that does is
20
+ * cycling through a spoofing list, which is behaviour with no innocent
21
+ * interpretation *for a single client*.
22
+ *
23
+ * **This detector is off by default, and you should think before enabling it.** With
24
+ * the default IP-based actor key, "one actor" routinely means "one NAT gateway", and
25
+ * a corporate office, a university, a coffee shop or a mobile carrier's CGNAT pool
26
+ * legitimately presents hundreds of distinct browsers behind one address. Under that
27
+ * key this detector fires on exactly the busiest legitimate networks on the internet.
28
+ *
29
+ * It becomes genuinely valuable once your `actorKey` identifies something narrower
30
+ * than an address — a session cookie, an authenticated user id, or an IP combined
31
+ * with a TLS fingerprint. Then a rotating User-Agent really is one client lying, and
32
+ * it is worth a lot. Enable it there, and only there.
33
+ */
34
+ export declare function identityRotationDetector(options?: IdentityRotationOptions): Detector;
@@ -0,0 +1,56 @@
1
+ import type { Detector } from "./types.js";
2
+ export type { DetectionContext, Detector, DetectorResult } from "./types.js";
3
+ export { evidence, absenceIsMeaningful } from "./types.js";
4
+ export { selfIdentifiedDetector } from "./self-identified.js";
5
+ export type { SelfIdentifiedOptions } from "./self-identified.js";
6
+ export { crawlerVerificationDetector } from "./crawler-verification.js";
7
+ export type { CrawlerVerificationOptions } from "./crawler-verification.js";
8
+ export { headerIntegrityDetector } from "./header-integrity.js";
9
+ export type { HeaderIntegrityOptions } from "./header-integrity.js";
10
+ export { clientHintsDetector } from "./client-hints.js";
11
+ export { fetchMetadataDetector } from "./fetch-metadata.js";
12
+ export { acceptSignatureDetector } from "./accept-signature.js";
13
+ export { headerOrderDetector, headerOrderFingerprint } from "./header-order.js";
14
+ export type { HeaderOrderOptions } from "./header-order.js";
15
+ export { rateAnomalyDetector } from "./rate-anomaly.js";
16
+ export type { RateAnomalyOptions } from "./rate-anomaly.js";
17
+ export { cadenceDetector } from "./cadence.js";
18
+ export type { CadenceOptions } from "./cadence.js";
19
+ export { crawlBreadthDetector } from "./crawl-breadth.js";
20
+ export type { CrawlBreadthOptions } from "./crawl-breadth.js";
21
+ export { sessionIntegrityDetector } from "./session-integrity.js";
22
+ export type { SessionIntegrityOptions } from "./session-integrity.js";
23
+ export { identityRotationDetector } from "./identity-rotation.js";
24
+ export type { IdentityRotationOptions } from "./identity-rotation.js";
25
+ export { TRAP_FIELD_SOURCE, trapDetector, renderTrapLink, renderTrapField, trapRobotsEntries, DEFAULT_TRAP_PATHS } from "./trap.js";
26
+ export type { TrapOptions, TrapLinkOptions } from "./trap.js";
27
+ export { ipIntelligenceDetector } from "./ip-intelligence.js";
28
+ export type { IpIntelligenceOptions } from "./ip-intelligence.js";
29
+ export { tlsFingerprintDetector } from "./tls-fingerprint.js";
30
+ export type { TlsFingerprintOptions, FingerprintProfile } from "./tls-fingerprint.js";
31
+ export { clearanceDetector } from "./clearance.js";
32
+ export { uaCoherenceDetector } from "./ua-coherence.js";
33
+ export { probeSignatureDetector } from "./probe-signature.js";
34
+ export type { ProbeSignatureOptions } from "./probe-signature.js";
35
+ export { browsingCoherenceDetector } from "./browsing-coherence.js";
36
+ export { clientSignalsDetector } from "./client-signals.js";
37
+ export type { ClientSignalsOptions } from "./client-signals.js";
38
+ export { BOT_SIGNATURES, BENIGN_CATEGORIES, compileSignatures, indexSignatures } from "./known-bots.js";
39
+ export type { BotSignature, BotCategory, Verification } from "./known-bots.js";
40
+ /**
41
+ * The detector set installed when you configure none.
42
+ *
43
+ * Two are missing on purpose, and both omissions are about false positives rather
44
+ * than about cost:
45
+ *
46
+ * - `identityRotationDetector` fires on any address that fronts several browsers,
47
+ * which describes every corporate NAT and mobile carrier on the internet. It is
48
+ * valuable, but only once your `actorKey` is narrower than an IP.
49
+ * - `tlsFingerprintDetector` needs a fingerprint from your edge and a profile table
50
+ * you maintain. With neither it is inert; with a stale table it misfires on anyone
51
+ * running a browser newer than your data.
52
+ *
53
+ * `clearanceDetector` is not here either, because it needs the challenge service —
54
+ * the engine adds it automatically once `challenge.secrets` is configured.
55
+ */
56
+ export declare function defaultDetectors(): Detector[];
@@ -0,0 +1,30 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface IpIntelligenceOptions {
3
+ /** Weight for a datacenter-range match. Default 0.3 (`moderate`). */
4
+ datacenterWeight?: number;
5
+ }
6
+ /**
7
+ * What do we know about where this request came from?
8
+ *
9
+ * Two range sets are consulted, and they are treated very differently.
10
+ *
11
+ * **`denylist`** is `certain`, and the justification is not technical. You configured
12
+ * it. The library is not inferring anything; it is carrying out an instruction you
13
+ * gave about addresses you have decided about. Certainty here means "this is a
14
+ * decision, not a guess" — and it means a bad entry in your denylist blocks real
15
+ * people, which is exactly why the range is required to be explicit rather than
16
+ * inherited from a feed by default.
17
+ *
18
+ * **`datacenter`** is `moderate` and always will be. Hosting-provider address space
19
+ * is where scrapers live, and it is *also* where every consumer VPN, every corporate
20
+ * egress gateway, every Tor exit, every privacy relay like iCloud Private Relay, and
21
+ * a growing share of mobile traffic lives. Treating "came from AWS" as proof of
22
+ * automation blocks a meaningful slice of ordinary users, disproportionately the
23
+ * privacy-conscious ones. As corroboration alongside a header failure it is
24
+ * genuinely useful; alone it is close to worthless.
25
+ *
26
+ * No range data ships with this library. Address-to-operator mappings go stale
27
+ * within weeks, and a stale mapping is a false positive with a long half-life —
28
+ * supply your own, from a source you refresh and can audit.
29
+ */
30
+ export declare function ipIntelligenceDetector(options?: IpIntelligenceOptions): Detector;
@@ -0,0 +1,94 @@
1
+ import { MultiPatternMatcher } from "../internal/matcher.js";
2
+ /**
3
+ * The known-bot signature database.
4
+ *
5
+ * Two things live here, and it is worth being precise about the difference.
6
+ *
7
+ * A **signature** matches a literal token in a User-Agent. That is a *claim* the
8
+ * client makes about itself, nothing more — anyone can send `Googlebot/2.1`. A claim
9
+ * is useful for two opposite reasons: an honest bot's claim tells us what it is, and
10
+ * a dishonest one gives us something to disprove.
11
+ *
12
+ * A **verification** is how that claim gets checked against an authority outside the
13
+ * request. Only the pairing of the two produces a `certain` verdict — either
14
+ * `verified-bot` (claim confirmed) or `impersonator` (claim disproved). A signature
15
+ * with no verification can never do better than `declared-bot`.
16
+ */
17
+ export type BotCategory = "search" | "ai" | "seo" | "social" | "monitoring" | "archive" | "feed" | "security" | "advertising" | "library" | "headless"
18
+ /** A real browser engine embedded in a desktop application, with a person driving it. */
19
+ | "embedded" | "other";
20
+ /** Every category, for anything that has to enumerate them — a rule editor, a report. */
21
+ export declare const BOT_CATEGORIES: readonly BotCategory[];
22
+ /**
23
+ * How a claimed identity is checked.
24
+ *
25
+ * - `fcrdns` — forward-confirmed reverse DNS. Reverse-resolve the client IP, require
26
+ * the name to sit under one of `domains`, then forward-resolve that name and
27
+ * require the original IP back. Forging this requires control of the operator's
28
+ * DNS, which is the property that makes it `certain` in both directions.
29
+ * - `ip-ranges` — the operator publishes IP ranges but sets no usable PTR records.
30
+ * Verifiable only if you supply the ranges (see `crawlerRanges` in the config);
31
+ * the library will not fetch them for you, because a detector that makes an
32
+ * outbound HTTP request on a schedule is a dependency you should opt into
33
+ * knowingly.
34
+ * - `none` — no published mechanism. The claim is unfalsifiable, so we neither
35
+ * confirm nor accuse.
36
+ */
37
+ export type Verification = {
38
+ kind: "fcrdns";
39
+ domains: readonly string[];
40
+ } | {
41
+ kind: "ip-ranges";
42
+ publishedAt?: string;
43
+ } | {
44
+ kind: "none";
45
+ };
46
+ export interface BotSignature {
47
+ /** Stable id, used in rules, logs and metrics. */
48
+ id: string;
49
+ name: string;
50
+ /** Lowercase literal tokens; a match on any one identifies this bot. */
51
+ tokens: readonly string[];
52
+ category: BotCategory;
53
+ verification: Verification;
54
+ /**
55
+ * Whether this client is generally *benign* — something most sites want to keep
56
+ * serving. Search and social preview crawlers are; scrapers and scanners are not.
57
+ * Drives the default policy's allow path, never a block.
58
+ */
59
+ benign: boolean;
60
+ /**
61
+ * Whether matching this signature is *conclusive* evidence of automation. Default
62
+ * true.
63
+ *
64
+ * Set it false when the token can legitimately appear on a request a person made.
65
+ * The motivating case is Electron: the UA is emitted by VS Code's Simple Browser,
66
+ * Slack, Discord and Postman, which are real Chromium instances with a human
67
+ * driving them. A signature match is only allowed into the `certain` tier when the
68
+ * claim "no honest client sends this" actually holds, and for embedded webviews it
69
+ * plainly does not.
70
+ */
71
+ conclusive?: boolean;
72
+ /** Why this signature is not conclusive. Surfaced in the evidence summary. */
73
+ caveat?: string;
74
+ /**
75
+ * The product token to name in a `User-agent:` line in `robots.txt`.
76
+ *
77
+ * Defaults to the first match token. Worth setting where the canonical spelling
78
+ * differs from the token we match on — `robots.txt` matching is case-insensitive,
79
+ * so this is about a file a person will read, not about correctness.
80
+ */
81
+ robotsAgent?: string;
82
+ docs?: string;
83
+ }
84
+ /** Every built-in signature, in one array. Extend it via `extraSignatures` rather than editing. */
85
+ export declare const BOT_SIGNATURES: readonly BotSignature[];
86
+ /**
87
+ * Compiles signatures into a single-pass matcher. Build this once per engine, never
88
+ * per request — construction is O(total pattern length) and matching is O(input).
89
+ */
90
+ export declare function compileSignatures(signatures?: readonly BotSignature[]): MultiPatternMatcher<BotSignature>;
91
+ /** Index by id, so rules can name a bot (`"googlebot"`) and get its metadata back. */
92
+ export declare function indexSignatures(signatures?: readonly BotSignature[]): ReadonlyMap<string, BotSignature>;
93
+ /** Categories whose members are, by default, worth serving. Used by `allowBenignBots`. */
94
+ export declare const BENIGN_CATEGORIES: ReadonlySet<BotCategory>;
@@ -0,0 +1,50 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * What is this request *asking for*?
4
+ *
5
+ * Every other detector here reads the client. This one reads the request target, and
6
+ * it exists because of a gap the rest of the library cannot close: the scanner that
7
+ * does not announce itself. `self-identified` catches sqlmap and Nikto because they
8
+ * say so, and a great deal of hostile traffic does say so. The rest arrives wearing a
9
+ * copied Chrome User-Agent, and from the headers alone it is indistinguishable from a
10
+ * person — until you look at what it asked for, which is `/.env`, then
11
+ * `/.git/config`, then `/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php`.
12
+ *
13
+ * Those paths are not obscure corners of a site. They are entries in a wordlist. No
14
+ * link points at them, no menu leads to them, and no person types them — the traffic
15
+ * that requests them is running a list against every address it can reach.
16
+ *
17
+ * **Why this stops at `strong` and never reaches `certain`.** The library's rule is
18
+ * that `certain` evidence must admit no benign explanation, and a *request target*
19
+ * always admits one: a URL is client-supplied text, and the client supplying it might
20
+ * be a security engineer testing their own site from a laptop, a researcher with a
21
+ * bug-bounty scope, or a monitoring check somebody wrote at three in the morning.
22
+ * Those are people, and the correct response to them is a challenge, not a closed
23
+ * door. A trap path is different — it is unreachable *by construction*, which is
24
+ * exactly the property a wordlist entry lacks.
25
+ *
26
+ * **What it does not do.** It reads one request at a time. A scanner walking a
27
+ * wordlist produces one of these observations per request rather than an escalating
28
+ * series, because nothing here remembers that the same actor asked for `/.env` a
29
+ * moment ago — the detectors that watch an actor over time are `rate-anomaly`,
30
+ * `cadence` and `crawl-breadth`, and enumeration shows up there. Keeping this one
31
+ * stateless is what lets it run unchanged over a log file in `replay`.
32
+ *
33
+ * **The one thing to configure.** The `platform` tier below is a list of admin and
34
+ * login paths that are probes on most sites and are the *front door* on the sites
35
+ * that run those platforms. If you run WordPress, `/wp-login.php` is where your
36
+ * authors sign in, and this detector will report each of them as a probe. Add such
37
+ * paths to `ignore`, or to the engine's `ignorePaths`, before turning any of this into
38
+ * a rule. The tier is capped at `moderate` precisely so that forgetting costs a
39
+ * tag rather than a door.
40
+ */
41
+ export interface ProbeSignatureOptions {
42
+ /**
43
+ * Paths this site genuinely serves, exempted from the platform tier. Matched as
44
+ * prefixes, so `/wp-admin` covers everything beneath it.
45
+ */
46
+ ignore?: readonly string[];
47
+ /** Additional exact paths or prefixes to treat as exploit-tier probes. */
48
+ extraPaths?: readonly string[];
49
+ }
50
+ export declare function probeSignatureDetector(options?: ProbeSignatureOptions): Detector;
@@ -0,0 +1,36 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface RateAnomalyOptions {
3
+ /** Window over which arrivals are counted, ms. Default 10000. */
4
+ windowMs?: number;
5
+ /** Requests in the window above which the rate is worth noting. Default 20. */
6
+ threshold?: number;
7
+ /**
8
+ * Requests in the window above which the rate is well beyond human. Default 30.
9
+ *
10
+ * Both defaults sit under the per-actor timestamp ring's capacity, because the
11
+ * count saturates there — a threshold above it could never be reached, which is a
12
+ * detector that silently never fires. If you raise these, raise them knowing the
13
+ * ceiling, and reach for the `rate-limit` action instead when you need real numbers.
14
+ */
15
+ hardThreshold?: number;
16
+ }
17
+ /**
18
+ * How fast is this actor going?
19
+ *
20
+ * Rate is the signal people reach for first and trust the most, and it deserves the
21
+ * least trust of anything in this library. The reason is that the *actor* behind a
22
+ * high rate is frequently not one client: a corporate NAT, a mobile carrier's CGNAT
23
+ * pool, a university, a VPN exit and a shared office all present hundreds of real
24
+ * people as one address. Blocking on rate blocks all of them.
25
+ *
26
+ * So this detector reports and never concludes. Its output tops out at `moderate`,
27
+ * which under the default policy cannot reach a terminal action no matter how
28
+ * extreme the number gets. What high rate is genuinely good for is *corroborating* —
29
+ * a client that already looks like a library and is also pulling 300 requests a
30
+ * minute is a different proposition from either fact alone.
31
+ *
32
+ * If you want rate to actually stop traffic, that is what rate *limiting* is for,
33
+ * and it belongs in the action layer where it applies to everyone equally and
34
+ * recovers on its own. See the `rate-limit` action.
35
+ */
36
+ export declare function rateAnomalyDetector(options?: RateAnomalyOptions): Detector;
@@ -0,0 +1,33 @@
1
+ import type { Detector } from "./types.js";
2
+ import type { BotCategory } from "./known-bots.js";
3
+ export interface SelfIdentifiedOptions {
4
+ /**
5
+ * Categories to report. Removing one does not make its traffic invisible — the
6
+ * behavioural detectors still see it — it only stops this detector naming it.
7
+ */
8
+ categories?: readonly BotCategory[];
9
+ }
10
+ /**
11
+ * The client told us what it is.
12
+ *
13
+ * This detector is the backbone of the `certain` tier, and the reason is a point
14
+ * about responsibility rather than about technology. When a request arrives saying
15
+ * `python-requests/2.31.0` or `Googlebot/2.1`, we are not *inferring* anything. We
16
+ * are taking the client at its word. If that word is a lie, the misclassification is
17
+ * the client's doing, not a failure of detection — and no honest client is ever
18
+ * harmed by being believed.
19
+ *
20
+ * That is why a self-declaration can safely gate a terminal action while a much
21
+ * "smarter" behavioural inference cannot. The behavioural inference can be wrong
22
+ * about someone who never made any claim at all.
23
+ *
24
+ * Two shapes qualify:
25
+ *
26
+ * - A **known signature** — a token from {@link BOT_SIGNATURES}. Library and headless
27
+ * tokens are conclusive on their own: no browser has ever sent `curl/8.4.0`.
28
+ * - An **unrecognised but self-announcing** UA — contains a word like `bot` or
29
+ * `crawler` *and* publishes a contact URL or email, the long-standing convention
30
+ * for well-behaved crawlers. Either half alone is only suggestive; together they
31
+ * are a declaration.
32
+ */
33
+ export declare function selfIdentifiedDetector(options?: SelfIdentifiedOptions): Detector;