@osqd/bothandlerjs 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +516 -0
- package/LICENSE +102 -0
- package/README.md +308 -0
- package/bin/bothandlerjs.mjs +14 -0
- package/dist/actions/index.d.ts +22 -0
- package/dist/actions/types.d.ts +59 -0
- package/dist/adapters/fastify.d.ts +43 -0
- package/dist/adapters/fetch.d.ts +51 -0
- package/dist/adapters/index.cjs +723 -0
- package/dist/adapters/index.cjs.map +1 -0
- package/dist/adapters/index.d.ts +8 -0
- package/dist/adapters/index.js +692 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/koa.d.ts +27 -0
- package/dist/adapters/node.d.ts +29 -0
- package/dist/adapters/shared.d.ts +39 -0
- package/dist/audit.d.ts +172 -0
- package/dist/challenge/index.d.ts +176 -0
- package/dist/challenge/language.d.ts +57 -0
- package/dist/challenge/page.d.ts +42 -0
- package/dist/challenge/pow.d.ts +33 -0
- package/dist/challenge/token.d.ts +72 -0
- package/dist/cli.cjs +13388 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.js +13358 -0
- package/dist/cli.js.map +1 -0
- package/dist/client/index.cjs +139 -0
- package/dist/client/index.cjs.map +1 -0
- package/dist/client/index.d.ts +63 -0
- package/dist/client/index.js +112 -0
- package/dist/client/index.js.map +1 -0
- package/dist/config.d.ts +316 -0
- package/dist/core.d.ts +443 -0
- package/dist/corpus/adversarial.d.ts +2 -0
- package/dist/corpus/advertising-email.d.ts +2 -0
- package/dist/corpus/ai-crawlers.d.ts +2 -0
- package/dist/corpus/benign-bots.d.ts +2 -0
- package/dist/corpus/cdn-gateways.d.ts +2 -0
- package/dist/corpus/cookies.d.ts +49 -0
- package/dist/corpus/crawlers-regional.d.ts +2 -0
- package/dist/corpus/crawlers-vertical.d.ts +2 -0
- package/dist/corpus/headers.d.ts +403 -0
- package/dist/corpus/humans-apps.d.ts +2 -0
- package/dist/corpus/humans-browsers.d.ts +2 -0
- package/dist/corpus/humans.d.ts +2 -0
- package/dist/corpus/index.cjs +4196 -0
- package/dist/corpus/index.cjs.map +1 -0
- package/dist/corpus/index.d.ts +37 -0
- package/dist/corpus/index.js +4132 -0
- package/dist/corpus/index.js.map +1 -0
- package/dist/corpus/infrastructure.d.ts +17 -0
- package/dist/corpus/libraries-extended.d.ts +2 -0
- package/dist/corpus/ranges.d.ts +31 -0
- package/dist/corpus/reputation.d.ts +21 -0
- package/dist/corpus/runner.d.ts +130 -0
- package/dist/corpus/schema.d.ts +185 -0
- package/dist/corpus/tooling.d.ts +2 -0
- package/dist/corpus/unwanted.d.ts +2 -0
- package/dist/crawler-ranges.d.ts +113 -0
- package/dist/dashboard/client.generated.d.ts +7 -0
- package/dist/dashboard/feed.d.ts +186 -0
- package/dist/dashboard/index.d.ts +5 -0
- package/dist/dashboard/page.d.ts +51 -0
- package/dist/dashboard/parse-request.d.ts +28 -0
- package/dist/dashboard/preview.d.ts +45 -0
- package/dist/dashboard/server.d.ts +70 -0
- package/dist/dashboard/types.d.ts +714 -0
- package/dist/detectors/accept-signature.d.ts +6 -0
- package/dist/detectors/browsing-coherence.d.ts +44 -0
- package/dist/detectors/cadence.d.ts +28 -0
- package/dist/detectors/clearance.d.ts +25 -0
- package/dist/detectors/client-hints.d.ts +19 -0
- package/dist/detectors/client-signals.d.ts +27 -0
- package/dist/detectors/crawl-breadth.d.ts +28 -0
- package/dist/detectors/crawler-verification.d.ts +39 -0
- package/dist/detectors/fetch-metadata.d.ts +20 -0
- package/dist/detectors/header-integrity.d.ts +26 -0
- package/dist/detectors/header-order.d.ts +27 -0
- package/dist/detectors/identity-rotation.d.ts +34 -0
- package/dist/detectors/index.d.ts +56 -0
- package/dist/detectors/ip-intelligence.d.ts +30 -0
- package/dist/detectors/known-bots.d.ts +94 -0
- package/dist/detectors/probe-signature.d.ts +50 -0
- package/dist/detectors/rate-anomaly.d.ts +36 -0
- package/dist/detectors/self-identified.d.ts +33 -0
- package/dist/detectors/session-integrity.d.ts +24 -0
- package/dist/detectors/tls-fingerprint.d.ts +34 -0
- package/dist/detectors/trap.d.ts +98 -0
- package/dist/detectors/types.d.ts +70 -0
- package/dist/detectors/ua-coherence.d.ts +32 -0
- package/dist/evidence.d.ts +45 -0
- package/dist/facts.d.ts +46 -0
- package/dist/index.cjs +9600 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +63 -0
- package/dist/index.js +9444 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/async.d.ts +16 -0
- package/dist/internal/clock.d.ts +17 -0
- package/dist/internal/crypto.d.ts +26 -0
- package/dist/internal/dns.d.ts +100 -0
- package/dist/internal/emitter.d.ts +18 -0
- package/dist/internal/http.d.ts +22 -0
- package/dist/internal/ip.d.ts +73 -0
- package/dist/internal/lru.d.ts +34 -0
- package/dist/internal/matcher.d.ts +41 -0
- package/dist/internal/pattern.d.ts +14 -0
- package/dist/internal/ua.d.ts +92 -0
- package/dist/metrics.d.ts +120 -0
- package/dist/notify/hub.d.ts +70 -0
- package/dist/notify/index.d.ts +7 -0
- package/dist/notify/redact.d.ts +48 -0
- package/dist/notify/sinks.d.ts +62 -0
- package/dist/notify/types.d.ts +56 -0
- package/dist/policy/index.d.ts +7 -0
- package/dist/policy/match.d.ts +20 -0
- package/dist/policy/policy.d.ts +123 -0
- package/dist/policy/presets.d.ts +161 -0
- package/dist/policy/types.d.ts +155 -0
- package/dist/robots.d.ts +83 -0
- package/dist/state.d.ts +161 -0
- package/dist/stores/index.d.ts +5 -0
- package/dist/stores/memory.d.ts +33 -0
- package/dist/stores/redis.d.ts +59 -0
- package/dist/stores/types.d.ts +37 -0
- package/dist/types.d.ts +250 -0
- package/docs/challenge/index.md +152 -0
- package/docs/challenge/localisation.md +105 -0
- package/docs/concepts/actors.md +113 -0
- package/docs/concepts/evidence.md +133 -0
- package/docs/concepts/the-guard.md +121 -0
- package/docs/concepts/threat-model.md +120 -0
- package/docs/concepts/verdicts.md +108 -0
- package/docs/course/01-first-assessment.md +178 -0
- package/docs/course/02-proof-and-suspicion.md +164 -0
- package/docs/course/03-verdicts-and-scores.md +160 -0
- package/docs/course/04-the-guard.md +200 -0
- package/docs/course/05-detectors.md +178 -0
- package/docs/course/06-identity.md +161 -0
- package/docs/course/07-actors.md +178 -0
- package/docs/course/08-traps.md +183 -0
- package/docs/course/09-rules.md +224 -0
- package/docs/course/10-actions-and-presets.md +216 -0
- package/docs/course/11-the-challenge.md +227 -0
- package/docs/course/12-going-live.md +211 -0
- package/docs/course/13-operating-it.md +238 -0
- package/docs/course/14-scaling.md +195 -0
- package/docs/course/15-extending.md +259 -0
- package/docs/course/16-proving-it.md +266 -0
- package/docs/course/index.md +112 -0
- package/docs/design/decisions.md +216 -0
- package/docs/detection/client-signals.md +96 -0
- package/docs/detection/detectors.md +376 -0
- package/docs/detection/index.md +134 -0
- package/docs/detection/signatures.md +111 -0
- package/docs/detection/verification.md +123 -0
- package/docs/detection/writing-a-detector.md +145 -0
- package/docs/index.md +100 -0
- package/docs/integration/adapters.md +138 -0
- package/docs/integration/client-ip.md +114 -0
- package/docs/integration/index.md +55 -0
- package/docs/integration/stores.md +76 -0
- package/docs/operations/audit.md +116 -0
- package/docs/operations/dashboard.md +725 -0
- package/docs/operations/index.md +101 -0
- package/docs/operations/metrics.md +74 -0
- package/docs/operations/notifications.md +72 -0
- package/docs/operations/runtime-changes.md +118 -0
- package/docs/policy/actions.md +196 -0
- package/docs/policy/index.md +84 -0
- package/docs/policy/presets.md +191 -0
- package/docs/policy/robots.md +128 -0
- package/docs/policy/rules.md +134 -0
- package/docs/reference/api.md +126 -0
- package/docs/reference/configuration.md +175 -0
- package/docs/start/choosing-a-policy.md +127 -0
- package/docs/start/first-integration.md +127 -0
- package/docs/start/installation.md +63 -0
- package/docs/start/upgrading.md +72 -0
- package/docs/testing/cli.md +119 -0
- package/docs/testing/corpus.md +129 -0
- package/docs/testing/index.md +61 -0
- package/docs/testing/replay.md +102 -0
- package/docs/testing/try-it.md +189 -0
- package/package.json +145 -0
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Reads the content-negotiation headers as a signature of the client.
|
|
4
|
+
*
|
|
5
|
+
* A browser asking for a page sends a long, specific `Accept` describing the
|
|
6
|
+
* document formats it renders. A scraper asks for `*/ export declare function acceptSignatureDetector(): Detector;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Evidence that this request came out of a browsing session.
|
|
4
|
+
*
|
|
5
|
+
* Almost every detector in this library argues in one direction. That asymmetry is a
|
|
6
|
+
* problem the scoring model makes visible: a person reading forty pages of
|
|
7
|
+
* documentation from a university's shared address accumulates `rate-anomaly`,
|
|
8
|
+
* `cadence`, `crawl-breadth` and `ip-intelligence` and can cross the suspicion
|
|
9
|
+
* threshold without a single thing being wrong with their request. The engine
|
|
10
|
+
* discounts a bot score by whatever human evidence it has — `pBot × (1 − pHuman)` —
|
|
11
|
+
* and until now the only things that produced any were your own application's
|
|
12
|
+
* assertion, a clearance token, a TLS profile table most people do not maintain, and
|
|
13
|
+
* a page script most people do not embed. For an ordinary request from an ordinary
|
|
14
|
+
* browser, `pHuman` was zero.
|
|
15
|
+
*
|
|
16
|
+
* So this detector reads the properties that come from a client having *state and
|
|
17
|
+
* history* rather than from a client being well-formed:
|
|
18
|
+
*
|
|
19
|
+
* - **A cache to revalidate.** `If-None-Match` and `If-Modified-Since` mean this
|
|
20
|
+
* client has been here before and kept what it was given. A stateless fetch loop
|
|
21
|
+
* has nothing to revalidate against and asks for the resource fresh every time.
|
|
22
|
+
* - **A cookie jar.** Something set state on this client and the client sent it back.
|
|
23
|
+
* - **A user gesture.** `Sec-Fetch-User: ?1` is a forbidden header the browser sets
|
|
24
|
+
* only when a *person* activated the navigation — a link click, a typed URL, a
|
|
25
|
+
* bookmark. Page script cannot set it, and a driver navigating a page
|
|
26
|
+
* programmatically does not produce it.
|
|
27
|
+
* - **A coherent modern fingerprint.** The Fetch Metadata group, the Client Hints and
|
|
28
|
+
* the negotiation headers all present and agreeing with each other.
|
|
29
|
+
*
|
|
30
|
+
* **None of this is proof and none of it ever will be.** Every one of these
|
|
31
|
+
* properties is copyable: a scraper that keeps a cookie jar, replays an ETag and
|
|
32
|
+
* copies a header set produces all four. That is why the ceiling here is `moderate`,
|
|
33
|
+
* why the whole set shares one {@link Evidence.family} so it can never stack into a
|
|
34
|
+
* large discount, and why it can only ever *reduce* a suspicion score rather than
|
|
35
|
+
* establish a `human` verdict on its own. The only conclusive human evidence in this
|
|
36
|
+
* library comes from you — `isHuman`, or an operator clearance token — because you
|
|
37
|
+
* are the one party in the exchange whose word the client cannot forge.
|
|
38
|
+
*
|
|
39
|
+
* What it buys is the honest half of a two-sided argument: a request that shows the
|
|
40
|
+
* marks of a real session is less suspicious than an identical one that does not,
|
|
41
|
+
* and the population that benefits is precisely the one this library exists to
|
|
42
|
+
* protect.
|
|
43
|
+
*/
|
|
44
|
+
export declare function browsingCoherenceDetector(): Detector;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface CadenceOptions {
|
|
3
|
+
/** Minimum gaps needed before the statistic means anything. Default 6. */
|
|
4
|
+
minSamples?: number;
|
|
5
|
+
/** Coefficient of variation at or below which the rhythm is machine-regular. Default 0.15. */
|
|
6
|
+
regularityThreshold?: number;
|
|
7
|
+
/** Ignore actors whose mean gap exceeds this, ms. Default 120000 (2 min). */
|
|
8
|
+
maxMeanIntervalMs?: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Is this actor's *rhythm* human?
|
|
12
|
+
*
|
|
13
|
+
* People generate ragged inter-arrival times. They read, scroll, get distracted, open
|
|
14
|
+
* three tabs at once, then nothing for four minutes. A loop calling `setInterval` or
|
|
15
|
+
* awaiting a fixed delay produces gaps clustered tightly around one value, and the
|
|
16
|
+
* coefficient of variation makes that visible in a single number.
|
|
17
|
+
*
|
|
18
|
+
* The check is on *regularity*, not speed, which is what makes it complementary to
|
|
19
|
+
* `rate-anomaly` — a slow, polite scraper deliberately pacing itself at one request
|
|
20
|
+
* every two seconds to stay under a rate limit is invisible to rate counting and
|
|
21
|
+
* extremely visible here.
|
|
22
|
+
*
|
|
23
|
+
* It stays `moderate` because a real page can produce regular traffic too: a polling
|
|
24
|
+
* XHR, a video player fetching segments, an SSE reconnect loop. Those are your own
|
|
25
|
+
* frontend, and the fix is to exclude their paths (see `ignorePaths` in the engine
|
|
26
|
+
* config) rather than to weaken the statistic.
|
|
27
|
+
*/
|
|
28
|
+
export declare function cadenceDetector(options?: CadenceOptions): Detector;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
import type { ChallengeService } from "../challenge/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* Reads a clearance token the client already holds.
|
|
5
|
+
*
|
|
6
|
+
* This is the library's only source of *human*-pointing evidence that is not a
|
|
7
|
+
* guess about headers, and it is worth being exact about what each level earns,
|
|
8
|
+
* because overstating any of them would undo the whole design.
|
|
9
|
+
*
|
|
10
|
+
* - **`operator`** is `certain`. Your application told us this is a person — an
|
|
11
|
+
* authenticated session, a completed purchase, whatever your own bar is. We are
|
|
12
|
+
* believing you, not deducing anything, which is the same reasoning that makes a
|
|
13
|
+
* client's self-declaration `certain` in the other direction.
|
|
14
|
+
* - **`interaction`** is `strong`. A trusted input event was observed. Automation
|
|
15
|
+
* driving a real browser can synthesise something close, so this is very good
|
|
16
|
+
* evidence and not proof.
|
|
17
|
+
* - **`pow`** is only `moderate`, and this is the number people are most tempted to
|
|
18
|
+
* inflate. A solved proof of work shows a JavaScript engine ran and CPU was spent.
|
|
19
|
+
* A headless Chrome does both, happily and at scale. It raises the cost of a scrape
|
|
20
|
+
* substantially; it says nothing whatsoever about whether a human is present.
|
|
21
|
+
*
|
|
22
|
+
* The token is bound to the actor and signed, so it cannot be lifted from one client
|
|
23
|
+
* and replayed by another under a different actor key.
|
|
24
|
+
*/
|
|
25
|
+
export declare function clearanceDetector(service: ChallengeService): Detector;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Cross-checks User-Agent Client Hints against the legacy User-Agent string.
|
|
4
|
+
*
|
|
5
|
+
* A real Chromium browser generates both from the same internal state, so they
|
|
6
|
+
* always agree. A client that rewrites one and forgets the other contradicts itself,
|
|
7
|
+
* and a contradiction is visible from a single request with no history, no state and
|
|
8
|
+
* no network call — which makes this one of the cheapest high-value checks there is.
|
|
9
|
+
*
|
|
10
|
+
* **Why none of this is `certain`, despite being a genuine contradiction.** The
|
|
11
|
+
* population that rewrites a User-Agent without touching Client Hints is not only
|
|
12
|
+
* scrapers: it is also every person running a UA-spoofing privacy extension, every
|
|
13
|
+
* enterprise browser with a rewritten UA policy, and every developer with device
|
|
14
|
+
* emulation open. Those are real people, and a `certain` tier that swept them up
|
|
15
|
+
* would make the word meaningless. So these observations score — sometimes heavily —
|
|
16
|
+
* and never block on their own. The lone exception is a hint that *self-declares*
|
|
17
|
+
* headless operation, which is a statement rather than an inference.
|
|
18
|
+
*/
|
|
19
|
+
export declare function clientHintsDetector(): Detector;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Detector, DetectionContext } from "./types.js";
|
|
2
|
+
export interface ClientSignalsOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Where to find the signals on the request. Default: `facts.extra.clientSignals`.
|
|
5
|
+
*
|
|
6
|
+
* You are responsible for putting them there — typically by storing what your
|
|
7
|
+
* signal endpoint received against the session and attaching it in the adapter's
|
|
8
|
+
* `enrich` hook. The library does not invent a storage mechanism for you, because
|
|
9
|
+
* where per-session data lives is a decision only your application can make.
|
|
10
|
+
*/
|
|
11
|
+
read?: (facts: DetectionContext["facts"]) => unknown;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Reads signals reported by the page script.
|
|
15
|
+
*
|
|
16
|
+
* The ceiling here is `moderate`, and it is a hard ceiling for a reason worth
|
|
17
|
+
* restating: every one of these values was produced by JavaScript running inside the
|
|
18
|
+
* client, which is the one place an adversary has complete control. A framework that
|
|
19
|
+
* wants `navigator.webdriver` to read `false` sets it to `false`, and everything this
|
|
20
|
+
* detector sees afterwards is whatever that framework decided to say.
|
|
21
|
+
*
|
|
22
|
+
* What it genuinely catches is automation that never bothered to hide — Selenium out
|
|
23
|
+
* of the box, a scripted Chrome someone pointed at your site this afternoon — which
|
|
24
|
+
* is a large share of real bot traffic. What it must never do is convince you that a
|
|
25
|
+
* clean report means a person.
|
|
26
|
+
*/
|
|
27
|
+
export declare function clientSignalsDetector(options?: ClientSignalsOptions): Detector;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface CrawlBreadthOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Distinct paths from one actor at or above which breadth is worth reporting.
|
|
5
|
+
* Default 30. Cannot exceed {@link MAX_TRACKED_PATHS}, which is where the count
|
|
6
|
+
* saturates; asking for more throws rather than never firing.
|
|
7
|
+
*/
|
|
8
|
+
threshold?: number;
|
|
9
|
+
/** Fraction of requests that must be to a path not seen before. Default 0.85. */
|
|
10
|
+
noveltyRatio?: number;
|
|
11
|
+
/** Minimum requests before the ratio is meaningful. Default 20. */
|
|
12
|
+
minRequests?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Is this actor *reading* the site or *enumerating* it?
|
|
16
|
+
*
|
|
17
|
+
* A person revisits. They land on an article, go back to the index, follow a related
|
|
18
|
+
* link, return to the article. Their ratio of distinct paths to total requests
|
|
19
|
+
* settles well below one. A crawler walking a sitemap almost never revisits, so its
|
|
20
|
+
* ratio sits near one and its distinct-path count climbs steadily.
|
|
21
|
+
*
|
|
22
|
+
* Worth being clear about what this cannot distinguish: a *welcome* crawler produces
|
|
23
|
+
* exactly this shape, and so does a person on a first visit to a documentation site
|
|
24
|
+
* clicking through the sidebar. It is a shape, not a motive — which is why it stays
|
|
25
|
+
* `weak` and why the interesting use is combining it with an actor that has already
|
|
26
|
+
* failed a header check.
|
|
27
|
+
*/
|
|
28
|
+
export declare function crawlBreadthDetector(options?: CrawlBreadthOptions): Detector;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface CrawlerVerificationOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Treat an address with no PTR record as a forged claim. Default true.
|
|
5
|
+
*
|
|
6
|
+
* Every operator whose crawler this applies to publishes PTR records precisely so
|
|
7
|
+
* that servers can check them; a claimed Googlebot with no reverse DNS at all is
|
|
8
|
+
* not a configuration accident, it is the cheapest possible forgery. Set false if
|
|
9
|
+
* you have a resolver that cannot be trusted to distinguish NXDOMAIN from failure.
|
|
10
|
+
*/
|
|
11
|
+
treatMissingPtrAsForgery?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Also assert forgery for `ip-ranges` crawlers when ranges *are* configured and the
|
|
14
|
+
* client is outside them. Default true. Has no effect for a crawler whose ranges
|
|
15
|
+
* you have not supplied — an unverifiable claim stays unverified, never accused.
|
|
16
|
+
*/
|
|
17
|
+
useConfiguredRanges?: boolean;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Confirms or refutes a claimed crawler identity against an authority outside the
|
|
21
|
+
* request.
|
|
22
|
+
*
|
|
23
|
+
* This is the only detector that can produce `verified-bot`, and the only one that
|
|
24
|
+
* can produce a `certain` `impersonator`. Both directions matter:
|
|
25
|
+
*
|
|
26
|
+
* - **Confirmed** — the operator's own DNS vouches for this address. Forging it needs
|
|
27
|
+
* control of `googlebot.com`'s DNS, so a pass here is proof, and the default policy
|
|
28
|
+
* uses it to *allow*: your SEO does not deserve to be collateral damage from a bot
|
|
29
|
+
* rule.
|
|
30
|
+
* - **Refuted** — the client specifically claimed to be a named third party and the
|
|
31
|
+
* claim is false. Note how narrow that is. A privacy extension rewriting a UA to a
|
|
32
|
+
* generic browser string never lands here, because it never claims to be Googlebot.
|
|
33
|
+
* Only a deliberate forgery of a *verifiable* identity does, which is why this one
|
|
34
|
+
* is allowed to block.
|
|
35
|
+
*
|
|
36
|
+
* Anything short of a clear answer — a timeout, SERVFAIL, an unconfigured range list
|
|
37
|
+
* — yields nothing at all. Silence is never treated as an accusation.
|
|
38
|
+
*/
|
|
39
|
+
export declare function crawlerVerificationDetector(options?: CrawlerVerificationOptions): Detector;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Checks the Fetch Metadata headers (`Sec-Fetch-Site`, `-Mode`, `-Dest`, `-User`).
|
|
4
|
+
*
|
|
5
|
+
* These are *forbidden* headers: page JavaScript cannot set or alter them, so in a
|
|
6
|
+
* real browser they are generated by the network stack from the actual context of
|
|
7
|
+
* the request. That makes them unusually hard to fake convincingly — a client
|
|
8
|
+
* assembling requests by hand either omits them or picks a combination the browser
|
|
9
|
+
* would never produce.
|
|
10
|
+
*
|
|
11
|
+
* Two failure modes are checked. **Absence** on an engine known to send them, over
|
|
12
|
+
* HTTPS, where they should be present. And **incoherence** — a `Sec-Fetch-Dest:
|
|
13
|
+
* document` that is not a navigation describes a request no browser makes.
|
|
14
|
+
*
|
|
15
|
+
* Neither is `certain`, for the same reason as elsewhere: a stripping proxy in front
|
|
16
|
+
* of your server produces the first, and an unusual-but-real client integration can
|
|
17
|
+
* produce the second. Together with a matching header set, though, they are among
|
|
18
|
+
* the sharpest probabilistic signals available.
|
|
19
|
+
*/
|
|
20
|
+
export declare function fetchMetadataDetector(): Detector;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface HeaderIntegrityOptions {
|
|
3
|
+
/** Report a browser-claiming client that omits `Accept-Language`. Default true. */
|
|
4
|
+
checkAcceptLanguage?: boolean;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Does this request's header set match the client it claims to be?
|
|
8
|
+
*
|
|
9
|
+
* Browsers are extremely consistent about which headers they attach; HTTP libraries
|
|
10
|
+
* attach the minimum that gets a response. The gap between the two is one of the
|
|
11
|
+
* most reliable probabilistic signals available from a single request.
|
|
12
|
+
*
|
|
13
|
+
* Almost everything here is deliberately kept below `certain`, because headers pass
|
|
14
|
+
* through corporate proxies, CDNs, privacy extensions and mobile carrier
|
|
15
|
+
* transcoders, any of which will strip or rewrite them for an entirely real person.
|
|
16
|
+
*
|
|
17
|
+
* The exceptions are the three genuine protocol violations: a connection-specific
|
|
18
|
+
* header on HTTP/2 or HTTP/3, a message carrying both `Content-Length` and
|
|
19
|
+
* `Transfer-Encoding`, and a repeated `Host` or `Content-Length`. Each is a rule the
|
|
20
|
+
* specification requires a recipient to *enforce* rather than merely recommends, so a
|
|
21
|
+
* client that breaks it cannot get a response through any compliant proxy — and each
|
|
22
|
+
* is read from what the request *contains*, never from what it lacks. That second
|
|
23
|
+
* property is the one doing the work: an absence is indistinguishable from a facts
|
|
24
|
+
* source that dropped the header, and no absence in this file may reach `certain`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function headerIntegrityDetector(options?: HeaderIntegrityOptions): Detector;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface HeaderOrderOptions {
|
|
3
|
+
/** Include the order fingerprint in evidence metadata for cross-actor correlation. Default true. */
|
|
4
|
+
emitFingerprint?: boolean;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Reads the *order* in which headers arrived.
|
|
8
|
+
*
|
|
9
|
+
* Header order is a genuine fingerprint: a browser's network stack emits a fixed
|
|
10
|
+
* sequence that has nothing to do with what the page requested, and it is one of the
|
|
11
|
+
* few properties a scraper cannot fix by copying a User-Agent string. HTTP client
|
|
12
|
+
* libraries have their own, very different, and equally fixed orders.
|
|
13
|
+
*
|
|
14
|
+
* The reason this detector stays weak is structural rather than a lack of
|
|
15
|
+
* confidence: **anything between the client and this process may reorder headers.**
|
|
16
|
+
* HTTP/2 and HTTP/3 do not preserve a meaningful order at all, some CDNs normalise
|
|
17
|
+
* it, and some proxies rebuild the request wholesale. So it is checked only on
|
|
18
|
+
* HTTP/1.x, only when the transport actually exposed an order, and it contributes a
|
|
19
|
+
* nudge rather than a conclusion.
|
|
20
|
+
*/
|
|
21
|
+
export declare function headerOrderDetector(options?: HeaderOrderOptions): Detector;
|
|
22
|
+
/**
|
|
23
|
+
* Stable fingerprint of a header order, for correlating one actor across rotating
|
|
24
|
+
* addresses. Exported because it is useful well outside this detector — an actor
|
|
25
|
+
* that changes IP every request but keeps the same order fingerprint is one actor.
|
|
26
|
+
*/
|
|
27
|
+
export declare function headerOrderFingerprint(order: readonly string[]): string;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface IdentityRotationOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Distinct User-Agents from one actor at or above which rotation is reported.
|
|
5
|
+
* Default 3.
|
|
6
|
+
*
|
|
7
|
+
* An actor remembers at most {@link MAX_TRACKED_USER_AGENTS} of them, so a threshold
|
|
8
|
+
* above that can never be reached — an actor cycling through thirty spoofed strings
|
|
9
|
+
* still reports four. Asking for more throws rather than producing a detector that
|
|
10
|
+
* runs on every request and can never fire.
|
|
11
|
+
*/
|
|
12
|
+
threshold?: number;
|
|
13
|
+
/** Minimum requests before the count is meaningful. Default 10. */
|
|
14
|
+
minRequests?: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* One actor, several identities.
|
|
18
|
+
*
|
|
19
|
+
* A single client does not change its User-Agent mid-session. Something that does is
|
|
20
|
+
* cycling through a spoofing list, which is behaviour with no innocent
|
|
21
|
+
* interpretation *for a single client*.
|
|
22
|
+
*
|
|
23
|
+
* **This detector is off by default, and you should think before enabling it.** With
|
|
24
|
+
* the default IP-based actor key, "one actor" routinely means "one NAT gateway", and
|
|
25
|
+
* a corporate office, a university, a coffee shop or a mobile carrier's CGNAT pool
|
|
26
|
+
* legitimately presents hundreds of distinct browsers behind one address. Under that
|
|
27
|
+
* key this detector fires on exactly the busiest legitimate networks on the internet.
|
|
28
|
+
*
|
|
29
|
+
* It becomes genuinely valuable once your `actorKey` identifies something narrower
|
|
30
|
+
* than an address — a session cookie, an authenticated user id, or an IP combined
|
|
31
|
+
* with a TLS fingerprint. Then a rotating User-Agent really is one client lying, and
|
|
32
|
+
* it is worth a lot. Enable it there, and only there.
|
|
33
|
+
*/
|
|
34
|
+
export declare function identityRotationDetector(options?: IdentityRotationOptions): Detector;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export type { DetectionContext, Detector, DetectorResult } from "./types.js";
|
|
3
|
+
export { evidence, absenceIsMeaningful } from "./types.js";
|
|
4
|
+
export { selfIdentifiedDetector } from "./self-identified.js";
|
|
5
|
+
export type { SelfIdentifiedOptions } from "./self-identified.js";
|
|
6
|
+
export { crawlerVerificationDetector } from "./crawler-verification.js";
|
|
7
|
+
export type { CrawlerVerificationOptions } from "./crawler-verification.js";
|
|
8
|
+
export { headerIntegrityDetector } from "./header-integrity.js";
|
|
9
|
+
export type { HeaderIntegrityOptions } from "./header-integrity.js";
|
|
10
|
+
export { clientHintsDetector } from "./client-hints.js";
|
|
11
|
+
export { fetchMetadataDetector } from "./fetch-metadata.js";
|
|
12
|
+
export { acceptSignatureDetector } from "./accept-signature.js";
|
|
13
|
+
export { headerOrderDetector, headerOrderFingerprint } from "./header-order.js";
|
|
14
|
+
export type { HeaderOrderOptions } from "./header-order.js";
|
|
15
|
+
export { rateAnomalyDetector } from "./rate-anomaly.js";
|
|
16
|
+
export type { RateAnomalyOptions } from "./rate-anomaly.js";
|
|
17
|
+
export { cadenceDetector } from "./cadence.js";
|
|
18
|
+
export type { CadenceOptions } from "./cadence.js";
|
|
19
|
+
export { crawlBreadthDetector } from "./crawl-breadth.js";
|
|
20
|
+
export type { CrawlBreadthOptions } from "./crawl-breadth.js";
|
|
21
|
+
export { sessionIntegrityDetector } from "./session-integrity.js";
|
|
22
|
+
export type { SessionIntegrityOptions } from "./session-integrity.js";
|
|
23
|
+
export { identityRotationDetector } from "./identity-rotation.js";
|
|
24
|
+
export type { IdentityRotationOptions } from "./identity-rotation.js";
|
|
25
|
+
export { TRAP_FIELD_SOURCE, trapDetector, renderTrapLink, renderTrapField, trapRobotsEntries, DEFAULT_TRAP_PATHS } from "./trap.js";
|
|
26
|
+
export type { TrapOptions, TrapLinkOptions } from "./trap.js";
|
|
27
|
+
export { ipIntelligenceDetector } from "./ip-intelligence.js";
|
|
28
|
+
export type { IpIntelligenceOptions } from "./ip-intelligence.js";
|
|
29
|
+
export { tlsFingerprintDetector } from "./tls-fingerprint.js";
|
|
30
|
+
export type { TlsFingerprintOptions, FingerprintProfile } from "./tls-fingerprint.js";
|
|
31
|
+
export { clearanceDetector } from "./clearance.js";
|
|
32
|
+
export { uaCoherenceDetector } from "./ua-coherence.js";
|
|
33
|
+
export { probeSignatureDetector } from "./probe-signature.js";
|
|
34
|
+
export type { ProbeSignatureOptions } from "./probe-signature.js";
|
|
35
|
+
export { browsingCoherenceDetector } from "./browsing-coherence.js";
|
|
36
|
+
export { clientSignalsDetector } from "./client-signals.js";
|
|
37
|
+
export type { ClientSignalsOptions } from "./client-signals.js";
|
|
38
|
+
export { BOT_SIGNATURES, BENIGN_CATEGORIES, compileSignatures, indexSignatures } from "./known-bots.js";
|
|
39
|
+
export type { BotSignature, BotCategory, Verification } from "./known-bots.js";
|
|
40
|
+
/**
|
|
41
|
+
* The detector set installed when you configure none.
|
|
42
|
+
*
|
|
43
|
+
* Two are missing on purpose, and both omissions are about false positives rather
|
|
44
|
+
* than about cost:
|
|
45
|
+
*
|
|
46
|
+
* - `identityRotationDetector` fires on any address that fronts several browsers,
|
|
47
|
+
* which describes every corporate NAT and mobile carrier on the internet. It is
|
|
48
|
+
* valuable, but only once your `actorKey` is narrower than an IP.
|
|
49
|
+
* - `tlsFingerprintDetector` needs a fingerprint from your edge and a profile table
|
|
50
|
+
* you maintain. With neither it is inert; with a stale table it misfires on anyone
|
|
51
|
+
* running a browser newer than your data.
|
|
52
|
+
*
|
|
53
|
+
* `clearanceDetector` is not here either, because it needs the challenge service —
|
|
54
|
+
* the engine adds it automatically once `challenge.secrets` is configured.
|
|
55
|
+
*/
|
|
56
|
+
export declare function defaultDetectors(): Detector[];
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface IpIntelligenceOptions {
|
|
3
|
+
/** Weight for a datacenter-range match. Default 0.3 (`moderate`). */
|
|
4
|
+
datacenterWeight?: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* What do we know about where this request came from?
|
|
8
|
+
*
|
|
9
|
+
* Two range sets are consulted, and they are treated very differently.
|
|
10
|
+
*
|
|
11
|
+
* **`denylist`** is `certain`, and the justification is not technical. You configured
|
|
12
|
+
* it. The library is not inferring anything; it is carrying out an instruction you
|
|
13
|
+
* gave about addresses you have decided about. Certainty here means "this is a
|
|
14
|
+
* decision, not a guess" — and it means a bad entry in your denylist blocks real
|
|
15
|
+
* people, which is exactly why the range is required to be explicit rather than
|
|
16
|
+
* inherited from a feed by default.
|
|
17
|
+
*
|
|
18
|
+
* **`datacenter`** is `moderate` and always will be. Hosting-provider address space
|
|
19
|
+
* is where scrapers live, and it is *also* where every consumer VPN, every corporate
|
|
20
|
+
* egress gateway, every Tor exit, every privacy relay like iCloud Private Relay, and
|
|
21
|
+
* a growing share of mobile traffic lives. Treating "came from AWS" as proof of
|
|
22
|
+
* automation blocks a meaningful slice of ordinary users, disproportionately the
|
|
23
|
+
* privacy-conscious ones. As corroboration alongside a header failure it is
|
|
24
|
+
* genuinely useful; alone it is close to worthless.
|
|
25
|
+
*
|
|
26
|
+
* No range data ships with this library. Address-to-operator mappings go stale
|
|
27
|
+
* within weeks, and a stale mapping is a false positive with a long half-life —
|
|
28
|
+
* supply your own, from a source you refresh and can audit.
|
|
29
|
+
*/
|
|
30
|
+
export declare function ipIntelligenceDetector(options?: IpIntelligenceOptions): Detector;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { MultiPatternMatcher } from "../internal/matcher.js";
|
|
2
|
+
/**
|
|
3
|
+
* The known-bot signature database.
|
|
4
|
+
*
|
|
5
|
+
* Two things live here, and it is worth being precise about the difference.
|
|
6
|
+
*
|
|
7
|
+
* A **signature** matches a literal token in a User-Agent. That is a *claim* the
|
|
8
|
+
* client makes about itself, nothing more — anyone can send `Googlebot/2.1`. A claim
|
|
9
|
+
* is useful for two opposite reasons: an honest bot's claim tells us what it is, and
|
|
10
|
+
* a dishonest one gives us something to disprove.
|
|
11
|
+
*
|
|
12
|
+
* A **verification** is how that claim gets checked against an authority outside the
|
|
13
|
+
* request. Only the pairing of the two produces a `certain` verdict — either
|
|
14
|
+
* `verified-bot` (claim confirmed) or `impersonator` (claim disproved). A signature
|
|
15
|
+
* with no verification can never do better than `declared-bot`.
|
|
16
|
+
*/
|
|
17
|
+
export type BotCategory = "search" | "ai" | "seo" | "social" | "monitoring" | "archive" | "feed" | "security" | "advertising" | "library" | "headless"
|
|
18
|
+
/** A real browser engine embedded in a desktop application, with a person driving it. */
|
|
19
|
+
| "embedded" | "other";
|
|
20
|
+
/** Every category, for anything that has to enumerate them — a rule editor, a report. */
|
|
21
|
+
export declare const BOT_CATEGORIES: readonly BotCategory[];
|
|
22
|
+
/**
|
|
23
|
+
* How a claimed identity is checked.
|
|
24
|
+
*
|
|
25
|
+
* - `fcrdns` — forward-confirmed reverse DNS. Reverse-resolve the client IP, require
|
|
26
|
+
* the name to sit under one of `domains`, then forward-resolve that name and
|
|
27
|
+
* require the original IP back. Forging this requires control of the operator's
|
|
28
|
+
* DNS, which is the property that makes it `certain` in both directions.
|
|
29
|
+
* - `ip-ranges` — the operator publishes IP ranges but sets no usable PTR records.
|
|
30
|
+
* Verifiable only if you supply the ranges (see `crawlerRanges` in the config);
|
|
31
|
+
* the library will not fetch them for you, because a detector that makes an
|
|
32
|
+
* outbound HTTP request on a schedule is a dependency you should opt into
|
|
33
|
+
* knowingly.
|
|
34
|
+
* - `none` — no published mechanism. The claim is unfalsifiable, so we neither
|
|
35
|
+
* confirm nor accuse.
|
|
36
|
+
*/
|
|
37
|
+
export type Verification = {
|
|
38
|
+
kind: "fcrdns";
|
|
39
|
+
domains: readonly string[];
|
|
40
|
+
} | {
|
|
41
|
+
kind: "ip-ranges";
|
|
42
|
+
publishedAt?: string;
|
|
43
|
+
} | {
|
|
44
|
+
kind: "none";
|
|
45
|
+
};
|
|
46
|
+
export interface BotSignature {
|
|
47
|
+
/** Stable id, used in rules, logs and metrics. */
|
|
48
|
+
id: string;
|
|
49
|
+
name: string;
|
|
50
|
+
/** Lowercase literal tokens; a match on any one identifies this bot. */
|
|
51
|
+
tokens: readonly string[];
|
|
52
|
+
category: BotCategory;
|
|
53
|
+
verification: Verification;
|
|
54
|
+
/**
|
|
55
|
+
* Whether this client is generally *benign* — something most sites want to keep
|
|
56
|
+
* serving. Search and social preview crawlers are; scrapers and scanners are not.
|
|
57
|
+
* Drives the default policy's allow path, never a block.
|
|
58
|
+
*/
|
|
59
|
+
benign: boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Whether matching this signature is *conclusive* evidence of automation. Default
|
|
62
|
+
* true.
|
|
63
|
+
*
|
|
64
|
+
* Set it false when the token can legitimately appear on a request a person made.
|
|
65
|
+
* The motivating case is Electron: the UA is emitted by VS Code's Simple Browser,
|
|
66
|
+
* Slack, Discord and Postman, which are real Chromium instances with a human
|
|
67
|
+
* driving them. A signature match is only allowed into the `certain` tier when the
|
|
68
|
+
* claim "no honest client sends this" actually holds, and for embedded webviews it
|
|
69
|
+
* plainly does not.
|
|
70
|
+
*/
|
|
71
|
+
conclusive?: boolean;
|
|
72
|
+
/** Why this signature is not conclusive. Surfaced in the evidence summary. */
|
|
73
|
+
caveat?: string;
|
|
74
|
+
/**
|
|
75
|
+
* The product token to name in a `User-agent:` line in `robots.txt`.
|
|
76
|
+
*
|
|
77
|
+
* Defaults to the first match token. Worth setting where the canonical spelling
|
|
78
|
+
* differs from the token we match on — `robots.txt` matching is case-insensitive,
|
|
79
|
+
* so this is about a file a person will read, not about correctness.
|
|
80
|
+
*/
|
|
81
|
+
robotsAgent?: string;
|
|
82
|
+
docs?: string;
|
|
83
|
+
}
|
|
84
|
+
/** Every built-in signature, in one array. Extend it via `extraSignatures` rather than editing. */
|
|
85
|
+
export declare const BOT_SIGNATURES: readonly BotSignature[];
|
|
86
|
+
/**
|
|
87
|
+
* Compiles signatures into a single-pass matcher. Build this once per engine, never
|
|
88
|
+
* per request — construction is O(total pattern length) and matching is O(input).
|
|
89
|
+
*/
|
|
90
|
+
export declare function compileSignatures(signatures?: readonly BotSignature[]): MultiPatternMatcher<BotSignature>;
|
|
91
|
+
/** Index by id, so rules can name a bot (`"googlebot"`) and get its metadata back. */
|
|
92
|
+
export declare function indexSignatures(signatures?: readonly BotSignature[]): ReadonlyMap<string, BotSignature>;
|
|
93
|
+
/** Categories whose members are, by default, worth serving. Used by `allowBenignBots`. */
|
|
94
|
+
export declare const BENIGN_CATEGORIES: ReadonlySet<BotCategory>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* What is this request *asking for*?
|
|
4
|
+
*
|
|
5
|
+
* Every other detector here reads the client. This one reads the request target, and
|
|
6
|
+
* it exists because of a gap the rest of the library cannot close: the scanner that
|
|
7
|
+
* does not announce itself. `self-identified` catches sqlmap and Nikto because they
|
|
8
|
+
* say so, and a great deal of hostile traffic does say so. The rest arrives wearing a
|
|
9
|
+
* copied Chrome User-Agent, and from the headers alone it is indistinguishable from a
|
|
10
|
+
* person — until you look at what it asked for, which is `/.env`, then
|
|
11
|
+
* `/.git/config`, then `/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php`.
|
|
12
|
+
*
|
|
13
|
+
* Those paths are not obscure corners of a site. They are entries in a wordlist. No
|
|
14
|
+
* link points at them, no menu leads to them, and no person types them — the traffic
|
|
15
|
+
* that requests them is running a list against every address it can reach.
|
|
16
|
+
*
|
|
17
|
+
* **Why this stops at `strong` and never reaches `certain`.** The library's rule is
|
|
18
|
+
* that `certain` evidence must admit no benign explanation, and a *request target*
|
|
19
|
+
* always admits one: a URL is client-supplied text, and the client supplying it might
|
|
20
|
+
* be a security engineer testing their own site from a laptop, a researcher with a
|
|
21
|
+
* bug-bounty scope, or a monitoring check somebody wrote at three in the morning.
|
|
22
|
+
* Those are people, and the correct response to them is a challenge, not a closed
|
|
23
|
+
* door. A trap path is different — it is unreachable *by construction*, which is
|
|
24
|
+
* exactly the property a wordlist entry lacks.
|
|
25
|
+
*
|
|
26
|
+
* **What it does not do.** It reads one request at a time. A scanner walking a
|
|
27
|
+
* wordlist produces one of these observations per request rather than an escalating
|
|
28
|
+
* series, because nothing here remembers that the same actor asked for `/.env` a
|
|
29
|
+
* moment ago — the detectors that watch an actor over time are `rate-anomaly`,
|
|
30
|
+
* `cadence` and `crawl-breadth`, and enumeration shows up there. Keeping this one
|
|
31
|
+
* stateless is what lets it run unchanged over a log file in `replay`.
|
|
32
|
+
*
|
|
33
|
+
* **The one thing to configure.** The `platform` tier below is a list of admin and
|
|
34
|
+
* login paths that are probes on most sites and are the *front door* on the sites
|
|
35
|
+
* that run those platforms. If you run WordPress, `/wp-login.php` is where your
|
|
36
|
+
* authors sign in, and this detector will report each of them as a probe. Add such
|
|
37
|
+
* paths to `ignore`, or to the engine's `ignorePaths`, before turning any of this into
|
|
38
|
+
* a rule. The tier is capped at `moderate` precisely so that forgetting costs a
|
|
39
|
+
* tag rather than a door.
|
|
40
|
+
*/
|
|
41
|
+
export interface ProbeSignatureOptions {
|
|
42
|
+
/**
|
|
43
|
+
* Paths this site genuinely serves, exempted from the platform tier. Matched as
|
|
44
|
+
* prefixes, so `/wp-admin` covers everything beneath it.
|
|
45
|
+
*/
|
|
46
|
+
ignore?: readonly string[];
|
|
47
|
+
/** Additional exact paths or prefixes to treat as exploit-tier probes. */
|
|
48
|
+
extraPaths?: readonly string[];
|
|
49
|
+
}
|
|
50
|
+
export declare function probeSignatureDetector(options?: ProbeSignatureOptions): Detector;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
export interface RateAnomalyOptions {
|
|
3
|
+
/** Window over which arrivals are counted, ms. Default 10000. */
|
|
4
|
+
windowMs?: number;
|
|
5
|
+
/** Requests in the window above which the rate is worth noting. Default 20. */
|
|
6
|
+
threshold?: number;
|
|
7
|
+
/**
|
|
8
|
+
* Requests in the window above which the rate is well beyond human. Default 30.
|
|
9
|
+
*
|
|
10
|
+
* Both defaults sit under the per-actor timestamp ring's capacity, because the
|
|
11
|
+
* count saturates there — a threshold above it could never be reached, which is a
|
|
12
|
+
* detector that silently never fires. If you raise these, raise them knowing the
|
|
13
|
+
* ceiling, and reach for the `rate-limit` action instead when you need real numbers.
|
|
14
|
+
*/
|
|
15
|
+
hardThreshold?: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* How fast is this actor going?
|
|
19
|
+
*
|
|
20
|
+
* Rate is the signal people reach for first and trust the most, and it deserves the
|
|
21
|
+
* least trust of anything in this library. The reason is that the *actor* behind a
|
|
22
|
+
* high rate is frequently not one client: a corporate NAT, a mobile carrier's CGNAT
|
|
23
|
+
* pool, a university, a VPN exit and a shared office all present hundreds of real
|
|
24
|
+
* people as one address. Blocking on rate blocks all of them.
|
|
25
|
+
*
|
|
26
|
+
* So this detector reports and never concludes. Its output tops out at `moderate`,
|
|
27
|
+
* which under the default policy cannot reach a terminal action no matter how
|
|
28
|
+
* extreme the number gets. What high rate is genuinely good for is *corroborating* —
|
|
29
|
+
* a client that already looks like a library and is also pulling 300 requests a
|
|
30
|
+
* minute is a different proposition from either fact alone.
|
|
31
|
+
*
|
|
32
|
+
* If you want rate to actually stop traffic, that is what rate *limiting* is for,
|
|
33
|
+
* and it belongs in the action layer where it applies to everyone equally and
|
|
34
|
+
* recovers on its own. See the `rate-limit` action.
|
|
35
|
+
*/
|
|
36
|
+
export declare function rateAnomalyDetector(options?: RateAnomalyOptions): Detector;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Detector } from "./types.js";
|
|
2
|
+
import type { BotCategory } from "./known-bots.js";
|
|
3
|
+
export interface SelfIdentifiedOptions {
|
|
4
|
+
/**
|
|
5
|
+
* Categories to report. Removing one does not make its traffic invisible — the
|
|
6
|
+
* behavioural detectors still see it — it only stops this detector naming it.
|
|
7
|
+
*/
|
|
8
|
+
categories?: readonly BotCategory[];
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The client told us what it is.
|
|
12
|
+
*
|
|
13
|
+
* This detector is the backbone of the `certain` tier, and the reason is a point
|
|
14
|
+
* about responsibility rather than about technology. When a request arrives saying
|
|
15
|
+
* `python-requests/2.31.0` or `Googlebot/2.1`, we are not *inferring* anything. We
|
|
16
|
+
* are taking the client at its word. If that word is a lie, the misclassification is
|
|
17
|
+
* the client's doing, not a failure of detection — and no honest client is ever
|
|
18
|
+
* harmed by being believed.
|
|
19
|
+
*
|
|
20
|
+
* That is why a self-declaration can safely gate a terminal action while a much
|
|
21
|
+
* "smarter" behavioural inference cannot. The behavioural inference can be wrong
|
|
22
|
+
* about someone who never made any claim at all.
|
|
23
|
+
*
|
|
24
|
+
* Two shapes qualify:
|
|
25
|
+
*
|
|
26
|
+
* - A **known signature** — a token from {@link BOT_SIGNATURES}. Library and headless
|
|
27
|
+
* tokens are conclusive on their own: no browser has ever sent `curl/8.4.0`.
|
|
28
|
+
* - An **unrecognised but self-announcing** UA — contains a word like `bot` or
|
|
29
|
+
* `crawler` *and* publishes a contact URL or email, the long-standing convention
|
|
30
|
+
* for well-behaved crawlers. Either half alone is only suggestive; together they
|
|
31
|
+
* are a declaration.
|
|
32
|
+
*/
|
|
33
|
+
export declare function selfIdentifiedDetector(options?: SelfIdentifiedOptions): Detector;
|