@osqd/bothandlerjs 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +227 -1
  2. package/README.md +18 -10
  3. package/dist/adapters/fastify.d.ts +10 -0
  4. package/dist/adapters/index.cjs +38 -10
  5. package/dist/adapters/index.cjs.map +1 -1
  6. package/dist/adapters/index.js +38 -10
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/challenge/index.d.ts +40 -0
  9. package/dist/cli.cjs +2256 -103
  10. package/dist/cli.cjs.map +1 -1
  11. package/dist/cli.js +2256 -103
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts +55 -0
  14. package/dist/core.d.ts +48 -21
  15. package/dist/corpus/index.cjs +365 -7
  16. package/dist/corpus/index.cjs.map +1 -1
  17. package/dist/corpus/index.js +365 -7
  18. package/dist/corpus/index.js.map +1 -1
  19. package/dist/corpus/schema.d.ts +33 -0
  20. package/dist/crawler-ranges.d.ts +31 -0
  21. package/dist/dashboard/client/actions.d.ts +1 -1
  22. package/dist/dashboard/client/app.d.ts +9 -2
  23. package/dist/dashboard/client/boot.d.ts +32 -3
  24. package/dist/dashboard/client/query.d.ts +72 -12
  25. package/dist/dashboard/client/registry.d.ts +25 -0
  26. package/dist/dashboard/client/saved.d.ts +29 -0
  27. package/dist/dashboard/client/store.d.ts +16 -2
  28. package/dist/dashboard/client/types.d.ts +2 -0
  29. package/dist/dashboard/client.generated.d.ts +1 -1
  30. package/dist/dashboard/types.d.ts +15 -0
  31. package/dist/detectors/blended-identity.d.ts +34 -0
  32. package/dist/detectors/challenge-integrity.d.ts +26 -0
  33. package/dist/detectors/challenge-reaction.d.ts +39 -0
  34. package/dist/detectors/clearance.d.ts +1 -23
  35. package/dist/detectors/id-enumeration.d.ts +31 -0
  36. package/dist/detectors/index.d.ts +24 -1
  37. package/dist/detectors/known-bots.d.ts +11 -0
  38. package/dist/detectors/marker.d.ts +106 -0
  39. package/dist/detectors/parameter-sweep.d.ts +39 -0
  40. package/dist/detectors/probe-signature.d.ts +27 -0
  41. package/dist/detectors/probe-volume.d.ts +26 -0
  42. package/dist/detectors/site-baseline.d.ts +135 -0
  43. package/dist/detectors/target-integrity.d.ts +16 -0
  44. package/dist/detectors/transport-coherence.d.ts +31 -0
  45. package/dist/detectors/trap.d.ts +10 -3
  46. package/dist/detectors/types.d.ts +17 -0
  47. package/dist/element/index.cjs +730 -80
  48. package/dist/element/index.cjs.map +1 -1
  49. package/dist/element/index.js +730 -80
  50. package/dist/element/index.js.map +1 -1
  51. package/dist/index.cjs +2044 -123
  52. package/dist/index.cjs.map +1 -1
  53. package/dist/index.d.ts +6 -2
  54. package/dist/index.js +2025 -123
  55. package/dist/index.js.map +1 -1
  56. package/dist/internal/async.d.ts +0 -3
  57. package/dist/internal/ip.d.ts +18 -0
  58. package/dist/internal/text.d.ts +28 -0
  59. package/dist/metrics.d.ts +18 -0
  60. package/dist/probe/index.d.ts +153 -0
  61. package/dist/probe/marker.d.ts +119 -0
  62. package/dist/site/index.d.ts +122 -0
  63. package/dist/state.d.ts +205 -0
  64. package/dist/stores/redis.d.ts +24 -1
  65. package/dist/types.d.ts +106 -0
  66. package/docs/course/05-detectors.md +9 -4
  67. package/docs/course/06-identity.md +1 -1
  68. package/docs/course/16-proving-it.md +15 -9
  69. package/docs/course/index.md +1 -1
  70. package/docs/design/decisions.md +1 -1
  71. package/docs/detection/correlation.md +284 -0
  72. package/docs/detection/detectors.md +259 -1
  73. package/docs/detection/index.md +2 -1
  74. package/docs/detection/shadow-mode.md +147 -0
  75. package/docs/detection/signatures.md +10 -2
  76. package/docs/index.md +3 -2
  77. package/docs/integration/client-ip.md +16 -0
  78. package/docs/operations/dashboard.md +40 -1
  79. package/docs/operations/filters.md +143 -0
  80. package/docs/operations/index.md +1 -0
  81. package/docs/operations/metrics.md +18 -0
  82. package/docs/policy/presets.md +1 -1
  83. package/docs/start/choosing-a-policy.md +1 -1
  84. package/docs/start/first-integration.md +1 -1
  85. package/docs/start/installation.md +2 -2
  86. package/docs/testing/cli.md +7 -1
  87. package/docs/testing/corpus.md +12 -8
  88. package/docs/testing/index.md +1 -1
  89. package/docs/testing/try-it.md +1 -1
  90. package/package.json +4 -1
@@ -443,6 +443,18 @@ export interface DashboardEntry {
443
443
  downgradedFrom?: ActionName | undefined;
444
444
  downgradeReason?: string | undefined;
445
445
  evidence: DashboardEvidence[];
446
+ /**
447
+ * What this request would have been had the shadowed detectors counted.
448
+ *
449
+ * Absent unless a shadowed detector found something, which is also when the `shadow`
450
+ * flag appears on one of the evidence entries above.
451
+ */
452
+ shadowVerdict?: {
453
+ verdict: Verdict;
454
+ botClass: BotClass;
455
+ score: number;
456
+ certain: boolean;
457
+ } | undefined;
446
458
  failures: Array<{
447
459
  detector: string;
448
460
  reason: string;
@@ -462,6 +474,8 @@ export interface DashboardEvidence {
462
474
  summary: string;
463
475
  certainty: Certainty;
464
476
  direction: EvidenceDirection;
477
+ /** From a shadowed detector: shown, counted, and part of no decision. */
478
+ shadow?: true | undefined;
465
479
  family?: string | undefined;
466
480
  deterministicBasis?: string | undefined;
467
481
  /**
@@ -541,6 +555,7 @@ export interface DashboardSnapshot {
541
555
  description: string;
542
556
  cost: string;
543
557
  stage: string;
558
+ shadow?: true | undefined;
544
559
  }>;
545
560
  rules: readonly string[];
546
561
  ranges: Array<{
@@ -0,0 +1,34 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface BlendedIdentityOptions {
3
+ /** Distinct security-tool identities from one actor before it is reported. Default 2. */
4
+ scannerIdentities?: number;
5
+ /** Distinct verifiable crawler identities before it is reported. Default 2. */
6
+ crawlerIdentities?: number;
7
+ }
8
+ /**
9
+ * What a *series* of requests claimed, read against itself.
10
+ *
11
+ * Every identity check in this library reads one request: this User-Agent names this bot,
12
+ * and that claim is either confirmable or it is not. The set of claims an actor has made
13
+ * over time is a different object, and some sets are self-contradictory in a way no single
14
+ * member of them is.
15
+ *
16
+ * Unlike `identity-rotation`, these hold up under the default address-based actor key,
17
+ * which is the whole reason they are on by default and that one is not. A NAT gateway
18
+ * presents many browsers — that is what makes counting User-Agents useless there. It does
19
+ * not present sqlmap *and* nikto, and it does not claim to be Googlebot *and* Bingbot. The
20
+ * innocent explanation for a hundred browsers behind one address is an office; there is no
21
+ * corresponding innocent explanation for these.
22
+ *
23
+ * Three readings, and each is about a combination rather than a claim:
24
+ *
25
+ * - **Several security tools.** One address arriving as two or more named scanners is a
26
+ * scan, not a coincidence. The tools announce themselves honestly, which is what makes
27
+ * the *set* readable even though each member is only `declared-bot` on its own.
28
+ * - **Several verifiable crawlers.** At most one of Googlebot, Bingbot and Yandex can be
29
+ * true of an address, because each publishes a proof tied to addresses it controls.
30
+ * Claiming two is a forgery whether or not either was checked.
31
+ * - **A crawler that also probes.** An actor that sent a scanner payload and also claimed
32
+ * to be a search crawler has told you which of the two is the lie.
33
+ */
34
+ export declare function blendedIdentityDetector(options?: BlendedIdentityOptions): Detector;
@@ -0,0 +1,26 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Answers to challenges that were well-formed and still wrong.
4
+ *
5
+ * The verification endpoint is deliberately not assessed — a client that has just been
6
+ * challenged must not be challenged again for trying to answer — so nothing else in the
7
+ * pipeline ever sees what happens there. Two of those outcomes say something about the
8
+ * client rather than about the request, and this is where they surface.
9
+ *
10
+ * **A solution submitted twice.** A challenge nonce is random, single-use and signed, so
11
+ * a second valid solution for one is not a coincidence: it is the same answer sent
12
+ * again, or one answer being shared out. The threshold is not one, because a flaky
13
+ * network and a retried POST produce exactly one.
14
+ *
15
+ * **A solution returned faster than the puzzle allows.** The work is measured on this
16
+ * server between issuing and receiving, so no client clock is involved, and the floor is
17
+ * set at a SHA-256 rate no browser has ever reached. Coming in under it means the answer
18
+ * was not computed by the script we sent.
19
+ */
20
+ export interface ChallengeIntegrityOptions {
21
+ /** Replayed solutions before this says anything. Default 3. */
22
+ minReplays?: number;
23
+ /** Implausibly fast solutions before this says anything. Default 1. */
24
+ minImplausible?: number;
25
+ }
26
+ export declare function challengeIntegrityDetector(options?: ChallengeIntegrityOptions): Detector;
@@ -0,0 +1,39 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * What a client does when it is asked to prove something.
4
+ *
5
+ * Every other detector here observes traffic that would have happened anyway, and has
6
+ * to argue backwards from it. This one reads a reaction to a stimulus **we chose**, and
7
+ * that difference is what makes it worth having: we decided when the challenge went
8
+ * out, so a client that changes what it claims to be within seconds of receiving one is
9
+ * responding to it. There was no reason to look at that moment other than that we made
10
+ * it happen.
11
+ *
12
+ * Two reactions are worth reporting, and they mean different things.
13
+ *
14
+ * **Changing identity.** A client challenged as Chrome that returns as curl, or as
15
+ * Googlebot, is trying a different disguise to see whether the door opens. Software
16
+ * does not change what it is; an operator changes it, and the only reason to change it
17
+ * at that moment is the challenge.
18
+ *
19
+ * **Never answering, repeatedly.** A person who abandons one challenge is ordinary — a
20
+ * slow phone, a lost tab, a change of mind. A client asked five times that has never
21
+ * once come back is not abandoning: it is unable or unwilling, and both are facts about
22
+ * software rather than about a person's patience.
23
+ *
24
+ * **Why neither reaches `certain`, and why one is weaker than the other.** Both join
25
+ * two requests, and the join is the weak link. With a marker the join is cryptographic
26
+ * and the two requests provably came from one client. Without one they are tied by
27
+ * address alone, and a busy NAT will eventually put a different person's browser in the
28
+ * seconds after somebody else was challenged — so the same observation is reported a
29
+ * tier lower, because that is genuinely how much less it is worth. Never answering is
30
+ * capped lower still: a person with JavaScript disabled produces it forever, and they
31
+ * are a person.
32
+ */
33
+ export interface ChallengeReactionOptions {
34
+ /** How soon after a challenge a change of identity counts as a reaction. Default 60s. */
35
+ windowMs?: number;
36
+ /** Unanswered challenges before that is worth reporting. Default 4. */
37
+ minUnsolved?: number;
38
+ }
39
+ export declare function challengeReactionDetector(options?: ChallengeReactionOptions): Detector;
@@ -1,25 +1,3 @@
1
1
  import type { Detector } from "./types.js";
2
2
  import type { ChallengeService } from "../challenge/index.js";
3
- /**
4
- * Reads a clearance token the client already holds.
5
- *
6
- * This is the library's only source of *human*-pointing evidence that is not a
7
- * guess about headers, and it is worth being exact about what each level earns,
8
- * because overstating any of them would undo the whole design.
9
- *
10
- * - **`operator`** is `certain`. Your application told us this is a person — an
11
- * authenticated session, a completed purchase, whatever your own bar is. We are
12
- * believing you, not deducing anything, which is the same reasoning that makes a
13
- * client's self-declaration `certain` in the other direction.
14
- * - **`interaction`** is `strong`. A trusted input event was observed. Automation
15
- * driving a real browser can synthesise something close, so this is very good
16
- * evidence and not proof.
17
- * - **`pow`** is only `moderate`, and this is the number people are most tempted to
18
- * inflate. A solved proof of work shows a JavaScript engine ran and CPU was spent.
19
- * A headless Chrome does both, happily and at scale. It raises the cost of a scrape
20
- * substantially; it says nothing whatsoever about whether a human is present.
21
- *
22
- * The token is bound to the actor and signed, so it cannot be lifted from one client
23
- * and replayed by another under a different actor key.
24
- */
25
- export declare function clearanceDetector(service: ChallengeService): Detector;
3
+ export declare function clearanceDetector(service: ChallengeService, sharingThreshold?: number): Detector;
@@ -0,0 +1,31 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface IdEnumerationOptions {
3
+ /** Requests to one path shape before a walk is worth reporting. Default 30. */
4
+ minRequests?: number;
5
+ /**
6
+ * How completely those requests must cover the range they span. Default 0.9 — thirty
7
+ * requests reaching from id 1 to id 33 report; the same thirty scattered across a
8
+ * thousand ids do not.
9
+ */
10
+ density?: number;
11
+ }
12
+ /**
13
+ * Somebody working through the identifiers rather than following the links.
14
+ *
15
+ * `crawl-breadth` sees this as "many distinct paths", which is what it also sees when a
16
+ * person reads a documentation site — so it stays `weak` and nothing separates the two.
17
+ * Measured: `/user/1` through `/user/120` in order scored exactly the same as a hundred
18
+ * and twenty scattered ids, and the same again as ordinary article paths. All three
19
+ * `unknown`, all three 57.
20
+ *
21
+ * What separates them is not which ids were asked for but whether they *cover a range*.
22
+ * People arrive at ids through links, and links do not densely enumerate an integer
23
+ * interval; a harvester does nothing else. Thirty requests reaching from id 1 to id 33 is
24
+ * a walk. Thirty scattered across a hundred thousand is somebody reading.
25
+ *
26
+ * `moderate`, and the bar is set high on purpose. The awkward case is real: products in
27
+ * one category often carry consecutive ids, so somebody browsing a catalogue can produce a
28
+ * smaller version of this. Thirty requests covering ninety per cent of their own span is
29
+ * meant to be past what that produces, and it is still a shape rather than a motive.
30
+ */
31
+ export declare function idEnumerationDetector(options?: IdEnumerationOptions): Detector;
@@ -18,7 +18,25 @@ export type { RateAnomalyOptions } from "./rate-anomaly.js";
18
18
  export { cadenceDetector } from "./cadence.js";
19
19
  export type { CadenceOptions } from "./cadence.js";
20
20
  export { crawlBreadthDetector } from "./crawl-breadth.js";
21
+ export { parameterSweepDetector } from "./parameter-sweep.js";
22
+ export { transportCoherenceDetector } from "./transport-coherence.js";
23
+ export { probeVolumeDetector } from "./probe-volume.js";
24
+ export { idEnumerationDetector } from "./id-enumeration.js";
25
+ export { blendedIdentityDetector } from "./blended-identity.js";
26
+ export { challengeReactionDetector } from "./challenge-reaction.js";
27
+ export type { ChallengeReactionOptions } from "./challenge-reaction.js";
28
+ export { challengeIntegrityDetector } from "./challenge-integrity.js";
29
+ export { distributedWalkDetector, pathNoveltyDetector, missBaselineDetector, pathCampaignDetector } from "./site-baseline.js";
30
+ export type { DistributedWalkOptions, PathNoveltyOptions, MissBaselineOptions, PathCampaignOptions } from "./site-baseline.js";
31
+ export type { ChallengeIntegrityOptions } from "./challenge-integrity.js";
32
+ export { identityDriftDetector, markerIntegrityDetector, markerPersistenceDetector, markerFanoutDetector } from "./marker.js";
33
+ export type { IdentityDriftOptions, MarkerIntegrityOptions, MarkerPersistenceOptions, MarkerFanoutOptions } from "./marker.js";
21
34
  export type { CrawlBreadthOptions } from "./crawl-breadth.js";
35
+ export type { ParameterSweepOptions } from "./parameter-sweep.js";
36
+ export type { TransportCoherenceOptions } from "./transport-coherence.js";
37
+ export type { ProbeVolumeOptions } from "./probe-volume.js";
38
+ export type { IdEnumerationOptions } from "./id-enumeration.js";
39
+ export type { BlendedIdentityOptions } from "./blended-identity.js";
22
40
  export { sessionIntegrityDetector } from "./session-integrity.js";
23
41
  export type { SessionIntegrityOptions } from "./session-integrity.js";
24
42
  export { identityRotationDetector } from "./identity-rotation.js";
@@ -31,6 +49,8 @@ export { tlsFingerprintDetector } from "./tls-fingerprint.js";
31
49
  export type { TlsFingerprintOptions, FingerprintProfile } from "./tls-fingerprint.js";
32
50
  export { clearanceDetector } from "./clearance.js";
33
51
  export { uaCoherenceDetector } from "./ua-coherence.js";
52
+ export { targetIntegrityDetector } from "./target-integrity.js";
53
+ export type { TargetIntegrityOptions } from "./target-integrity.js";
34
54
  export { probeSignatureDetector } from "./probe-signature.js";
35
55
  export type { ProbeSignatureOptions } from "./probe-signature.js";
36
56
  export { browsingCoherenceDetector } from "./browsing-coherence.js";
@@ -52,7 +72,10 @@ export type { BotSignature, BotCategory, Verification } from "./known-bots.js";
52
72
  * running a browser newer than your data.
53
73
  *
54
74
  * `clearanceDetector` is not here either, because it needs the challenge service —
55
- * the engine adds it automatically once `challenge.secrets` is configured.
75
+ * the engine adds it automatically once `challenge.secrets` is configured. The three
76
+ * marker detectors work the same way: with no `probe` there is no cookie to have been
77
+ * issued, so they would be three permanently silent entries in every deployment that
78
+ * does not use one, and the engine adds them once `probe` is.
56
79
  */
57
80
  export declare function defaultDetectors(options?: {
58
81
  crawlerVerification?: CrawlerVerificationOptions;
@@ -35,6 +35,17 @@ export type BotCategory = "search" | "ai" | "seo" | "social" | "monitoring" | "a
35
35
  * to reach the page, and frequently does not know it is arriving as a bot at all.
36
36
  */
37
37
  | "accessibility"
38
+ /**
39
+ * A mail or messaging gateway checking a link on somebody's behalf.
40
+ *
41
+ * Its own category because of who pays when it is blocked. A social preview that fails
42
+ * costs a card; one of these failing tells a real person, in their inbox, that the link
43
+ * they were sent could not be verified — and they were never the one crawling. They also
44
+ * arrive with none of a browser's marks: from a datacentre, once, with no cookie and no
45
+ * referer, moments after a message was delivered, which is a shape that reads as
46
+ * automation because it *is* automation, acting for a human.
47
+ */
48
+ | "email-security"
38
49
  /**
39
50
  * Research and measurement: universities, internet-measurement projects, plagiarism
40
51
  * and citation indexes.
@@ -0,0 +1,106 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * What the marker cookie said about a client across its requests.
4
+ *
5
+ * These three read the same observation from three angles, and none of them can say
6
+ * anything at all unless `probe` is configured — a marker nobody issued is a marker
7
+ * nobody can fail to return. When the probe is off they return `undefined` on every
8
+ * request, which is the correct answer rather than a silent failure: absence of a
9
+ * marker is absence of information.
10
+ *
11
+ * See `docs/detection/correlation.md` for why a marker exists and what it costs.
12
+ */
13
+ export interface IdentityDriftOptions {
14
+ /** Report a platform-only or language-only change. Default true, at `moderate`. */
15
+ reportSoftDrift?: boolean;
16
+ }
17
+ /**
18
+ * One client, two identities.
19
+ *
20
+ * This is the detector this library could not previously write. Correlating by address
21
+ * cannot distinguish "a client that claimed to be Chrome and then curl" from "two
22
+ * people behind one office connection", and guessing between them would either miss
23
+ * every rotation or accuse every shared network. A marker removes the ambiguity: both
24
+ * requests carried an HMAC only this server can produce, so they came from one client,
25
+ * and that client described itself two different ways.
26
+ *
27
+ * **The parts are weighed separately, because they are not equally suspicious.**
28
+ * A browser family that changes — Chrome to curl, Firefox to Googlebot — has no benign
29
+ * reading; software does not change what it is. A *platform* that changes does have
30
+ * one, and it is common: "Request desktop site" on a phone rewrites the User-Agent to
31
+ * claim a desktop, and the person doing it is a person. Language changes when someone
32
+ * changes their language. So the family carries the weight and the rest is reported
33
+ * softly, which is the difference between catching a rotation and blaming a visitor for
34
+ * using a browser feature.
35
+ */
36
+ export declare function identityDriftDetector(options?: IdentityDriftOptions): Detector;
37
+ export interface MarkerIntegrityOptions {
38
+ /** Ignored below this many forged presentations. Default 1. */
39
+ minForgeries?: number;
40
+ }
41
+ /**
42
+ * A marker that this server did not sign.
43
+ *
44
+ * Browsers do not edit their cookies. A marker that fails its HMAC was altered by
45
+ * whoever was holding it, and the only reason to alter an opaque signed value is to
46
+ * find out what the server does with a different one — which is what a scanner does and
47
+ * what a person browsing does not.
48
+ *
49
+ * **Why this stops at `strong`.** `certain` in this library means no benign explanation
50
+ * exists, and one does, thinly: a middlebox or a broken cookie jar can truncate or
51
+ * re-encode a value in transit. It is rare, it is not the client's fault, and it should
52
+ * cost a challenge rather than a door.
53
+ */
54
+ export declare function markerIntegrityDetector(options?: MarkerIntegrityOptions): Detector;
55
+ export interface MarkerPersistenceOptions {
56
+ /** Markers handed out with none returned before this says anything. Default 5. */
57
+ minIssued?: number;
58
+ }
59
+ /**
60
+ * A client that keeps cookies, but never ours.
61
+ *
62
+ * The obvious version of this detector reports any client that is handed a marker and
63
+ * never returns one — and that version is worth almost nothing, because
64
+ * `session-integrity` already reports a client that sends no cookie at all, and it does
65
+ * so with a better-calibrated weight. Two moderate signals for one observation is a
66
+ * double count, and the population it lands on is people who block cookies. Measured on
67
+ * the corpus, the overlapping version took `cookies-blocked` from 21 to 38 and put +24
68
+ * on five ordinary browsing sessions.
69
+ *
70
+ * So this asks the narrower question the marker can uniquely answer: the client is
71
+ * *demonstrably* keeping cookies — it sent some — and ours is not among them. A browser
72
+ * with a cookie jar puts every first-party cookie in it; a scraper replaying a captured
73
+ * session header sends the one cookie it was told to and stores nothing new.
74
+ *
75
+ * **Why `moderate` and never more.** A marker can go missing for reasons that are the
76
+ * operator's fault rather than the client's: `secure: true` on a page served over plain
77
+ * HTTP is never stored at all, and a `domain` that does not match the host is not sent
78
+ * back. Both would produce this for every visitor, which is exactly why it may never
79
+ * deny anybody on its own. Verified crawlers are never issued a marker, so they never
80
+ * appear here.
81
+ */
82
+ export declare function markerPersistenceDetector(options?: MarkerPersistenceOptions): Detector;
83
+ export interface MarkerFanoutOptions {
84
+ /** Distinct networks one marker may be presented from before this says anything. Default 16. */
85
+ minNetworks?: number;
86
+ }
87
+ /**
88
+ * One client, many networks.
89
+ *
90
+ * A marker comes back only from the client that received it, so a marker presented from
91
+ * sixteen different networks is one client that has moved across sixteen networks. The
92
+ * shape that produces is a scraper on a rotating proxy pool that keeps its cookie jar —
93
+ * which most of them do, because discarding it breaks the sites they are scraping.
94
+ *
95
+ * **Why this is capped at `moderate` and offered no higher.** The honest reading is that
96
+ * this signal cannot separate a proxy pool from a heavily mobile person. A phone on a
97
+ * carrier using CGNAT can be renumbered across a great many `/24`s in the twelve hours a
98
+ * marker lives, and so can anyone whose employer egresses through a rotating pool. Those
99
+ * are people. The count is real and it is worth combining with everything else, and it
100
+ * is never worth denying somebody on by itself.
101
+ *
102
+ * The counting is per process and bounded, so across replicas each sees only its share.
103
+ * That direction is deliberate: it undercounts, and undercounting says nothing where
104
+ * overcounting would accuse somebody.
105
+ */
106
+ export declare function markerFanoutDetector(options?: MarkerFanoutOptions): Detector;
@@ -0,0 +1,39 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface ParameterSweepOptions {
3
+ /**
4
+ * Distinct path-and-query combinations at or above which a sweep is worth reporting.
5
+ * Default 25. Cannot exceed {@link MAX_TRACKED_QUERIES}, where the count saturates;
6
+ * asking for more throws rather than never firing.
7
+ */
8
+ threshold?: number;
9
+ /**
10
+ * How many variants must sit on one path before this is a sweep rather than browsing.
11
+ * Default 8 — that is, twenty-five variants across three paths reports, and
12
+ * twenty-five variants across twenty paths does not.
13
+ */
14
+ variantsPerPath?: number;
15
+ /** Minimum requests before the shape means anything. Default 20. */
16
+ minRequests?: number;
17
+ }
18
+ /**
19
+ * The scraping that `crawl-breadth` cannot see.
20
+ *
21
+ * Breadth counts distinct *paths*, and a path has no query string on it. So the shape
22
+ * it reads as "somebody rereading one page" is also the shape of enumerating a
23
+ * catalogue: `/products?page=1` through `?page=200` is one path and two hundred
24
+ * requests. Measured, on the same two hundred requests expressed both ways — as
25
+ * distinct paths it scored 62 and was called `suspected-bot`; as `?page=N` it scored 55
26
+ * and passed as `unknown`. Paginated collection is not an exotic case, it is how
27
+ * catalogues, search results and APIs are actually taken.
28
+ *
29
+ * So this counts the other thing: distinct parameterisations, and how many of them
30
+ * stack onto a single path. Both halves are needed. A high variant count alone is
31
+ * ordinary — a shop's own visitors filter and sort — and it is the *concentration* that
32
+ * separates a person changing their mind from a machine walking an index.
33
+ *
34
+ * `weak`, and deliberately. A person paging through search results produces a smaller
35
+ * version of exactly this, and someone with a slow connection retrying looks similar
36
+ * again. It is a shape, not a motive; its value is as a second signal beside an actor
37
+ * that has already failed something sharper.
38
+ */
39
+ export declare function parameterSweepDetector(options?: ParameterSweepOptions): Detector;
@@ -47,4 +47,31 @@ export interface ProbeSignatureOptions {
47
47
  /** Additional exact paths or prefixes to treat as exploit-tier probes. */
48
48
  extraPaths?: readonly string[];
49
49
  }
50
+ /**
51
+ * Payload shapes in a query value or a path segment.
52
+ *
53
+ * Deliberately narrow, and split by how much of the observation is the payload itself.
54
+ * A generic "looks like SQL" pattern fires on a search box — someone looking up
55
+ * `union select` in the documentation of a database — and a generic "looks like HTML"
56
+ * pattern fires on a CMS preview and on every forum thread about cross-site
57
+ * scripting. Those are people, reading a site about the thing the pattern describes.
58
+ *
59
+ * So `sql` and `markup` patterns are reported at `moderate` on their own and are
60
+ * promoted to `strong` only when the value also carries **injection punctuation** — a
61
+ * quote, a comment marker, a statement separator — which is the part a person
62
+ * searching for the phrase does not type and the part an injection cannot work
63
+ * without. `always` patterns need no such qualification: nobody searches for a JNDI
64
+ * lookup with a live LDAP URL in it.
65
+ */
66
+ type PayloadTier = "always" | "sql" | "markup";
50
67
  export declare function probeSignatureDetector(options?: ProbeSignatureOptions): Detector;
68
+ /** Exported for the fuzz that proves the gate cannot reject something a pattern would match. */
69
+ export declare const __payloadInternals: {
70
+ PAYLOADS: readonly {
71
+ pattern: RegExp;
72
+ what: string;
73
+ tier: PayloadTier;
74
+ }[];
75
+ PAYLOAD_GATE: RegExp;
76
+ };
77
+ export {};
@@ -0,0 +1,26 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface ProbeVolumeOptions {
3
+ /** Reported responses before the ratio means anything. Default 20. */
4
+ minResponses?: number;
5
+ /** Share of them that must be misses. Default 0.8. */
6
+ missRatio?: number;
7
+ }
8
+ /**
9
+ * An actor that is looking for something rather than reading anything.
10
+ *
11
+ * The oldest tell there is for a scanner, and the one this library could not see: it
12
+ * decides *before* the response exists, which is what lets it shape the response and also
13
+ * what hides the status code from it. Fed back through `recordOutcome`, the shape is
14
+ * unmistakable — a person browsing a site does not generate forty misses in a row, and a
15
+ * wordlist does almost nothing else.
16
+ *
17
+ * Counts 404 and 410 only. A 403 is usually this library's own doing, and counting it
18
+ * would let a rule that challenges an actor manufacture the evidence for challenging it.
19
+ * A 500 is the site's problem and says nothing about the client.
20
+ *
21
+ * `moderate`, not higher. A site that has just moved its URLs produces this from perfectly
22
+ * ordinary readers, and so does a feed reader working through a list of removed articles.
23
+ * It is also entirely absent unless the application reports outcomes, which is why nothing
24
+ * else depends on it.
25
+ */
26
+ export declare function probeVolumeDetector(options?: ProbeVolumeOptions): Detector;
@@ -0,0 +1,135 @@
1
+ import type { Detector } from "./types.js";
2
+ /**
3
+ * Three questions that can only be answered by comparing a client with everybody else.
4
+ *
5
+ * All of them read {@link SiteProfile}, all of them are silent unless `site` is
6
+ * configured, and all of them are silent again until it has warmed up. That last point
7
+ * is the one worth repeating: during warmup every path is rare because nothing has been
8
+ * seen, so a baseline consulted early does not merely fail, it fails *confidently*.
9
+ *
10
+ * None of them exceeds `moderate`. A baseline is a claim about what is normal for a
11
+ * site, and it is wrong in exactly the circumstances a site is most unusual — the day of
12
+ * a redesign, the hour a campaign lands, the migration that leaves half the URLs
13
+ * missing. That is not a reason to skip the comparison; it is a reason never to let one
14
+ * close a door on its own.
15
+ */
16
+ export interface DistributedWalkOptions {
17
+ /** Distinct actors on one shape before it is worth reporting. Default 8. */
18
+ minActors?: number;
19
+ /** Distinct ids that must have been covered. Default 150. */
20
+ minIds?: number;
21
+ /** Fraction of the id range that must actually have been requested. Default 0.6. */
22
+ minCoverage?: number;
23
+ /** Highest visits-per-id that still reads as enumeration rather than reading. Default 1.3. */
24
+ maxRevisitRatio?: number;
25
+ /**
26
+ * Lowest visits-per-id that still reads as enumeration. Default 0.7.
27
+ *
28
+ * Guards the estimate rather than the traffic — see the note on coarsening below.
29
+ */
30
+ minRevisitRatio?: number;
31
+ }
32
+ /**
33
+ * An enumeration split across many clients so that no single one looks like one.
34
+ *
35
+ * This is the threat every per-actor threshold misses by construction. Take a wordlist,
36
+ * or an id range, and divide it between five hundred addresses at one request a minute
37
+ * each: every actor is unremarkable, `id-enumeration` never fires for anybody, and the
38
+ * range is still walked end to end. The only place it is visible is in the union.
39
+ *
40
+ * **What separates it from a popular site.** Many clients requesting numbered pages is
41
+ * ordinary — that is what a catalogue is. Two things are not. Enumeration *covers* a
42
+ * contiguous range rather than sampling the popular parts of it, and it visits each id
43
+ * about once, because there is no reason to fetch the same record twice. Real readers
44
+ * are the opposite on both counts: they cluster on a few popular ids and return to them.
45
+ * So coverage and the revisit ratio are both required, and either one alone would report
46
+ * an ordinary shop.
47
+ *
48
+ * **Why this is `moderate` and why it stays there.** The evidence is about the *shape*,
49
+ * and it is attached to a client that contributed to it — which is the closest this
50
+ * library comes to holding one client responsible for what others did. It is defensible
51
+ * only because the actor is genuinely part of the pattern being described, and only at a
52
+ * weight that cannot deny anybody by itself. A partner integration syncing a catalogue
53
+ * from a pool of workers produces this exactly, and is welcome traffic.
54
+ */
55
+ export declare function distributedWalkDetector(options?: DistributedWalkOptions): Detector;
56
+ export interface PathNoveltyOptions {
57
+ /** Requests before the ratio means anything. Default 30. */
58
+ minRequests?: number;
59
+ /** Share of an actor's requests that must be for paths nobody else asked for. Default 0.95. */
60
+ minNovelShare?: number;
61
+ }
62
+ /**
63
+ * A client asking for things this site has never been asked for.
64
+ *
65
+ * A wordlist is a list of paths that exist on *some* sites. On yours, most of them do
66
+ * not exist and nobody has ever requested them — which makes "nobody else has ever asked
67
+ * for this" a self-maintaining wordlist detector that needs no wordlist. It catches the
68
+ * scanner whose list is newer than `EXPLOIT_PATHS`, and it costs nothing to keep current.
69
+ *
70
+ * **The bar is deliberately close to 1.** Plenty of ordinary traffic requests novel
71
+ * paths: a search page with the query in the path, a CMS with per-article slugs, a
72
+ * long-tail catalogue where most items are viewed once a month. What none of those do is
73
+ * request *only* novel paths for thirty requests in a row. The share is the signal, not
74
+ * the count.
75
+ *
76
+ * **What it cannot see.** The profile is per process and its table evicts, so a path the
77
+ * site serves rarely can read as novel after it ages out. That direction produces false
78
+ * positives rather than misses, which is why this is capped at `moderate` and why the
79
+ * share is set where it is.
80
+ */
81
+ export declare function pathNoveltyDetector(options?: PathNoveltyOptions): Detector;
82
+ export interface MissBaselineOptions {
83
+ /** Answered requests from this actor before the comparison means anything. Default 20. */
84
+ minResponses?: number;
85
+ /** How many times the site's own miss rate this actor must exceed. Default 5. */
86
+ minRatio?: number;
87
+ /** Below this the actor's own miss rate is unremarkable whatever the site's is. Default 0.5. */
88
+ floor?: number;
89
+ }
90
+ /**
91
+ * A client missing far more than this site's visitors normally do.
92
+ *
93
+ * `probe-volume` asks whether an actor's requests are mostly misses, against a fixed
94
+ * threshold. That threshold is wrong on both kinds of site: on one mid-migration, where
95
+ * half of all traffic 404s, it reports everybody; on a tidy one where a miss is genuinely
96
+ * rare, a client missing a third of the time is remarkable and it says nothing.
97
+ *
98
+ * Comparing against the site's own rate fixes both, and is why this reports the ratio
99
+ * rather than the number. It needs the application to report outcomes, which every
100
+ * bundled adapter does.
101
+ */
102
+ export declare function missBaselineDetector(options?: MissBaselineOptions): Detector;
103
+ export interface PathCampaignOptions {
104
+ /** Distinct clients on a newly-appeared path before it is worth reporting. Default 12. */
105
+ minClients?: number;
106
+ /** Share of the answers to it that must be misses. Default 0.9. */
107
+ minMissShare?: number;
108
+ /** Answers needed before that share means anything. Default 10. */
109
+ minAnswered?: number;
110
+ }
111
+ /**
112
+ * A path this site has never served, suddenly being asked for by everybody.
113
+ *
114
+ * This is what a freshly published vulnerability looks like from inside a site: a URL
115
+ * nobody had ever requested is requested by hundreds of unrelated clients within an hour
116
+ * of the disclosure, because they are all running the same new list. It is the exact
117
+ * inverse of `path-novelty`, which reads one client asking for many unknown paths, and
118
+ * it catches the traffic that detector misses — a client running one probe and moving on
119
+ * looks like nothing at all on its own.
120
+ *
121
+ * **Why the miss rate is required, and not optional.** Many clients arriving at once on
122
+ * a brand-new URL is also precisely what a successful launch looks like: a page goes up,
123
+ * a newsletter goes out, and thousands of people request a path that did not exist
124
+ * yesterday. Counting clients alone would report every marketing campaign a site ever
125
+ * runs. What separates them is what the application answered — a launch returns a page,
126
+ * a probe returns nothing — so the miss share is what makes this a signal rather than a
127
+ * traffic alarm, and the detector is silent without outcomes being reported.
128
+ *
129
+ * **Why it is still `moderate`.** The client this is attached to did request the path,
130
+ * so it is describing something that client did; but *most* of the evidence is about
131
+ * everybody else's behaviour, and a client that followed a bad link from somewhere is
132
+ * indistinguishable here from one running the list. That is a real limit, not a
133
+ * conservative gesture.
134
+ */
135
+ export declare function pathCampaignDetector(options?: PathCampaignOptions): Detector;
@@ -0,0 +1,16 @@
1
+ import type { Detector } from "./types.js";
2
+ export interface TargetIntegrityOptions {
3
+ /**
4
+ * Report a target that walks above the site root, e.g. `/../../etc/passwd`. Default
5
+ * true.
6
+ *
7
+ * Worth a switch because it is the one signal here a *broken* client produces as
8
+ * readily as a hostile one: a relative link resolved against the wrong base gives a
9
+ * client dots to send, and some feed readers and old link checkers duly send them.
10
+ * Encoding those dots is deliberate; writing them is a bug. Both are reported, the
11
+ * second at a lower tier, and this turns the second off for a site that has one of
12
+ * those clients and would rather not hear about it on every request.
13
+ */
14
+ reportPlainTraversal?: boolean;
15
+ }
16
+ export declare function targetIntegrityDetector(options?: TargetIntegrityOptions): Detector;