@osqd/bothandlerjs 0.4.0 → 0.6.0

package/dist/detectors/crawler-verification.d.ts

@@ -1,5 +1,38 @@
-import type { Detector } from "./types.js";
+import type { Detector, DetectionContext } from "./types.js";
+import type { BotSignature } from "./known-bots.js";
+/**
+ * What an operator's own check concluded about a claimed identity.
+ *
+ * Three answers, and the third is not a formality. "I could not tell" has to be
+ * expressible and has to mean *silence* — a verifier that returned false for both "this
+ * is a forgery" and "my key server timed out" would turn an outage into an accusation.
+ */
+export type VerificationOutcome = "verified" | "refuted" | "unknown";
+/**
+ * Your own answer to "is this really who it says it is".
+ *
+ * Called with the same context the built-in checks get, for one claimed signature. It
+ * may be async: the natural implementations are a lookup or a signature check.
+ */
+export type CrawlerVerifier = (ctx: DetectionContext, signature: BotSignature) => VerificationOutcome | Promise<VerificationOutcome>;
 export interface CrawlerVerificationOptions {
+    /**
+     * Verifiers of your own, by signature id — `{ googlebot: ..., gptbot: ... }`.
+     *
+     * Most of this database cannot be checked from inside a request: the operator
+     * publishes no DNS proof and no range list, and the claim is simply unfalsifiable.
+     * That is most bots, and until now it meant the library had nothing to offer an
+     * operator who *could* check — because their CDN had already verified the crawler and
+     * said so in a header, because the bot signs its requests, or because they hold the
+     * ASN data. Writing a whole detector to say so meant reimplementing the confirm and
+     * refute semantics in this file, including the part where an inconclusive answer must
+     * stay silent.
+     *
+     * A verifier here runs before the built-in check for that signature and a definite
+     * answer settles it, which also means no DNS lookup. `unknown` falls through to
+     * whatever this library can do on its own.
+     */
+    verifiers?: Readonly<Record<string, CrawlerVerifier>>;
     /**
      * Treat an address with no PTR record as a forged claim. Default true.
      *

package/dist/detectors/id-enumeration.d.ts

@@ -0,0 +1,31 @@
+import type { Detector } from "./types.js";
+export interface IdEnumerationOptions {
+    /** Requests to one path shape before a walk is worth reporting. Default 30. */
+    minRequests?: number;
+    /**
+     * How completely those requests must cover the range they span. Default 0.9 — thirty
+     * requests reaching from id 1 to id 33 report; the same thirty scattered across a
+     * thousand ids do not.
+     */
+    density?: number;
+}
+/**
+ * Somebody working through the identifiers rather than following the links.
+ *
+ * `crawl-breadth` sees this as "many distinct paths", which is what it also sees when a
+ * person reads a documentation site — so it stays `weak` and nothing separates the two.
+ * Measured: `/user/1` through `/user/120` in order scored exactly the same as a hundred
+ * and twenty scattered ids, and the same again as ordinary article paths. All three
+ * `unknown`, all three 57.
+ *
+ * What separates them is not which ids were asked for but whether they *cover a range*.
+ * People arrive at ids through links, and links do not densely enumerate an integer
+ * interval; a harvester does nothing else. Thirty requests reaching from id 1 to id 33 is
+ * a walk. Thirty scattered across a hundred thousand is somebody reading.
+ *
+ * `moderate`, and the bar is set high on purpose. The awkward case is real: products in
+ * one category often carry consecutive ids, so somebody browsing a catalogue can produce a
+ * smaller version of this. Thirty requests covering ninety per cent of their own span is
+ * meant to be past what that produces, and it is still a shape rather than a motive.
+ */
+export declare function idEnumerationDetector(options?: IdEnumerationOptions): Detector;

package/dist/detectors/index.d.ts

@@ -1,3 +1,4 @@
+import type { CrawlerVerificationOptions } from "./crawler-verification.js";
 import type { Detector } from "./types.js";
 export type { DetectionContext, Detector, DetectorResult } from "./types.js";
 export { evidence, absenceIsMeaningful } from "./types.js";
@@ -17,7 +18,15 @@ export type { RateAnomalyOptions } from "./rate-anomaly.js";
 export { cadenceDetector } from "./cadence.js";
 export type { CadenceOptions } from "./cadence.js";
 export { crawlBreadthDetector } from "./crawl-breadth.js";
+export { parameterSweepDetector } from "./parameter-sweep.js";
+export { transportCoherenceDetector } from "./transport-coherence.js";
+export { probeVolumeDetector } from "./probe-volume.js";
+export { idEnumerationDetector } from "./id-enumeration.js";
 export type { CrawlBreadthOptions } from "./crawl-breadth.js";
+export type { ParameterSweepOptions } from "./parameter-sweep.js";
+export type { TransportCoherenceOptions } from "./transport-coherence.js";
+export type { ProbeVolumeOptions } from "./probe-volume.js";
+export type { IdEnumerationOptions } from "./id-enumeration.js";
 export { sessionIntegrityDetector } from "./session-integrity.js";
 export type { SessionIntegrityOptions } from "./session-integrity.js";
 export { identityRotationDetector } from "./identity-rotation.js";
@@ -53,4 +62,6 @@ export type { BotSignature, BotCategory, Verification } from "./known-bots.js";
  * `clearanceDetector` is not here either, because it needs the challenge service —
  * the engine adds it automatically once `challenge.secrets` is configured.
  */
-export declare function defaultDetectors(): Detector[];
+export declare function defaultDetectors(options?: {
+    crawlerVerification?: CrawlerVerificationOptions;
+}): Detector[];

package/dist/detectors/known-bots.d.ts

@@ -16,7 +16,46 @@ import { MultiPatternMatcher } from "../internal/matcher.js";
  */
 export type BotCategory = "search" | "ai" | "seo" | "social" | "monitoring" | "archive" | "feed" | "security" | "advertising" | "library" | "headless"
 /** A real browser engine embedded in a desktop application, with a person driving it. */
- | "embedded" | "other";
+ | "embedded"
+/**
+ * Price, stock and catalogue collection: comparison shopping, marketplace feeds,
+ * repricing tools.
+ *
+ * Its own category because it is the one kind of crawling a shop has a commercial
+ * opinion about rather than a technical one. It is not `seo` — nothing here is
+ * auditing your site for you — and it is not `scraper`, which is a *behavioural*
+ * verdict this library reaches on its own. This is a client that says what it is.
+ */
+ | "commerce"
+/**
+ * Accessibility auditing: contrast, landmarks, ARIA, WCAG conformance.
+ *
+ * Separated from `monitoring` because the answer is almost always different. A site
+ * owner who blocks uptime probes still wants the tool their accessibility team runs
+ * to reach the page, and frequently does not know it is arriving as a bot at all.
+ */
+ | "accessibility"
+/**
+ * A mail or messaging gateway checking a link on somebody's behalf.
+ *
+ * Its own category because of who pays when it is blocked. A social preview that fails
+ * costs a card; one of these failing tells a real person, in their inbox, that the link
+ * they were sent could not be verified — and they were never the one crawling. They also
+ * arrive with none of a browser's marks: from a datacentre, once, with no cookie and no
+ * referer, moments after a message was delivered, which is a shape that reads as
+ * automation because it *is* automation, acting for a human.
+ */
+ | "email-security"
+/**
+ * Research and measurement: universities, internet-measurement projects, plagiarism
+ * and citation indexes.
+ *
+ * Distinct from `ai` on purpose. Both read the whole page and neither sends a person,
+ * but the decision differs: an operator refusing to feed a commercial model may be
+ * perfectly happy to appear in a citation index, and folding the two together forces
+ * one answer onto two questions.
+ */
+ | "academic" | "other";
 /** Every category, for anything that has to enumerate them — a rule editor, a report. */
 export declare const BOT_CATEGORIES: readonly BotCategory[];
 /**
@@ -40,6 +79,26 @@ export type Verification = {
 } | {
     kind: "ip-ranges";
     publishedAt?: string;
+}
+/**
+ * The operator publishes a proof this library cannot check by itself, and you can.
+ *
+ * A signed request under [Web Bot Auth](https://www.rfc-editor.org/rfc/rfc9421), a
+ * CDN that has already verified the crawler and says so in a header it adds, an ASN
+ * lookup against data you hold — all of them are conclusive, and none of them are
+ * something a detection library should be doing on its own: two need a network
+ * dependency and the third needs a key it has no business fetching.
+ *
+ * So the claim is marked verifiable-by-you, and stays *unverified* until you supply a
+ * verifier for it in `crawlerVerification.verifiers`. Marked and unsupplied behaves
+ * exactly like `none`: neither confirmed nor accused.
+ *
+ * `via` names the mechanism, for the operator reading this table to know what they
+ * would have to write.
+ */
+ | {
+    kind: "proof";
+    via: string;
 } | {
     kind: "none";
 };

package/dist/detectors/parameter-sweep.d.ts

@@ -0,0 +1,39 @@
+import type { Detector } from "./types.js";
+export interface ParameterSweepOptions {
+    /**
+     * Distinct path-and-query combinations at or above which a sweep is worth reporting.
+     * Default 25. Cannot exceed {@link MAX_TRACKED_QUERIES}, where the count saturates;
+     * asking for more throws rather than never firing.
+     */
+    threshold?: number;
+    /**
+     * How many variants must sit on one path before this is a sweep rather than browsing.
+     * Default 8 — that is, twenty-five variants across three paths reports, and
+     * twenty-five variants across twenty paths does not.
+     */
+    variantsPerPath?: number;
+    /** Minimum requests before the shape means anything. Default 20. */
+    minRequests?: number;
+}
+/**
+ * The scraping that `crawl-breadth` cannot see.
+ *
+ * Breadth counts distinct *paths*, and a path has no query string on it. So the shape
+ * it reads as "somebody rereading one page" is also the shape of enumerating a
+ * catalogue: `/products?page=1` through `?page=200` is one path and two hundred
+ * requests. Measured, on the same two hundred requests expressed both ways — as
+ * distinct paths it scored 62 and was called `suspected-bot`; as `?page=N` it scored 55
+ * and passed as `unknown`. Paginated collection is not an exotic case, it is how
+ * catalogues, search results and APIs are actually taken.
+ *
+ * So this counts the other thing: distinct parameterisations, and how many of them
+ * stack onto a single path. Both halves are needed. A high variant count alone is
+ * ordinary — a shop's own visitors filter and sort — and it is the *concentration* that
+ * separates a person changing their mind from a machine walking an index.
+ *
+ * `weak`, and deliberately. A person paging through search results produces a smaller
+ * version of exactly this, and someone with a slow connection retrying looks similar
+ * again. It is a shape, not a motive; its value is as a second signal beside an actor
+ * that has already failed something sharper.
+ */
+export declare function parameterSweepDetector(options?: ParameterSweepOptions): Detector;

package/dist/detectors/probe-volume.d.ts

@@ -0,0 +1,26 @@
+import type { Detector } from "./types.js";
+export interface ProbeVolumeOptions {
+    /** Reported responses before the ratio means anything. Default 20. */
+    minResponses?: number;
+    /** Share of them that must be misses. Default 0.8. */
+    missRatio?: number;
+}
+/**
+ * An actor that is looking for something rather than reading anything.
+ *
+ * The oldest tell there is for a scanner, and the one this library could not see: it
+ * decides *before* the response exists, which is what lets it shape the response and also
+ * what hides the status code from it. Fed back through `recordOutcome`, the shape is
+ * unmistakable — a person browsing a site does not generate forty misses in a row, and a
+ * wordlist does almost nothing else.
+ *
+ * Counts 404 and 410 only. A 403 is usually this library's own doing, and counting it
+ * would let a rule that challenges an actor manufacture the evidence for challenging it.
+ * A 500 is the site's problem and says nothing about the client.
+ *
+ * `moderate`, not higher. A site that has just moved its URLs produces this from perfectly
+ * ordinary readers, and so does a feed reader working through a list of removed articles.
+ * It is also entirely absent unless the application reports outcomes, which is why nothing
+ * else depends on it.
+ */
+export declare function probeVolumeDetector(options?: ProbeVolumeOptions): Detector;

package/dist/detectors/transport-coherence.d.ts

@@ -0,0 +1,31 @@
+import type { Detector } from "./types.js";
+export interface TransportCoherenceOptions {
+    /**
+     * Report a claimed browser arriving over HTTP/1.0. Default true.
+     *
+     * Turn it off if something in front of this application speaks HTTP/1.0 to it. A few
+     * older load balancers and reverse proxies still do, and where that is true every
+     * request arrives that way — so the signal says something about your infrastructure
+     * rather than about your visitors, and a detector that fires on all of them is worse
+     * than one that fires on none.
+     */
+    legacyHttp?: boolean;
+    /** Requests an actor must have made before an all-HEAD visit means anything. Default 8. */
+    minHeadRequests?: number;
+}
+/**
+ * How a claimed browser *moves*, rather than what it says.
+ *
+ * The header checks read one request against the client it claims to be. This reads the
+ * transport underneath and the verbs across a visit, which are harder to copy because
+ * they are not in the part of the request most tools let you set.
+ *
+ * Two things, both measured as blind spots before this existed — a client claiming
+ * Chrome 120 over HTTP/1.0, and one whose entire visit is HEAD, each scored exactly what
+ * the honest control scored.
+ *
+ * Neither goes above `moderate`, and the reasons are different. HTTP/1.0 can be an
+ * intermediary's doing rather than the client's. An all-HEAD visit is a strong shape but a
+ * link checker is a real and mostly harmless thing to be.
+ */
+export declare function transportCoherenceDetector(options?: TransportCoherenceOptions): Detector;