@osqd/bothandlerjs 0.4.0 → 0.6.0

package/dist/state.d.ts

@@ -12,6 +12,7 @@ import type { ActorSnapshot, RequestFacts } from "./types.js";
  */
 export declare const MAX_TRACKED_ARRIVALS = 32;
 export declare const MAX_TRACKED_PATHS = 64;
+export declare const MAX_TRACKED_QUERIES = 64;
 export declare const MAX_TRACKED_USER_AGENTS = 4;
 export declare class ActorState {
     readonly key: string;
@@ -53,6 +54,45 @@ export declare class ActorState {
     private readonly paths;
     private pathsOverflowed;
     private pathsSaturatedAtTotal;
+    /**
+     * Distinct *parameterised* requests: the path together with its query.
+     *
+     * Counted apart from `paths` because the two answer different questions and a scraper
+     * lives in the gap between them. `/products?page=1` through `?page=200` is one path and
+     * two hundred requests, so breadth reads it as somebody rereading a single page — which
+     * is exactly what enumerating a catalogue looks like from the path alone.
+     */
+    private readonly queries;
+    private queriesOverflowed;
+    /**
+     * Which HTTP methods this actor has used.
+     *
+     * A browser navigating issues GET. Something that has issued nothing but HEAD across a
+     * long visit is checking what exists rather than reading it, and that is a fact about
+     * the actor rather than about any one of its requests — which is why it is kept here.
+     */
+    private readonly methods;
+    /**
+     * Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
+     *
+     * Three numbers per shape, deliberately — a count, a lowest and a highest — rather than
+     * the ids themselves. What separates enumeration from reading is not which ids were
+     * asked for but whether they *cover a range*: thirty requests spanning thirty
+     * consecutive ids is a walk, and thirty scattered across a hundred thousand is somebody
+     * following links. Both are answerable from a count and a span, and only the count and
+     * the span survive an actor asking for ten thousand of them.
+     */
+    private readonly walks;
+    /**
+     * What the application answered, for the requests anybody bothered to tell us about.
+     *
+     * The engine decides *before* the response exists, so this arrives afterwards and only
+     * when the adapter reports it. Kept as two counters rather than a list because the one
+     * question worth asking is a ratio: an actor whose requests are almost all misses is
+     * looking for something rather than reading anything.
+     */
+    private responsesSeen;
+    private missesSeen;
     private readonly userAgents;
     constructor(key: string, now: number);
     /** Records an arrival. Called exactly once per request, by the engine. */
@@ -60,6 +100,41 @@ export declare class ActorState {
     /** Distinct paths seen, by hash. Saturates at {@link PATH_CAP}; `pathsSaturated` says whether it did. */
     get distinctPaths(): number;
     get pathsSaturated(): boolean;
+    /** Distinct path-and-query combinations seen. Saturates at {@link QUERY_CAP}. */
+    get distinctQueries(): number;
+    get queriesSaturated(): boolean;
+    /**
+     * Records what the application answered. Called after the response, if at all.
+     *
+     * 404 and 410 only. A 403 is usually this library's own doing and counting it would
+     * make the detector that reads this argue with itself; a 500 is the site's problem and
+     * says nothing about the client.
+     */
+    recordOutcome(status: number): void;
+    /** Responses reported for this actor. Zero unless something is reporting them. */
+    get responses(): number;
+    /** Of those, how many were 404 or 410. */
+    get misses(): number;
+    /**
+     * Files a request under the shape of its path, if that path carries a number.
+     *
+     * The last numeric segment is the one taken to be the identifier: in `/api/v2/orders/42`
+     * the version is part of the shape and the order id is what is being walked.
+     */
+    private noteWalk;
+    /**
+     * The path shape this actor has walked hardest, with how far it reached.
+     *
+     * `span` is inclusive of both ends, so a walk of 1 to 30 spans 30. Comparing the count
+     * against it is what separates covering a range from visiting a few points in one.
+     */
+    densestWalk(): {
+        template: string;
+        count: number;
+        span: number;
+    } | undefined;
+    /** Every HTTP method this actor has used, in first-seen order. */
+    get methodsSeen(): readonly string[];
     /**
      * Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
      * still can. Over that many requests the distinct count is exact, so it is the only
@@ -153,7 +228,7 @@ export declare class ActorRegistry {
      * A read, and only a read: it neither records a request against an actor nor moves
      * one up the LRU, so watching the list cannot change what it lists.
      */
-    top(limit: number, now: number): ActorSummary[];
+    top(limit: number, now: number, offset?: number): ActorSummary[];
     forget(key: string): void;
     clear(): void;
     /** Marks an actor as holding valid human clearance until `until`. */

package/dist/types.d.ts

@@ -178,6 +178,42 @@ export interface ActorSnapshot {
     requests: number;
     /** Distinct paths seen inside the window — breadth of crawl. */
     distinctPaths: number;
+    /**
+     * Distinct path-and-query combinations seen from this actor.
+     *
+     * Separate from `distinctPaths` because a scraper lives in the gap between them:
+     * `/products?page=1` through `?page=200` is one path and two hundred requests.
+     */
+    distinctQueries: number;
+    /** Whether `distinctQueries` stopped being able to grow. */
+    queriesSaturated: boolean;
+    /**
+     * Every HTTP method this actor has used.
+     *
+     * A browser navigating issues GET; an actor whose whole visit is HEAD is checking what
+     * exists rather than reading it.
+     */
+    methodsSeen: readonly string[];
+    /**
+     * Responses reported back for this actor, and how many of them were misses.
+     *
+     * Both zero unless something calls `recordOutcome` — the engine decides before a
+     * response exists, so this is knowledge only the application has.
+     */
+    responses: number;
+    /** Of `responses`, how many were 404 or 410. */
+    misses: number;
+    /**
+     * The path shape this actor has walked hardest — `/user/#` — with how many requests
+     * went to it and how wide a range of numbers they covered.
+     *
+     * Undefined when no path carried a number. `span` is inclusive, so ids 1 to 30 span 30.
+     */
+    walk?: {
+        template: string;
+        count: number;
+        span: number;
+    } | undefined;
     /** First and last sighting, ms since epoch. */
     firstSeen: number;
     lastSeen: number;

package/docs/course/05-detectors.md

@@ -82,7 +82,11 @@ fails closed is an outage with extra steps.
 | `rate-anomaly` | `moderate` | Arrivals in a short window. Reports; never concludes |
 | `cadence` | `moderate` | Coefficient of variation of the gaps. Catches the polite scraper pacing itself *under* your rate limit |
 | `crawl-breadth` | `weak` | Distinct paths against total requests: reading a site against enumerating it |
+| `parameter-sweep` | `weak` | Distinct query strings against the paths they sit on. Catches the collection that leaves the path unchanged — `?page=1..200` |
 | `session-integrity` | `moderate` | A "browser" that never carries a cookie |
+| `id-enumeration` | `moderate` | A contiguous run of numeric ids under one path shape — walking `/user/1..n` rather than following links |
+| `probe-volume` | `moderate` | The share of an actor's requests answered 404. Needs `recordOutcome`; the Node adapter wires it up |
+| `transport-coherence` | `moderate` | The HTTP version and the verbs across a visit: a "Chrome" on HTTP/1.0, a visit made only of HEAD |
 | `identity-rotation` | `moderate` | One actor, several User-Agents. **Off by default** — under an IP actor key this fires on every corporate NAT |
 | `browsing-coherence` | `moderate` | The only detector arguing *for* the client. Human-pointing, so it discounts |
 

package/docs/detection/detectors.md

@@ -211,6 +211,27 @@ new BotHandler({
 The library ships **no address data** and refuses to guess any; see
 [design decisions](../design/decisions.md).
 
+**Loading a feed.** `fetchAddressList` reads a published list — a reputation feed, a hosting
+provider's own ranges — over the same hardened path the crawler ranges use: HTTPS only, a
+size cap, `#` and `;` comments stripped, JSON `prefixes` documents or one prefix per line,
+and a list that is empty, oversized or contains a block big enough to matter is refused
+**whole** rather than in part.
+
+```ts
+import { fetchAddressList } from "@osqd/bothandlerjs";
+
+const prefixes = await fetchAddressList({ id: "denylist", url: "https://example.org/drop.txt" });
+detector.updateRanges("denylist", prefixes);
+```
+
+Two steps, on purpose: fetching is the part that can fail, and installing is the part that
+changes what happens to somebody. Nothing is fetched on a schedule unless you schedule it.
+
+And think hard before pointing that at `denylist` rather than `datacenter`. A denylist entry
+does not corroborate anything — it decides, and it blocks people. A feed is somebody else's
+judgement about an address, refreshed on somebody else's schedule, and an address that was a
+bot last month may be a customer's home connection this month.
+
 ### `trap`
 
 **cheap · always · ceiling `certain`**
@@ -276,6 +297,107 @@ walking a sitemap almost never revisits, so its ratio sits near one.
 `weak`, because a *welcome* crawler produces exactly this shape and so does a person on a
 first visit to a documentation site.
 
+### `id-enumeration`
+
+**cheap · always · ceiling `moderate`**
+
+Somebody working through the identifiers rather than following the links.
+
+`crawl-breadth` sees this as "many distinct paths" — which is also what it sees when a
+person reads a documentation site, so it stays `weak` and nothing separates the two.
+Measured before this existed: `/user/1` through `/user/120` in order scored 57, a hundred
+and twenty scattered ids scored 57, and ordinary article paths scored 57.
+
+What separates them is not *which* ids were asked for but whether they **cover a range**.
+People arrive at ids through links, and links do not densely enumerate an integer interval;
+a harvester does nothing else. Thirty requests reaching from id 1 to id 33 is a walk; thirty
+scattered across a hundred thousand is somebody reading.
+
+It costs three numbers per path shape — a count, a lowest and a highest — rather than a
+list of every id seen, which is what makes it affordable for an actor that asks for ten
+thousand of them. The last numeric segment is taken as the identifier, so in
+`/api/v2/orders/42` the version is part of the shape and the order id is the walk. Numbers
+too large to be a counter are ignored: nobody walks epoch seconds.
+
+`moderate`, with the bar set high on purpose. Products in one category often carry
+consecutive ids, so somebody browsing a catalogue produces a smaller version of this.
+
+### `probe-volume`
+
+**cheap · always · ceiling `moderate`**
+
+An actor that is looking for something rather than reading anything.
+
+The oldest tell there is for a scanner, and the one this library could not see. Every
+verdict here is reached *before* the response exists — that is what lets it shape the
+response, and it is also what hides the status code from it. So the application reports it
+back:
+
+```ts
+const { outcome } = await handler.handle(facts);
+// …your application answers…
+handler.recordOutcome(facts, response.statusCode);
+```
+
+The bundled Node adapter does this for you. Nothing else depends on it: every other
+detector works unchanged if you never call it, and this one is simply absent.
+
+Counts **404 and 410 only**. A 403 is usually this library's own doing, and counting it
+would let a rule that challenges an actor manufacture the evidence for having challenged
+it. A 500 is the site's problem and says nothing about the client.
+
+`moderate`, because a site that has just moved its URLs produces exactly this shape from
+perfectly ordinary readers, and so does a feed reader working through removed articles.
+Eighty per cent of at least twenty reported responses, by default.
+
+### `transport-coherence`
+
+**cheap · always · ceiling `moderate`**
+
+How a claimed browser *moves*, rather than what it says.
+
+The header checks read one request against the client it claims to be. This reads the
+transport underneath and the verbs across a visit — harder to copy, because neither is in
+the part of a request most tooling lets you set.
+
+Two things. A claimed browser that negotiated **HTTP/1.0**, which no shipping browser has
+offered in over a decade. And a visit made **entirely of HEAD**: one HEAD is a browser
+checking a link it is about to follow or a cache revalidating, but a whole visit of them is
+something checking what exists without reading any of it.
+
+Both were measured as blind spots before this existed — a client claiming Chrome 120 over
+HTTP/1.0, and one whose whole visit was HEAD, each scored exactly what the honest control
+scored.
+
+Capped at `moderate`, for different reasons each. HTTP/1.0 is not always the client's
+doing: a few older load balancers speak it to the origin, and behind one of those every
+request looks like this — which is what `transportCoherenceDetector({ legacyHttp: false })`
+is for. An all-HEAD visit is a stronger shape, but a link checker is a real and mostly
+harmless thing to be.
+
+### `parameter-sweep`
+
+**cheap · always · ceiling `weak`**
+
+The collection `crawl-breadth` cannot see.
+
+Breadth counts distinct *paths*, and a path carries no query string — so the shape it reads
+as "somebody rereading one page" is also the shape of enumerating a catalogue.
+`/products?page=1` through `?page=200` is one path and two hundred requests. Measured on
+the same two hundred requests expressed both ways: as distinct paths they scored 62 and
+were called `suspected-bot`; as `?page=N` they scored 55 and passed as `unknown`. Paginated
+collection is not an exotic case — it is how catalogues, search results and APIs are
+actually taken.
+
+So this counts the other thing: distinct parameterisations, and how many of them stack onto
+a single path. Both halves matter. A high variant count on its own is ordinary — a shop's
+own visitors filter and sort — and it is the *concentration* that separates a person
+changing their mind from a machine walking an index.
+
+`weak`, for the same reason as breadth: a person paging through search results produces a
+smaller version of exactly this. Its value is as a second signal beside an actor that has
+already failed something sharper.
+
 ### `identity-rotation`
 
 **cheap · always · ceiling `moderate` · off by default**

package/docs/detection/signatures.md

@@ -41,6 +41,26 @@ individual name does not.
 | `library` | curl, wget, python-requests, Go-http-client, okhttp | usually challenge |
 | `headless` | HeadlessChrome, Playwright, Puppeteer, Selenium | usually challenge |
 | `embedded` | Smart TVs, set-top boxes, game consoles | allow |
+| `commerce` | idealo, Kelkoo, PriceRunner, Trivago, Skyscanner, Indeedbot | a commercial decision |
+| `academic` | Crossref, OpenAlex, university web-science crawls | usually allow |
+| `accessibility` | Siteimprove and other WCAG auditors | allow |
+| `email-security` | Proofpoint, Mimecast, Barracuda, Cisco Secure Email | allow |
+
+`email-security` is a mail or messaging gateway checking a link somebody was sent, before
+they are allowed to click it — and it is separate because of who pays when it is blocked. A
+social preview that fails costs a card; one of these failing tells a real person, in their
+inbox, that their mail contained a link that could not be verified, and they were never the
+one crawling. They arrive with none of a browser's marks — from a datacentre, once, no
+cookie, no referer — because the request is automation acting on a human's behalf.
+
+The other three are separate from their nearest neighbours because the decision is. A price
+comparator is not an SEO auditor — nothing there is auditing your site for you, and the
+same crawler is a distribution channel to one retailer and a competitor's research tool to
+the next, which is why nothing in `commerce` is marked benign. A citation index is not a
+model being trained, and an operator refusing the second is often glad of the first. And an
+accessibility crawler is not an uptime probe: it is usually commissioned by the site's own
+owner and then forgotten about, so blocking it does not reduce load, it makes an
+accessibility report look clean by removing the evidence.
 
 ```ts
 { id: "no-ai", match: { category: "ai" }, action: "block", reason: "Not for model training." }
@@ -54,6 +74,7 @@ What, if anything, can check the claim:
 type Verification =
   | { kind: "fcrdns"; domains: readonly string[] }   // reverse DNS, forward-confirmed
   | { kind: "ip-ranges"; publishedAt?: string }      // an address list the operator publishes
+  | { kind: "proof"; via: string }                   // something only you can check
   | { kind: "none" };                                // no published mechanism
 ```
 
@@ -62,6 +83,29 @@ them, and for those the claim is **unfalsifiable**. The library neither confirms
 accuses — it records what the client said and lets the policy decide what a self-declared
 identity is worth. See [verifying a crawler](verification.md).
 
+**But you may be able to check what this library cannot.** Your CDN has often already
+verified the crawler and says so in a header it adds; some bots now sign their requests;
+you may hold ASN data. None of those belong inside a detection library — two need a
+network dependency and the third needs a key it has no business fetching — so instead you
+supply the answer:
+
+```ts
+new BotHandler({
+  crawlerVerification: {
+    verifiers: {
+      // Whatever you can prove, however you prove it.
+      gptbot: (ctx) => (ctx.facts.headers["cf-verified-bot"] === "gptbot" ? "verified" : "unknown"),
+    },
+  },
+});
+```
+
+A verifier returns `"verified"`, `"refuted"` or `"unknown"`, runs before the built-in check
+for that signature, and a definite answer settles it — which also means no DNS lookup.
+`"unknown"` falls through to whatever the library can do on its own, and so does a throw:
+an unreachable key server must never read as an accusation, or an outage becomes a wave of
+blocked crawlers.
+
 ## What a name is worth
 
 A matched signature makes `self-identified` produce `certain` evidence, and this is the

package/docs/operations/dashboard.md

@@ -29,6 +29,12 @@ User-Agent, the verdict, the score, the action and the rule that chose it. Filte
 proven, suspected, human, guard stops, denied, mitigated or served; or type into the
 search box.
 
+The feed pages, fifty requests at a time. The newest page follows the stream; stepping
+back holds the list still while you read it — a feed that renumbers itself under somebody
+paging through it cannot be read — and says **held while you read** so the stillness is
+not mistaken for quiet. Requests keep arriving and keep being counted; they are at the
+front when you return to it.
+
 The search takes fields, and negation, because the two cases people actually reach for
 it are an address that also appears inside a User-Agent and a path that is a prefix of
 ten others:
@@ -108,6 +114,17 @@ regularity (near zero is a metronome, which no person is), prior confirmations,
 whether they hold clearance. **In feed** sends one to the live feed as an `actor:` filter,
 which makes it a shareable URL like every other view.
 
+This screen pages too, twenty-five at a time, busiest first. The registry holds far more
+clients than the feed's ring holds requests, and paging is what reaches them: the feed
+already shows you what is loudest, and the population behind it is the reason this screen
+exists. Ranking something that is still moving means a client can shift between pages
+while you read; the order is a snapshot of a live list, not a stable index.
+
+The **Actors tracked** counter above the tab strip is the way in: it is a button, so
+pressing it — or reaching it with the keyboard and pressing Enter — opens this screen.
+Where the `registry` section is switched off there is no Actors screen to open, and the
+counter stays an ordinary tile rather than offering to go somewhere that does not exist.
+
 **Statistics** — a traffic timeline (1m/5m/15m/1h) split by outcome, which says how
 much history the window actually holds rather than drawing a flat line through time it
 never had; the assessment-latency histogram with mean, p95 and max; and the **score
@@ -383,6 +400,15 @@ capped at `maxEventsPerSecond` (100 by default, `0` to remove it). What is cappe
 still see every request, and the feed says how many were not streamed. A thinned feed
 must never look like a quiet one.
 
+**And it does not stop at saying so.** Next to that count is **Load them**, which fetches
+the ring and merges what is missing back into the feed, in the order it happened. Nothing
+was ever lost — the cap and the lag guard both keep entries off the *stream* and leave the
+ring alone — so this is the same data arriving by a different road. It is a button rather
+than something automatic on purpose: entries are skipped exactly when the origin is
+busiest, and a dashboard that answered every skip by re-fetching the whole ring would be a
+load amplifier pointed at the process it is watching, which is what the cap exists to
+prevent.
+
 **A viewer that stops reading is not allowed to cost you memory.** A socket that has
 stopped draining — a laptop that slept with the tab open, a phone in a tunnel, a proxy
 that stopped reading — used to accumulate frames in this process, one queue per viewer,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@osqd/bothandlerjs",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "publishConfig": {
     "access": "public"
   },