@osqd/bothandlerjs 0.5.0 → 0.6.0

package/dist/corpus/schema.d.ts

@@ -61,6 +61,14 @@ export interface CaseRequest {
     tlsFingerprint?: string;
     /** Milliseconds after the case's start time. Drives the behavioural detectors. */
     atMs?: number;
+    /**
+     * What the application answered, if the case is about that.
+     *
+     * The engine decides before a response exists, so this is reported back afterwards the
+     * way an adapter reports it. Only cases about response shape need it — a scan that is
+     * almost all misses being the one that matters.
+     */
+    status?: number;
     /** The source could not supply the full header set. See `RequestFacts.partialHeaders`. */
     partialHeaders?: boolean;
 }

package/dist/crawler-ranges.d.ts

@@ -81,6 +81,37 @@ export interface RefreshResult {
  * an error rather than a guess.
  */
 export declare function fetchCrawlerRanges(source: PublishedRangeSource, options?: RefreshOptions): Promise<string[]>;
+/** Where a reputation or hosting-provider address list is published. */
+export interface AddressListSource {
+    /** The range set to install it as. `denylist` blocks; `datacenter` corroborates. */
+    id: "denylist" | "datacenter" | "allowlist";
+    url: string;
+}
+export interface AddressListOptions extends RefreshOptions {
+    /** Entries to accept before refusing the list. Default 100,000. */
+    maxPrefixes?: number;
+}
+/**
+ * Fetches a published address list — a reputation feed, a hosting provider's own ranges.
+ *
+ * The same hardened path as {@link fetchCrawlerRanges}: HTTPS only, a size cap, comments
+ * stripped, JSON `prefixes` documents or one prefix per line, and a list that is empty,
+ * oversized or contains a block big enough to matter is refused **whole** rather than in
+ * part. It is a separate entry point because the limits differ — a reputation feed is
+ * thousands of entries where a crawler's is hundreds, and calling one a crawler's list in
+ * an error message helps nobody.
+ *
+ * It does not install anything. Hand the result to `updateRanges("denylist", …)` when you
+ * are ready, which is the same two-step the crawler path takes and for the same reason:
+ * fetching is the part that can fail, and installing is the part that changes behaviour.
+ *
+ * **A denylist entry is `certain`.** It does not corroborate anything — it decides, and it
+ * blocks people. A feed is somebody else's judgement about an address, refreshed on
+ * somebody else's schedule, and an address that was a bot last month may be a customer's
+ * home connection this month. Read what you are subscribing to, and prefer `datacenter`
+ * for anything you have not decided about yourself.
+ */
+export declare function fetchAddressList(source: AddressListSource, options?: AddressListOptions): Promise<string[]>;
 /**
  * Fetches every published list and installs it.
  *

package/dist/detectors/id-enumeration.d.ts

@@ -0,0 +1,31 @@
+import type { Detector } from "./types.js";
+export interface IdEnumerationOptions {
+    /** Requests to one path shape before a walk is worth reporting. Default 30. */
+    minRequests?: number;
+    /**
+     * How completely those requests must cover the range they span. Default 0.9 — thirty
+     * requests reaching from id 1 to id 33 report; the same thirty scattered across a
+     * thousand ids do not.
+     */
+    density?: number;
+}
+/**
+ * Somebody working through the identifiers rather than following the links.
+ *
+ * `crawl-breadth` sees this as "many distinct paths", which is what it also sees when a
+ * person reads a documentation site — so it stays `weak` and nothing separates the two.
+ * Measured: `/user/1` through `/user/120` in order scored exactly the same as a hundred
+ * and twenty scattered ids, and the same again as ordinary article paths. All three
+ * `unknown`, all three 57.
+ *
+ * What separates them is not which ids were asked for but whether they *cover a range*.
+ * People arrive at ids through links, and links do not densely enumerate an integer
+ * interval; a harvester does nothing else. Thirty requests reaching from id 1 to id 33 is
+ * a walk. Thirty scattered across a hundred thousand is somebody reading.
+ *
+ * `moderate`, and the bar is set high on purpose. The awkward case is real: products in
+ * one category often carry consecutive ids, so somebody browsing a catalogue can produce a
+ * smaller version of this. Thirty requests covering ninety per cent of their own span is
+ * meant to be past what that produces, and it is still a shape rather than a motive.
+ */
+export declare function idEnumerationDetector(options?: IdEnumerationOptions): Detector;

package/dist/detectors/index.d.ts

@@ -18,7 +18,15 @@ export type { RateAnomalyOptions } from "./rate-anomaly.js";
 export { cadenceDetector } from "./cadence.js";
 export type { CadenceOptions } from "./cadence.js";
 export { crawlBreadthDetector } from "./crawl-breadth.js";
+export { parameterSweepDetector } from "./parameter-sweep.js";
+export { transportCoherenceDetector } from "./transport-coherence.js";
+export { probeVolumeDetector } from "./probe-volume.js";
+export { idEnumerationDetector } from "./id-enumeration.js";
 export type { CrawlBreadthOptions } from "./crawl-breadth.js";
+export type { ParameterSweepOptions } from "./parameter-sweep.js";
+export type { TransportCoherenceOptions } from "./transport-coherence.js";
+export type { ProbeVolumeOptions } from "./probe-volume.js";
+export type { IdEnumerationOptions } from "./id-enumeration.js";
 export { sessionIntegrityDetector } from "./session-integrity.js";
 export type { SessionIntegrityOptions } from "./session-integrity.js";
 export { identityRotationDetector } from "./identity-rotation.js";

package/dist/detectors/known-bots.d.ts

@@ -35,6 +35,17 @@ export type BotCategory = "search" | "ai" | "seo" | "social" | "monitoring" | "a
  * to reach the page, and frequently does not know it is arriving as a bot at all.
  */
  | "accessibility"
+/**
+ * A mail or messaging gateway checking a link on somebody's behalf.
+ *
+ * Its own category because of who pays when it is blocked. A social preview that fails
+ * costs a card; one of these failing tells a real person, in their inbox, that the link
+ * they were sent could not be verified — and they were never the one crawling. They also
+ * arrive with none of a browser's marks: from a datacentre, once, with no cookie and no
+ * referer, moments after a message was delivered, which is a shape that reads as
+ * automation because it *is* automation, acting for a human.
+ */
+ | "email-security"
 /**
  * Research and measurement: universities, internet-measurement projects, plagiarism
  * and citation indexes.

package/dist/detectors/parameter-sweep.d.ts

@@ -0,0 +1,39 @@
+import type { Detector } from "./types.js";
+export interface ParameterSweepOptions {
+    /**
+     * Distinct path-and-query combinations at or above which a sweep is worth reporting.
+     * Default 25. Cannot exceed {@link MAX_TRACKED_QUERIES}, where the count saturates;
+     * asking for more throws rather than never firing.
+     */
+    threshold?: number;
+    /**
+     * How many variants must sit on one path before this is a sweep rather than browsing.
+     * Default 8 — that is, twenty-five variants across three paths reports, and
+     * twenty-five variants across twenty paths does not.
+     */
+    variantsPerPath?: number;
+    /** Minimum requests before the shape means anything. Default 20. */
+    minRequests?: number;
+}
+/**
+ * The scraping that `crawl-breadth` cannot see.
+ *
+ * Breadth counts distinct *paths*, and a path has no query string on it. So the shape
+ * it reads as "somebody rereading one page" is also the shape of enumerating a
+ * catalogue: `/products?page=1` through `?page=200` is one path and two hundred
+ * requests. Measured, on the same two hundred requests expressed both ways — as
+ * distinct paths it scored 62 and was called `suspected-bot`; as `?page=N` it scored 55
+ * and passed as `unknown`. Paginated collection is not an exotic case, it is how
+ * catalogues, search results and APIs are actually taken.
+ *
+ * So this counts the other thing: distinct parameterisations, and how many of them
+ * stack onto a single path. Both halves are needed. A high variant count alone is
+ * ordinary — a shop's own visitors filter and sort — and it is the *concentration* that
+ * separates a person changing their mind from a machine walking an index.
+ *
+ * `weak`, and deliberately. A person paging through search results produces a smaller
+ * version of exactly this, and someone with a slow connection retrying looks similar
+ * again. It is a shape, not a motive; its value is as a second signal beside an actor
+ * that has already failed something sharper.
+ */
+export declare function parameterSweepDetector(options?: ParameterSweepOptions): Detector;

package/dist/detectors/probe-volume.d.ts

@@ -0,0 +1,26 @@
+import type { Detector } from "./types.js";
+export interface ProbeVolumeOptions {
+    /** Reported responses before the ratio means anything. Default 20. */
+    minResponses?: number;
+    /** Share of them that must be misses. Default 0.8. */
+    missRatio?: number;
+}
+/**
+ * An actor that is looking for something rather than reading anything.
+ *
+ * The oldest tell there is for a scanner, and the one this library could not see: it
+ * decides *before* the response exists, which is what lets it shape the response and also
+ * what hides the status code from it. Fed back through `recordOutcome`, the shape is
+ * unmistakable — a person browsing a site does not generate forty misses in a row, and a
+ * wordlist does almost nothing else.
+ *
+ * Counts 404 and 410 only. A 403 is usually this library's own doing, and counting it
+ * would let a rule that challenges an actor manufacture the evidence for challenging it.
+ * A 500 is the site's problem and says nothing about the client.
+ *
+ * `moderate`, not higher. A site that has just moved its URLs produces this from perfectly
+ * ordinary readers, and so does a feed reader working through a list of removed articles.
+ * It is also entirely absent unless the application reports outcomes, which is why nothing
+ * else depends on it.
+ */
+export declare function probeVolumeDetector(options?: ProbeVolumeOptions): Detector;

package/dist/detectors/transport-coherence.d.ts

@@ -0,0 +1,31 @@
+import type { Detector } from "./types.js";
+export interface TransportCoherenceOptions {
+    /**
+     * Report a claimed browser arriving over HTTP/1.0. Default true.
+     *
+     * Turn it off if something in front of this application speaks HTTP/1.0 to it. A few
+     * older load balancers and reverse proxies still do, and where that is true every
+     * request arrives that way — so the signal says something about your infrastructure
+     * rather than about your visitors, and a detector that fires on all of them is worse
+     * than one that fires on none.
+     */
+    legacyHttp?: boolean;
+    /** Requests an actor must have made before an all-HEAD visit means anything. Default 8. */
+    minHeadRequests?: number;
+}
+/**
+ * How a claimed browser *moves*, rather than what it says.
+ *
+ * The header checks read one request against the client it claims to be. This reads the
+ * transport underneath and the verbs across a visit, which are harder to copy because
+ * they are not in the part of the request most tools let you set.
+ *
+ * Two things, both measured as blind spots before this existed — a client claiming
+ * Chrome 120 over HTTP/1.0, and one whose entire visit is HEAD, each scored exactly what
+ * the honest control scored.
+ *
+ * Neither goes above `moderate`, and the reasons are different. HTTP/1.0 can be an
+ * intermediary's doing rather than the client's. An all-HEAD visit is a strong shape but a
+ * link checker is a real and mostly harmless thing to be.
+ */
+export declare function transportCoherenceDetector(options?: TransportCoherenceOptions): Detector;

package/dist/element/index.cjs

@@ -3878,7 +3878,10 @@ button[disabled] { opacity: .5; cursor: default; }
 .tab[aria-selected="true"] { color: var(--ink); border-bottom-color: var(--s1); }
 
 /* --- layout ------------------------------------------------------------- */
-main { padding: 18px 20px 64px; max-width: 1680px; margin: 0 auto; }
+/* The top padding is the gap under the sticky header. At 18px the counter row sat almost
+   against the header's border and read as part of it; the tiles carry their own border, so
+   two lines were meeting with nothing between them. */
+main { padding: 28px 20px 64px; max-width: 1680px; margin: 0 auto; }
 .stack { display: grid; gap: 16px; }
 /* Everything above the feed is drawn by script once the first snapshot arrives, which
    inserts a block of content above what is already laid out. The browser's scroll