@osqd/bothandlerjs 0.3.0

package/dist/corpus/infrastructure.d.ts

@@ -0,0 +1,17 @@
+import type { TrafficCase } from "./schema.js";
+/**
+ * Traffic from the machinery around your application.
+ *
+ * Load balancer probes, container orchestration health checks, your own mobile app,
+ * your own server-side renderer, webhooks arriving from a payment processor, a
+ * browser prefetching a page a person has not asked for yet. None of it is a person,
+ * almost none of it is unwelcome, and most of it will be classified as automation
+ * because that is exactly what it is.
+ *
+ * The lesson this section is here to teach is that **the answer is usually the
+ * allowlist or `ignorePaths`, not a detector**. A health check from your own load
+ * balancer should never reach detection at all: it costs work, it inflates every
+ * per-actor counter with perfectly regular traffic, and the one thing you must never
+ * do is let a bot policy decide whether your orchestrator thinks you are alive.
+ */
+export declare const INFRASTRUCTURE_CASES: TrafficCase[];

package/dist/corpus/libraries-extended.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const EXTENDED_LIBRARY_CASES: TrafficCase[];

package/dist/corpus/ranges.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Published IP ranges the corpus assumes.
+ *
+ * Several crawlers publish an exhaustive address list instead of setting PTR records,
+ * and the library can only confirm or refute those claims when it has been handed the
+ * list. These are the fictional stand-ins the corpus uses, drawn entirely from the
+ * documentation ranges reserved by RFC 5737 so that nothing here can ever collide
+ * with a real network.
+ *
+ * The convention, which the cases rely on:
+ *
+ * - **inside** the range   → the claim is confirmed  → `verified-bot`
+ * - **outside** the range  → the claim is refuted    → `impersonator`
+ *
+ * Cases pick their address to land on the side they mean to test.
+ */
+export declare const CORPUS_CRAWLER_RANGES: Readonly<Record<string, readonly string[]>>;
+/** An address inside a crawler's published range. */
+export declare const IN_RANGE: Readonly<Record<string, string>>;
+/** Outside every published range above — the address a forgery would come from. */
+export declare const OUT_OF_RANGE = "192.0.2.66";
+/**
+ * Real Googlebot address space, used only as a *shape* for the FCrDNS cases.
+ *
+ * No lookup ever leaves the process: the runner answers from each case's own `dns`
+ * map. The address is realistic so that a reader recognises what is being modelled.
+ */
+export declare const GOOGLEBOT_IP = "66.249.66.1";
+export declare const GOOGLEBOT_PTR = "crawl-66-249-66-1.googlebot.com";
+export declare const BINGBOT_IP = "40.77.167.1";
+export declare const BINGBOT_PTR = "msnbot-40-77-167-1.search.msn.com";

package/dist/corpus/reputation.d.ts

@@ -0,0 +1,21 @@
+import type { TrafficCase } from "./schema.js";
+/**
+ * Address reputation and prior clearance.
+ *
+ * Two mechanisms that only work when you have configured them, and that behave very
+ * differently from everything else in the corpus:
+ *
+ * **Address ranges** are operator data. The library ships none, because an
+ * address-to-operator mapping goes stale within weeks and a stale mapping is a false
+ * positive with a long half-life. A denylist entry is `certain` — not because we
+ * deduced anything, but because you decided it — while a datacenter match is capped
+ * at `moderate` forever, since consumer VPNs, corporate gateways, Tor exits and
+ * privacy relays all live in the same address space as the scrapers.
+ *
+ * **Clearance** is the only source of human-pointing evidence the library has, and
+ * the levels are not interchangeable. A solved proof of work shows that a JavaScript
+ * engine ran and CPU was spent — a headless Chrome does both. Only `operator`
+ * clearance, which your own application issues on evidence the request does not
+ * carry, is treated as conclusive.
+ */
+export declare const REPUTATION_CASES: TrafficCase[];

package/dist/corpus/runner.d.ts

@@ -0,0 +1,130 @@
+import { BotHandler } from "../core.js";
+import { ManualClock } from "../internal/clock.js";
+import type { Assessment } from "../types.js";
+import type { Decision } from "../policy/types.js";
+import type { ActionOutcome } from "../actions/types.js";
+import type { DnsResolver } from "../internal/dns.js";
+import type { Audience, TrafficCase } from "./schema.js";
+/**
+ * The harness.
+ *
+ * It takes a factory rather than a handler because the corpus tests **your**
+ * configuration, not the defaults. Hand it whatever you run in production and it will
+ * tell you what that policy does to every shape of traffic in the corpus — including,
+ * and this is the part worth reading, which of them are people.
+ *
+ * Two things it controls that a real deployment does not:
+ *
+ * - **DNS.** No lookup leaves the process. Each case declares the answers it wants,
+ *   so the difference between "the operator's DNS disproves this claim" and "our
+ *   resolver was briefly unhappy" can actually be tested. Those must reach different
+ *   verdicts, and only a controlled resolver can prove they do.
+ * - **Time.** A `ManualClock` advances by each request's `atMs`, so rate and cadence
+ *   are exercised deterministically. A corpus that slept would take hours and still
+ *   be flaky.
+ */
+export interface RunnerOptions {
+    /** Builds the handler under test. The runner supplies the resolver and the clock. */
+    create: (dependencies: {
+        resolver: DnsResolver;
+        clock: ManualClock;
+    }) => BotHandler;
+    cases?: readonly TrafficCase[];
+    /**
+     * Load the corpus's fictional crawler ranges into the handler. Default true.
+     *
+     * Without them the `ip-ranges` verification cases cannot pass, because the library
+     * ships no address data and correctly refuses to guess.
+     */
+    applyCorpusRanges?: boolean;
+    /** Epoch ms the first request of every case is stamped with. */
+    startedAt?: number;
+    /**
+     * Capability names the handler under test provides, matched against a case's
+     * `requires`. A case naming something absent here is skipped and reported.
+     */
+    provides?: readonly string[];
+    /**
+     * Assert each case's `expect.action`. Default true.
+     *
+     * Turn it off when running the corpus across several policies. Which *action* a
+     * case receives is a property of the policy — a case that is allowed under one
+     * preset is logged under another and both are correct — whereas the verdict, the
+     * certainty and `neverAction` are properties of the traffic and hold everywhere.
+     * `neverAction` is always asserted, because it is the invariant, not a preference.
+     */
+    assertActions?: boolean;
+}
+export interface RequestResult {
+    assessment: Assessment;
+    decision: Decision;
+    outcome: ActionOutcome;
+}
+export interface CaseResult {
+    case: TrafficCase;
+    /** Every request in the case, in order. */
+    requests: RequestResult[];
+    /** The request expectations are asserted against — the last one. */
+    final: RequestResult;
+    /** Human-readable expectation violations. Empty means the case passed. */
+    failures: string[];
+    /** Set when a case marked `human` was denied service. The failure that matters most. */
+    falsePositive: boolean;
+    /** Why the case did not run, when it did not. Never counted as a pass. */
+    skipped?: string;
+    durationMs: number;
+}
+export interface AudienceTally {
+    total: number;
+    passed: number;
+    failed: number;
+    actions: Record<string, number>;
+}
+export interface Scorecard {
+    results: CaseResult[];
+    total: number;
+    passed: number;
+    failed: number;
+    /** Human cases that were denied service. Must be empty. */
+    falsePositives: CaseResult[];
+    /** Cases the handler under test was not configured to exercise. */
+    skipped: CaseResult[];
+    /**
+     * People whose client software declares itself automated.
+     *
+     * Exempt from the never-deny rule by construction, and listed here so that the
+     * exemption is visible in every report rather than buried in a flag.
+     */
+    selfDeclaredHumans: CaseResult[];
+    byAudience: Record<Audience, AudienceTally>;
+    byCategory: Record<string, {
+        total: number;
+        failed: number;
+    }>;
+    /** How many cases each detector produced evidence on. */
+    detectorCoverage: Record<string, number>;
+    /**
+     * Detectors installed in the handler that no case exercised.
+     *
+     * A gap in the corpus, not in the library: an untested detector is one whose next
+     * regression nobody will notice.
+     */
+    unexercisedDetectors: string[];
+    /** Proven-verdict rate among cases that are genuinely automated. */
+    provenAutomation: {
+        total: number;
+        proven: number;
+    };
+    durationMs: number;
+}
+/**
+ * Derives a stable address for a case from its id.
+ *
+ * Distinct per case, so no case inherits another's behavioural history, and stable
+ * across runs, so a failure is reproducible. Drawn from 198.18.0.0/15, which RFC 2544
+ * reserves for benchmarking — it cannot collide with anything real, and its meaning is
+ * exactly what this is.
+ */
+export declare function addressFor(id: string): string;
+export declare function runCase(handler: BotHandler, clock: ManualClock, item: TrafficCase, startedAt: number, provides: ReadonlySet<string>, assertActions?: boolean): Promise<CaseResult>;
+export declare function runCorpus(options: RunnerOptions): Promise<Scorecard>;

package/dist/corpus/schema.d.ts

@@ -0,0 +1,185 @@
+import type { ActionName } from "../policy/types.js";
+import type { BotClass, Verdict } from "../types.js";
+/**
+ * The corpus schema.
+ *
+ * This is a body of *realistic web traffic* — real User-Agent strings, real header
+ * sets in the order real clients send them, real behavioural shapes — paired with
+ * what the library ought to conclude about each. It exists to answer one question
+ * that no unit test can:
+ *
+ *   > If I point this configuration at the actual internet, who gets hurt?
+ *
+ * The corpus is deliberately separate from the configuration it tests. Point the
+ * runner at your own `BotHandler` and it tells you how *your* policy behaves against
+ * every shape of traffic in here — not how the defaults behave.
+ *
+ * ## Adding cases
+ *
+ * Add to the file matching the audience, or create a new file and register it in
+ * `index.ts`. Every case needs `provenance`: where the shape came from. A fixture
+ * somebody invented is worth much less than one copied from a log, and six months
+ * later nobody can tell them apart unless it is written down.
+ */
+/**
+ * Who is behind the request — and, more to the point, what it costs to be wrong.
+ *
+ * This is the axis that matters, because the corpus's central assertion is expressed
+ * in terms of it: **no case marked `human` may ever be denied service.** That check
+ * runs against every case in the corpus regardless of what its own expectations say,
+ * so a new human case protects you the moment it is added.
+ */
+export type Audience = 
+/** A person. Being wrong here means turning away a customer. */
+"human"
+/** Automation almost every site wants: search crawlers, link unfurlers, uptime monitors. */
+ | "benign-bot"
+/** Honest automation whose treatment is a business decision, not a security one. */
+ | "declared-bot"
+/** Automation most sites would rather not serve: aggressive SEO, bulk extraction. */
+ | "unwanted-bot"
+/** Scanners, forged identities, credential attacks. */
+ | "hostile"
+/** Something in the middle of the path: a proxy, a CDN, a corporate gateway. */
+ | "infrastructure";
+/** One HTTP request, as a client would actually send it. */
+export interface CaseRequest {
+    method?: string;
+    /** Path plus query, as it appears on the request line. */
+    path?: string;
+    /**
+     * Headers **in the order the client sends them**.
+     *
+     * A tuple list rather than an object, because order is itself a fingerprint and an
+     * object literal's key order is too easy to disturb by accident. `headers.ts` builds
+     * these for the real browsers.
+     */
+    headers: ReadonlyArray<readonly [name: string, value: string]>;
+    ip?: string;
+    protocol?: "http" | "https";
+    httpVersion?: string;
+    tlsFingerprint?: string;
+    /** Milliseconds after the case's start time. Drives the behavioural detectors. */
+    atMs?: number;
+    /** The source could not supply the full header set. See `RequestFacts.partialHeaders`. */
+    partialHeaders?: boolean;
+}
+export type OneOrMany<T> = T | readonly T[];
+/** What the library should conclude. Every field is optional; absent means "don't care". */
+export interface Expectation {
+    verdict?: OneOrMany<Verdict>;
+    botClass?: OneOrMany<BotClass>;
+    /** Whether the conclusion must rest on proven evidence. */
+    certain?: boolean;
+    /** Established or claimed identity, e.g. `"googlebot"`. */
+    identity?: string;
+    minScore?: number;
+    maxScore?: number;
+    /** Detectors that must all have produced evidence. */
+    detectors?: readonly string[];
+    /** Detectors that must not have fired. The false-positive guard, stated per case. */
+    notDetectors?: readonly string[];
+    /** Actions that are acceptable. */
+    action?: OneOrMany<ActionName>;
+    /** Actions that must never be taken. Enforced in addition to the audience rule. */
+    neverAction?: readonly ActionName[];
+    /**
+     * What the action layer actually did, which is not always what the policy decided.
+     *
+     * A decision to challenge an actor that already holds clearance is refused and the
+     * request is served instead — the guard that stops a proven bot from looping on the
+     * challenge forever. That distinction is invisible from `action` alone.
+     */
+    outcome?: "continue" | "respond" | "drop";
+}
+export interface TrafficCase {
+    /** Stable, unique, kebab-case. Appears in every report; treat it as an identifier. */
+    id: string;
+    title: string;
+    audience: Audience;
+    /** Finer grouping within an audience, e.g. `"desktop-browser"`, `"ai-crawler"`. */
+    category: string;
+    /**
+     * Where this shape came from: a published User-Agent list, a vendor's documentation,
+     * an observed log line, a specification. Required — a fixture nobody can trace is a
+     * fixture nobody can update when the world moves.
+     */
+    provenance: string;
+    /** Anything a reader needs in order to judge whether the expectation is right. */
+    notes?: string;
+    /** A single request, or an ordered sequence from one actor. */
+    requests: readonly CaseRequest[];
+    expect: Expectation;
+    tags?: readonly string[];
+    /**
+     * Configuration this case depends on, as free-text capability names.
+     *
+     * Some traffic can only be classified by a detector that has been *told* something
+     * — the trap detector cannot recognise a honeypot form field whose name it was
+     * never given. Rather than quietly failing against a default configuration, such a
+     * case declares what it needs; the runner skips it and says so when the handler
+     * under test does not provide it. A skipped case is reported, never counted as a
+     * pass, because "we did not check" and "it worked" must not look the same.
+     */
+    requires?: readonly string[];
+    /**
+     * Mint a clearance token for this actor before the case runs, and attach it to
+     * every request.
+     *
+     * The token has to be signed by the handler under test, so the corpus cannot carry
+     * a literal cookie — it declares the level it wants and the runner asks the handler
+     * for one. `operator` is the only level the library treats as conclusive proof of a
+     * person, because it is an assertion by your application rather than an inference
+     * from the request.
+     */
+    clearance?: "pow" | "interaction" | "operator";
+    /**
+     * A person whose *client software* declares itself automated, and why.
+     *
+     * The never-deny guarantee attached to `human` cases is a promise about guesses:
+     * nobody is refused on the strength of an inference. It is not a promise that the
+     * library can see through a client that announces itself as a bot. A podcast
+     * application uses one User-Agent both to fetch feeds and to open the links a
+     * listener taps; when the listener taps one, the library reads the declaration the
+     * software made and is correct to.
+     *
+     * Setting this exempts the case from the audience rule and, crucially, lists it in
+     * the scorecard under its own heading. These are the people the design knowingly
+     * cannot protect, and the corpus makes them countable rather than invisible.
+     */
+    selfDeclared?: string;
+    /**
+     * DNS the runner should present while this case runs.
+     *
+     * Forward-confirmed reverse DNS is the only mechanism that can produce a
+     * `verified-bot` or prove an `impersonator`, so a corpus that cannot control DNS
+     * cannot test the two most consequential verdicts the library reaches. `reverse`
+     * maps address to PTR names; `forward` maps name to addresses. A name absent from
+     * either map resolves to NXDOMAIN, which is a *definitive* negative answer — the
+     * runner never presents a timeout unless a case asks for one, because "no answer"
+     * and "the wrong answer" must lead to different verdicts.
+     */
+    dns?: {
+        reverse?: Readonly<Record<string, readonly string[]>>;
+        forward?: Readonly<Record<string, readonly string[]>>;
+        /** Make every lookup time out, to exercise the indeterminate path. */
+        unavailable?: boolean;
+    };
+}
+/** Terminal actions. A human case reaching any of these is a corpus failure. */
+export declare const DENYING_ACTIONS: readonly ActionName[];
+/**
+ * Declares a case involving a person.
+ *
+ * Adds the never-deny guarantee automatically, so it cannot be forgotten, and defaults
+ * the verdict expectation to "not classified as a bot". Override `expect` for the
+ * awkward cases — and there are several, because some real people do look automated.
+ */
+export declare function human(input: Omit<TrafficCase, "audience"> & {
+    audience?: never;
+}): TrafficCase;
+/** Declares a case involving automation. No implicit guarantees; say what you mean. */
+export declare function bot(input: TrafficCase): TrafficCase;
+/** Repeats a request `count` times at a fixed interval. For rate and cadence shapes. */
+export declare function repeat(template: CaseRequest, count: number, everyMs: number, pathAt?: (index: number) => string): CaseRequest[];
+export declare function humanPaced(template: CaseRequest, paths: readonly string[]): CaseRequest[];

package/dist/corpus/tooling.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const TOOLING_CASES: TrafficCase[];

package/dist/corpus/unwanted.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const UNWANTED_BOT_CASES: TrafficCase[];

package/dist/crawler-ranges.d.ts

@@ -0,0 +1,113 @@
+import type { BotHandler } from "./core.js";
+/**
+ * Published crawler address ranges.
+ *
+ * Twelve of the signatures in `known-bots.ts` verify by address rather than by reverse
+ * DNS — every AI crawler among them — and until this file existed nothing in the
+ * library ever filled those ranges in. `updateCrawlerRanges()` was a method waiting for
+ * a caller, and `crawler-verification` correctly declined to confirm anybody, because a
+ * claim it cannot check is a claim it must not endorse.
+ *
+ * Verifying by range rather than by reverse DNS is better in three ways where it is
+ * available: it is a lookup instead of a network round trip on the request path, it
+ * cannot be broken by somebody else's DNS having a bad afternoon, and it works for the
+ * several crawlers that publish ranges and no useful `PTR` record at all.
+ *
+ * **This is opt-in, and it makes outbound requests.** The library otherwise talks to
+ * nothing but your own DNS resolver, and a dependency-free package quietly fetching
+ * URLs on a timer is not something to inherit by accident.
+ *
+ * ```ts
+ * const stop = startCrawlerRangeRefresh(botHandler, { intervalMs: 12 * 60 * 60_000 });
+ * ```
+ */
+/** One publisher, and where it says its addresses are. */
+export interface PublishedRangeSource {
+    /** The signature id these ranges belong to. Must match a `BotSignature.id`. */
+    id: string;
+    /** Where the list is published. HTTPS only — see {@link fetchCrawlerRanges}. */
+    url: string;
+}
+/**
+ * The lists shipped with the library, as pointers rather than as data.
+ *
+ * The distinction matters. This library ships **no address data** and refuses to guess
+ * any, because a range baked into a release is a range that is wrong by the time
+ * somebody installs it — and being wrong here means either failing to verify a real
+ * crawler or, far worse, verifying somebody who has since been handed the address. What
+ * it ships is the URL each operator publishes, so the answer always comes from the
+ * party entitled to give it.
+ *
+ * Nothing here is guaranteed to stay reachable. A publisher that moves its file simply
+ * stops being verifiable, which is the same position the library was in before, and the
+ * refresh says so rather than failing quietly. Pass your own `sources` to add, replace
+ * or pin any of it.
+ */
+export declare const PUBLISHED_CRAWLER_RANGES: readonly PublishedRangeSource[];
+export interface RefreshOptions {
+    /** Which lists to fetch. Defaults to {@link PUBLISHED_CRAWLER_RANGES}. */
+    sources?: readonly PublishedRangeSource[];
+    /** How long any one request may take. Default 10 seconds. */
+    timeoutMs?: number;
+    /**
+     * The fetcher. Defaults to the global `fetch`.
+     *
+     * Injectable because a test must not reach the internet, and because some
+     * deployments only reach the outside world through a proxy they configure
+     * themselves.
+     */
+    fetch?: typeof globalThis.fetch;
+    /** Attributed on the `range-change` event, like any other runtime change. */
+    by?: string;
+}
+export interface RefreshResult {
+    updated: Array<{
+        id: string;
+        prefixes: number;
+    }>;
+    /** Sources that could not be used, and why. Never throws — see below. */
+    failed: Array<{
+        id: string;
+        reason: string;
+    }>;
+}
+/**
+ * Fetches one published list and returns the prefixes in it.
+ *
+ * Two formats, because those are the two anybody publishes: a JSON document with a
+ * `prefixes` array of `{ ipv4Prefix }` / `{ ipv6Prefix }` objects, which is what Google
+ * standardised and what the AI crawlers copied; and a plain-text list of one address or
+ * CIDR per line, which is what the older monitoring services publish. Anything else is
+ * an error rather than a guess.
+ */
+export declare function fetchCrawlerRanges(source: PublishedRangeSource, options?: RefreshOptions): Promise<string[]>;
+/**
+ * Fetches every published list and installs it.
+ *
+ * **Never throws and never partially applies.** Each source is independent: one
+ * publisher being down, having moved its file or serving something unrecognisable
+ * leaves every other crawler's ranges exactly as they were, and leaves *that* crawler's
+ * ranges as they were too. The failure is reported in the result and raised as a
+ * warning on the handler, which is where the dashboard's notices panel picks it up.
+ *
+ * That is the same fail-open rule the rest of the library follows: an outage in
+ * something the detection consults must degrade detection, never take down the site it
+ * protects — and here "degrade" means a crawler goes back to being unverifiable, which
+ * is exactly the state it was in before this was ever called.
+ */
+export declare function refreshCrawlerRanges(handler: BotHandler, options?: RefreshOptions): Promise<RefreshResult>;
+export interface ScheduleOptions extends RefreshOptions {
+    /** How often to refresh. Default 12 hours; the minimum is one hour. */
+    intervalMs?: number;
+    /** Fetch immediately as well as on the interval. Default true. */
+    immediate?: boolean;
+}
+/**
+ * Refreshes on a schedule, and returns the way to stop.
+ *
+ * Twice a day by default, because these lists change on the order of weeks and a
+ * library that polls somebody else's endpoint more often than that is being rude with
+ * your egress and their bandwidth. The timer is unreferenced, so it never keeps a
+ * process alive on its own.
+ */
+export declare function startCrawlerRangeRefresh(handler: BotHandler, options?: ScheduleOptions): () => void;