@osqd/bothandlerjs 0.3.0

package/dist/core.d.ts

@@ -0,0 +1,443 @@
+import { ActorRegistry } from "./state.js";
+import { ChallengeService } from "./challenge/index.js";
+import { TrafficAudit } from "./audit.js";
+import { NotificationHub } from "./notify/hub.js";
+import { Policy } from "./policy/policy.js";
+import type { GuardSettings } from "./policy/policy.js";
+import type { ActionOutcome } from "./actions/types.js";
+import type { Rule } from "./policy/types.js";
+import type { Assessment, RequestFacts } from "./types.js";
+import type { BotHandlerConfig, ResolvedConfig } from "./config.js";
+import type { TrafficAnomaly } from "./audit.js";
+import type { DashboardOptions, DashboardServer } from "./dashboard/types.js";
+import type { MetricsSnapshot, PrometheusOptions } from "./metrics.js";
+import type { BotHandlerStore } from "./stores/types.js";
+import type { Decision } from "./policy/types.js";
+import type { SolutionOutcome } from "./challenge/index.js";
+export interface HandleResult {
+    assessment: Assessment;
+    decision: Decision;
+    outcome: ActionOutcome;
+}
+/**
+ * Everything the engine will tell you about, and the whole integration surface.
+ *
+ * Each of these has a matching `onX` in the config — the same mechanism, registered
+ * for you at construction. Subscribe here when you want to add a listener later,
+ * add several, or remove one; use the config form when you just want a callback.
+ *
+ * They are deliberately *derived* rather than raw: `denial` and `downgrade` exist
+ * because "a request was refused" and "the guard refused a rule" are the two things
+ * an operator wants an alert on, and making every subscriber re-derive them from
+ * `decision` is how two integrations end up disagreeing about what a denial is.
+ */
+export interface BotHandlerEvents extends Record<string, unknown> {
+    /** Every assessment, including the ones that concluded nothing. */
+    assessment: Assessment;
+    /** Every decision, with the assessment behind it. */
+    decision: {
+        assessment: Assessment;
+        decision: Decision;
+    };
+    /** A request that was actually denied: `block`, `drop` or `redirect`. */
+    denial: {
+        assessment: Assessment;
+        decision: Decision;
+    };
+    /** The guard replaced a terminal action with something recoverable. */
+    downgrade: {
+        assessment: Assessment;
+        decision: Decision;
+    };
+    /** A challenge was issued, solved or rejected. */
+    challenge: {
+        phase: "issued" | "solved" | "rejected";
+        actorKey?: string | undefined;
+    };
+    /**
+     * A detector threw or timed out.
+     *
+     * `requestId` is empty when the failure was noticed before the assessment was
+     * assembled — the failure list on the assessment is the per-request record; this
+     * event is the operational one, for the alert that says a resolver is down.
+     */
+    "detector-failure": {
+        detector: string;
+        reason: string;
+        message: string;
+        requestId: string;
+    };
+    /** The rule set was replaced at runtime. */
+    "policy-change": {
+        rules: readonly string[];
+        warnings: readonly string[];
+        by?: string | undefined;
+    };
+    /**
+     * The guard settings were changed at runtime.
+     *
+     * Separate from `policy-change` on purpose. "Which rules exist" and "how far a rule
+     * is allowed to go" are different powers with different consequences, and an
+     * operator wiring an alert almost always wants the second one and not the first.
+     * `before` and `after` both travel so the alert can say what actually moved.
+     */
+    "guard-change": {
+        before: GuardSettings & {
+            suspectThreshold: number;
+        };
+        after: GuardSettings & {
+            suspectThreshold: number;
+        };
+        by?: string | undefined;
+    };
+    /**
+     * A range set was replaced at runtime — an allowlist entry added, a crawler's
+     * published ranges refreshed.
+     *
+     * Worth an event of its own because the allowlist is the one list that stops
+     * detection *running*: an address on it is not judged leniently, it is not judged at
+     * all. A change to it is a change to what your bot handling can see.
+     */
+    "range-change": {
+        name: string;
+        size: number;
+        entries: readonly string[];
+        by?: string | undefined;
+    };
+    /**
+     * An actor's behavioural memory was changed by hand — forgotten, or granted human
+     * clearance.
+     *
+     * The remedy for a false positive that has stuck to somebody, and therefore exactly
+     * the operation an audit trail wants to have seen.
+     */
+    "actor-change": {
+        key: string;
+        action: "forget" | "clear";
+        until?: number | undefined;
+        by?: string | undefined;
+    };
+    /** The audit noticed the traffic change shape. */
+    anomaly: TrafficAnomaly;
+    warning: string;
+    error: {
+        error: unknown;
+        source: string;
+    };
+}
+/**
+ * Who asked for a runtime change.
+ *
+ * Every mutating method takes one, and every one of them puts it in the warning and in
+ * the event. The library has no idea who anybody is — it has no user model and does not
+ * want one — so this is a string the caller supplies, and the caller is the thing that
+ * *does* know: an authenticating dashboard, a deploy pipeline, an admin CLI. An absent
+ * `by` is not a failure; it means whatever made the change could not say who.
+ */
+export interface ChangeContext {
+    by?: string | undefined;
+}
+/** Options for a single assessment. */
+export interface AssessOptions {
+    /**
+     * Whether this request is part of your traffic. Default true.
+     *
+     * `record: false` is a **dry run**: the detectors all run and the verdict is real,
+     * but nothing is written down. No actor state moves, no counter increments, no
+     * `assessment` event fires and no notification is sent — so asking "what would this
+     * request be judged as?" does not become a row in the answer to "what is my traffic
+     * doing?".
+     *
+     * It is what the dashboard's request tester uses, and what to reach for anywhere else
+     * you want the engine's opinion about a request that is not happening: a support
+     * ticket, a test, a rule you are drafting. The one thing a dry run cannot see is
+     * history — it gets an actor with no past, because the alternative is to record the
+     * request against a real one, which is the thing it promised not to do.
+     */
+    record?: boolean;
+}
+/** Description of a registered detector, for documentation and diagnostics. */
+export interface DetectorDescription {
+    id: string;
+    description: string;
+    cost: "cheap" | "io";
+    stage: "always" | "confirming";
+}
+/**
+ * The engine.
+ *
+ * Three separable steps, in a fixed order, each usable on its own:
+ *
+ * 1. {@link assess} — gather evidence and reach a verdict. Reads the request, touches
+ *    no response, and is safe to call anywhere, including from a log processor
+ *    replaying yesterday's traffic.
+ * 2. {@link decide} — apply the policy, subject to the safety guard.
+ * 3. {@link handle} — do both, then turn the decision into an outcome an adapter can
+ *    apply.
+ *
+ * Keeping them apart is what makes the library testable and what makes a
+ * monitor-only deployment a first-class mode rather than a configuration trick: call
+ * `assess` alone and you have a detector with no opinions about your traffic at all.
+ *
+ * Everything is failure-isolated. A detector that throws, a store that will not
+ * answer, a notification sink that hangs — each degrades exactly itself and is
+ * reported through `onError`. Nothing in this file can turn a bad day for a
+ * dependency into a bad day for the site it is protecting.
+ */
+export declare class BotHandler {
+    readonly config: ResolvedConfig;
+    readonly registry: ActorRegistry;
+    readonly store: BotHandlerStore;
+    readonly policy: Policy;
+    readonly challenge: ChallengeService | undefined;
+    readonly notifications: NotificationHub;
+    /**
+     * The traffic audit, or `undefined` when it was switched off with `audit: false`.
+     *
+     * Exposed so you can ask it questions on your own schedule — `summary()` for the
+     * two windows as they stand, `evaluate()` to run the checks now — which is what a
+     * health endpoint or a cron job wants rather than waiting for the timer.
+     */
+    readonly audit: TrafficAudit | undefined;
+    private readonly signatures;
+    private readonly resolver;
+    private readonly handlers;
+    private readonly cheapDetectors;
+    private readonly ioDetectors;
+    private readonly confirmingDetectors;
+    private readonly events;
+    private readonly ignoreExact;
+    private readonly ignorePatterns;
+    private readonly isHuman;
+    private readonly meter;
+    /** Hoisted out of the metrics object: read once per detector per request. */
+    private readonly timing;
+    constructor(options?: BotHandlerConfig);
+    /**
+     * Runs the audit's checks now and emits whatever they found.
+     *
+     * The timer does this on a schedule; calling it yourself is for the cases a timer
+     * cannot serve — a health endpoint that should report the current picture, a cron
+     * job, a test with a manual clock.
+     */
+    runAudit(): TrafficAnomaly[];
+    private raiseAnomaly;
+    on<K extends keyof BotHandlerEvents & string>(event: K, listener: (payload: BotHandlerEvents[K]) => void): () => void;
+    /**
+     * A point-in-time copy of the counters. Cheap enough to call on every scrape.
+     *
+     * Returns `undefined` when metrics were switched off with `metrics: false`.
+     */
+    metrics(): MetricsSnapshot | undefined;
+    /** The same counters in Prometheus text exposition format. */
+    prometheus(options?: PrometheusOptions): string | undefined;
+    /**
+     * Starts the operator dashboard on a listener of its own.
+     *
+     * ```ts
+     * const dashboard = await botHandler.serveDashboard({
+     *   port: 9674,
+     *   auth: { username: "ops", password: process.env.DASH_PASSWORD! },
+     * });
+     * console.log(dashboard.url);
+     * ```
+     *
+     * **Its own listener, not a route in your application.** Mounting it inside the app
+     * it reports on puts it behind the bot handler, and that arrangement has three
+     * separate failure modes: reading the dashboard shows up in the dashboard, a
+     * challenge served to your site can lock you out of the tool you are using to read
+     * about it, and the page becomes reachable at whatever authentication your public
+     * site happens to have. A second port is the cheapest fix for all three.
+     *
+     * **Loopback by default, and it refuses to bind anywhere else without an explicit
+     * `auth`.** This page shows client addresses and, per request, the exact evidence
+     * behind the verdict — which is a tuning guide for anyone building a scraper
+     * against you. See {@link DashboardOptions.host}.
+     *
+     * The returned handle carries the URL to open (with the real port, which matters if
+     * you passed 0) and a `close()` that stops the listener, drops the event streams and
+     * unsubscribes from this handler.
+     */
+    serveDashboard(options?: DashboardOptions): Promise<DashboardServer>;
+    /**
+     * Replaces an IP range set while the process runs.
+     *
+     * Published crawler ranges are the reason this exists. Operators revise them, a
+     * stale list turns a verified crawler into an accused impersonator, and requiring a
+     * restart to pick up a new one means the list is refreshed roughly never. Fetch
+     * them on whatever schedule you like and hand them here.
+     *
+     * The swap is atomic — the new set is fully parsed and validated before it replaces
+     * the old one — so a request assessed mid-update sees one list or the other, never
+     * a half-built one. Invalid input throws and leaves the existing set in place,
+     * because a range set that silently matches nothing is worse than a stale one.
+     */
+    updateRanges(name: string, entries: readonly string[], context?: ChangeContext): void;
+    /** The ranges in a set, as written. `undefined` when no set of that name exists. */
+    rangeEntries(name: string): readonly string[] | undefined;
+    /**
+     * Discards one actor's behavioural memory.
+     *
+     * The remedy for a false positive that has stuck: a person whose actor key collected
+     * a `confirmed-bot` — a shared office address, a phone that reused an IP — carries
+     * `priorConfirmations` for the rest of the window, and any rule reading
+     * `minPriorConfirmations` keeps matching them. Until this existed the only cure was
+     * `registry.clear()`, which throws away every actor's history to fix one.
+     *
+     * It is not an allowlist: the next request from this actor is assessed exactly as any
+     * first request would be.
+     */
+    forgetActor(key: string, context?: ChangeContext): void;
+    /**
+     * Grants an actor human clearance for a while, as though it had solved a challenge.
+     *
+     * The `clearance` detector reads it, so this is an assertion about a person made by
+     * an operator on evidence the request does not carry — the same category of claim as
+     * `isHuman`, and it expires the same way a solved challenge does.
+     */
+    clearActor(key: string, forMs: number, context?: ChangeContext): void;
+    /** Convenience for `updateRanges("crawler:<id>", …)`, matching a signature id. */
+    updateCrawlerRanges(signatureId: string, entries: readonly string[], context?: ChangeContext): void;
+    /**
+     * Replaces the rule list while the process runs.
+     *
+     * The dashboard's policy editor is the reason this exists, and the shape of it is
+     * deliberate: you may change **which rules exist**, and nothing about **how far a
+     * rule is allowed to go**. `falsePositivePolicy`, `fallbackAction` and
+     * `terminalScoreThreshold` are fixed at construction. A dashboard — or anything
+     * else holding a reference to this handler — therefore cannot relax the guard that
+     * stops an unproven verdict from denying somebody. Loosening that is a deploy, on
+     * purpose, reviewed by whoever reviews deploys.
+     *
+     * Validation runs first and throws on a rule that could never work; the swap only
+     * happens if every rule survives it, so a bad edit leaves the running policy exactly
+     * as it was. Warnings — a duplicate id, a rule that can never match — come back to
+     * the caller *and* go to `onWarning`, because a policy changed at runtime should
+     * leave a trace in the same place a policy loaded at startup does.
+     */
+    updatePolicy(rules: readonly Rule[], context?: ChangeContext): {
+        warnings: string[];
+    };
+    /**
+     * Changes the guard settings while the process runs.
+     *
+     * The one runtime change that can start denying people, and it is kept apart from
+     * {@link updatePolicy} for exactly that reason: a separate method, a separate
+     * dashboard control, a separate event. Fields left out keep their current value.
+     *
+     * Everything is validated before anything is applied, so a rejected change leaves
+     * the running guard exactly as it was — the same contract `updatePolicy` has. What
+     * is refused and why is documented on {@link Policy.replaceGuard}; the short version
+     * is that a terminal `fallbackAction` would turn every downgrade into the denial the
+     * downgrade exists to prevent.
+     *
+     * It is announced twice on purpose: a `warning`, which lands wherever your startup
+     * warnings land and in the dashboard's notices, and a `guard-change` event carrying
+     * both the before and the after.
+     */
+    updateGuard(settings: Partial<GuardSettings> & {
+        suspectThreshold?: number;
+    }, context?: ChangeContext): {
+        guard: GuardSettings & {
+            suspectThreshold: number;
+        };
+    };
+    /**
+     * Tells the other replicas that this actor has been proven a bot.
+     *
+     * Fire and forget, off the request path, and failure is silence: a store that is
+     * unavailable costs the *other* instances a fact they would have liked, and costs
+     * this request nothing at all. The window is the registry's own, so a shared count
+     * ages out at the same rate the local one does rather than accumulating for ever.
+     */
+    private publishConfirmation;
+    /**
+     * Asks the store what other replicas already know about an actor we have just met.
+     *
+     * Once per actor per instance, and deliberately *not* awaited: making it synchronous
+     * would put a network round trip on the request path, which is the one thing the
+     * behavioural state is arranged to avoid.
+     *
+     * So the request that triggers the read is assessed with whatever has arrived by the
+     * time the detectors finish — which for a fast store is usually the answer, and for a
+     * slow or distant one is nothing. Every request after it has the number. Neither
+     * outcome is wrong: an actor nobody here has seen before is an actor with no local
+     * history either, so the first request was always going to be judged on the request
+     * alone.
+     *
+     * The read subtracts nothing. This instance has not confirmed anything about a
+     * brand-new actor yet, so whatever the store holds was put there by somebody else,
+     * and the two counts add up cleanly from here.
+     */
+    private loadSharedConfirmations;
+    /** Names and sizes of every configured range set. */
+    listRanges(): Array<{
+        name: string;
+        size: number;
+    }>;
+    /** Every registered detector, for documentation and for checking what is actually installed. */
+    describeDetectors(): DetectorDescription[];
+    /** Recovers the client address from a socket address and headers, honouring the proxy config. */
+    resolveIp(socketAddress: string | undefined, headers: Record<string, string | undefined>): string;
+    actorKeyFor(facts: RequestFacts): string;
+    isAllowlisted(ip: string): boolean;
+    isIgnoredPath(path: string): boolean;
+    /**
+     * Gathers evidence and reaches a verdict. Never throws, never writes a response.
+     *
+     * Detectors run in three phases, ordered by what they cost:
+     *
+     * 1. **Cheap** — pure string and header inspection, sequential, microseconds.
+     * 2. **I/O** — concurrent, each under its own timeout, so one slow lookup cannot
+     *    hold the request open.
+     * 3. **Confirming** — only when the request actually claimed an identity worth
+     *    checking. A request that named no crawler resolves no DNS.
+     */
+    assess(facts: RequestFacts, options?: AssessOptions): Promise<Assessment>;
+    /** Applies the policy. Pure — no I/O, no side effects beyond the downgrade notification. */
+    decide(assessment: Assessment): Decision;
+    /** Assess, decide, and turn the decision into an outcome for an adapter to apply. */
+    handle(facts: RequestFacts): Promise<HandleResult>;
+    /** True when this request is the challenge verification endpoint. */
+    isChallengeEndpoint(facts: Pick<RequestFacts, "method" | "path">): boolean;
+    /**
+     * Verifies a submitted challenge solution.
+     *
+     * Deliberately its own entry point rather than something `handle` detects: the
+     * verification endpoint must not itself be assessed, or a client that has just been
+     * challenged would be challenged again for trying to answer.
+     */
+    verifyChallenge(facts: RequestFacts, body: unknown): Promise<SolutionOutcome>;
+    /**
+     * Mints a clearance cookie for an actor, bypassing the challenge.
+     *
+     * Call it the moment your application knows something the request cannot show — a
+     * completed login, a verified payment. `operator` clearance is the only conclusive
+     * human signal this library recognises.
+     */
+    grantClearance(facts: RequestFacts, level?: "pow" | "interaction" | "operator"): string | undefined;
+    /**
+     * Runs one detector.
+     *
+     * Returns `undefined` when the detector completed synchronously — the common case,
+     * and the one worth keeping off the microtask queue — or a promise the caller must
+     * await. Failures are absorbed here in both paths: a detector can throw, reject or
+     * hang, and none of those may reach the request.
+     */
+    private run;
+    private collect;
+    private recordFailure;
+    private bypassed;
+    /**
+     * Raises a warning through the handler's own channel.
+     *
+     * Public because things outside this class legitimately have something to say: an
+     * adapter that has spotted a misconfiguration, the crawler-range refresher reporting
+     * a publisher it could not reach. Both used to call `config.onWarning` directly,
+     * which reaches the callback and *not* the `warning` event — so a subscriber, and
+     * with it the dashboard's notices panel, never heard about either. One channel, one
+     * set of listeners.
+     */
+    warn(message: string): void;
+    private fail;
+}

package/dist/corpus/adversarial.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const ADVERSARIAL_CASES: TrafficCase[];

package/dist/corpus/advertising-email.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const ADVERTISING_EMAIL_CASES: TrafficCase[];

package/dist/corpus/ai-crawlers.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const AI_CRAWLER_CASES: TrafficCase[];

package/dist/corpus/benign-bots.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const BENIGN_BOT_CASES: TrafficCase[];

package/dist/corpus/cdn-gateways.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const CDN_GATEWAY_CASES: TrafficCase[];

package/dist/corpus/cookies.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Realistic cookie jars.
+ *
+ * A browser that has been to a commercial site once is carrying a dozen cookies it
+ * never asked for, and their *shape* is as characteristic as any header: Google
+ * Analytics' `_ga` encodes a client id and a first-seen timestamp, Meta's `_fbp`
+ * encodes a version, a subdomain index and a creation time, a TCF consent string is
+ * base64 with a version prefix. Automation carries none of this, or carries a single
+ * hand-set session cookie and nothing else.
+ *
+ * The values below are structurally correct and entirely synthetic. They exist so a
+ * fixture looks like a request from somebody who has actually used the web, rather
+ * than like `Cookie: sid=abc`.
+ */
+export interface CookieJarOptions {
+    /** Seed for the synthetic identifiers, so two visitors differ. */
+    visitor?: string;
+    /** Epoch seconds the visitor was first seen. */
+    firstSeen?: number;
+    /** Epoch seconds of this session's start. */
+    sessionStart?: number;
+    /** Include Google Analytics 4 cookies. Present on most of the commercial web. */
+    analytics?: boolean;
+    /** Include Meta's advertising cookies. */
+    advertising?: boolean;
+    /** Include a consent-management platform's cookies. */
+    consent?: boolean;
+    /** Include Cloudflare's bot-management cookies, present on a large slice of sites. */
+    cloudflare?: boolean;
+    /** Include product-analytics and support-widget cookies. */
+    productAnalytics?: boolean;
+    /** Application cookies: session, cart, CSRF. */
+    application?: boolean;
+    /** Extra pairs appended verbatim. */
+    extra?: readonly (readonly [string, string])[];
+}
+/**
+ * Builds a `Cookie` header value.
+ *
+ * Order matters a little: browsers send cookies sorted by path length descending and
+ * then by creation time, which in practice means the host-wide analytics cookies set
+ * on the first visit come before the application cookies set later. Reproducing that
+ * ordering is the kind of detail a hand-written fixture never has.
+ */
+export declare function cookieJar(options?: CookieJarOptions): string;
+/** A first-time visitor: no analytics history, only whatever this page set. */
+export declare function freshVisitorJar(visitor?: string): string;
+/** A returning customer of a commercial site — the fullest jar in ordinary use. */
+export declare function returningCustomerJar(visitor?: string): string;

package/dist/corpus/crawlers-regional.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const REGIONAL_CRAWLER_CASES: TrafficCase[];

package/dist/corpus/crawlers-vertical.d.ts

@@ -0,0 +1,2 @@
+import type { TrafficCase } from "./schema.js";
+export declare const VERTICAL_CRAWLER_CASES: TrafficCase[];