@osqd/bothandlerjs 0.5.0 → 0.7.0

package/dist/internal/async.d.ts

@@ -11,6 +11,3 @@
  * unhandled rejection.
  */
 export declare function withTimeout<T>(work: Promise<T>, ms: number, fallback: T): Promise<T>;
-/** `Promise.allSettled` semantics without allocating a settled-result wrapper per item. */
-export declare function settleAll(work: Iterable<Promise<void>>): Promise<void>;
-export declare function isPromise(value: unknown): value is Promise<unknown>;

package/dist/internal/ip.d.ts

@@ -13,6 +13,24 @@ export type IpBytes = Uint8Array;
  * address. Deliberately strict: no octal, no hex, no shorthand octets, no zone
  * ids. Anything ambiguous is rejected rather than guessed at.
  */
+/**
+ * An address with its source port removed, when it carried one.
+ *
+ * Forwarded headers are not consistent about this. Most proxies write a bare address,
+ * but Azure's Application Gateway and Front Door write `1.2.3.4:5678`, and the
+ * bracketed `[2001:db8::1]:5678` is the form RFC 7239 defines for IPv6. An entry that
+ * carries a port parses as nothing at all, and the consequence is not that one entry is
+ * skipped: every entry in the chain looks the same way, the chain empties, and the whole
+ * internet collapses onto the proxy's own address as a single actor. Rate limits, actor
+ * history and reputation then apply to everyone at once, so one bot locks out every real
+ * visitor — and none of it announces itself.
+ *
+ * The rule has to be narrow, because a bare IPv6 address is *made of* colons and must
+ * never be mistaken for a host and port. Only two shapes are a port: a bracketed host,
+ * which is unambiguous, and a single colon whose left side is an IPv4 address. Anything
+ * with more colons and no brackets is IPv6 and is returned untouched.
+ */
+export declare function stripPort(value: string): string;
 export declare function parseIp(value: string): IpBytes | null;
 /** Canonical string form, so the same host always produces the same actor key. */
 export declare function formatIp(bytes: IpBytes): string;

package/dist/internal/text.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Text that is safe to print.
+ *
+ * Everything here exists because this library quotes its input back: an evidence
+ * summary names the path that was requested and the header that was sent, and an actor
+ * label is whatever an operator typed. That text is then written to a log file, a
+ * terminal and a JSON feed — three readers that treat some characters as instructions
+ * rather than as letters.
+ */
+/**
+ * Makes one line of prose safe to print.
+ *
+ * Evidence summaries name what was seen, and naming it often means quoting the client:
+ * the path it asked for, the header it sent, the identity it claimed. That text is
+ * written by whoever sent the request, and it lands in three places that read control
+ * characters as instructions rather than as letters — a log file, where a carriage
+ * return and a newline let a client forge a second log line of its own composition; a
+ * terminal, where an escape sequence repaints or erases what is already on screen; and
+ * a JSON feed, where a lone surrogate is not encodable.
+ *
+ * So the characters that mean something other than themselves are replaced with U+FFFD
+ * and the text is capped. What a client says about itself is quoted, never obeyed.
+ *
+ * The scan allocates nothing and returns the original string when there is nothing to
+ * fix, which is every request that is not an attack: this runs on every piece of
+ * evidence on every request.
+ */
+export declare function safeSummary(text: string): string;

package/dist/metrics.d.ts

@@ -29,6 +29,22 @@ export interface MetricsSnapshot {
     detectorFirings: Record<string, number>;
     /** How often each detector threw or timed out. */
     detectorFailures: Record<string, number>;
+    /**
+     * How often each *shadowed* detector produced evidence, counted separately.
+     *
+     * Separately on purpose: a shadowed detector took no part in any verdict, so folding
+     * its firings into `detectorFirings` would put work into a chart of decisions that
+     * decided nothing. See {@link BotHandlerConfig.shadowDetectors}.
+     */
+    shadowFirings: Record<string, number>;
+    /**
+     * Assessments whose verdict the shadowed detectors would have changed.
+     *
+     * Keyed by the verdict they would have produced, so the number that matters is legible
+     * on its own: `suspected-bot` here, against a `human` count that did not move, is the
+     * shape of a detector about to start challenging people.
+     */
+    shadowChanges: Record<Verdict, number>;
     /**
      * Time spent inside each detector, when `metrics.perDetectorTiming` is on.
      *
@@ -100,6 +116,8 @@ export declare class Metrics {
     private readonly detectorFirings;
     private readonly detectorFailures;
     private readonly detectorTimings;
+    private readonly shadowFirings;
+    private readonly shadowChanges;
     private challengesIssued;
     private challengesSolved;
     private challengesRejected;

package/dist/probe/index.d.ts

@@ -0,0 +1,153 @@
+import type { IdentityShape, MarkerReading, ShapeDrift } from "./marker.js";
+import type { RequestFacts } from "../types.js";
+import type { ParsedUserAgent } from "../internal/ua.js";
+import type { Clock } from "../internal/clock.js";
+export type { IdentityShape, MarkerClaims, MarkerReading, ShapeDrift } from "./marker.js";
+export { identityShape, driftBetween, readMarker } from "./marker.js";
+/**
+ * Asking the client to hold something, and reading what comes back.
+ *
+ * This is the one part of the library that *acts* in order to detect, rather than
+ * reading what a request happened to carry. That difference is worth being explicit
+ * about, because it changes what the evidence is worth: everything else here is an
+ * inference about a client, while a marker is a controlled experiment on one. We choose
+ * the stimulus, we sign it, and the response is either the cookie we issued or it is
+ * not. There is very little room left for a coincidence to explain.
+ *
+ * It is **off by default**, for two reasons that have nothing to do with detection.
+ * A `Set-Cookie` on a response makes it uncacheable by most shared caches and CDNs, so
+ * switching this on without knowing that can quietly move a site's cache-hit ratio; the
+ * probe therefore issues a marker only when the client does not already hold a valid
+ * one, which for an ordinary visitor is the first request of a session and no other.
+ * And a cookie is a cookie: it is first-party, carries no identifier of a person and
+ * expires on its own, but the decision to set one belongs to the operator rather than
+ * to a library they installed to read headers.
+ */
+export interface MarkerProbeOptions {
+    /**
+     * HMAC secrets. The first signs; all of them verify.
+     *
+     * Required, and deliberately not defaulted to something generated at startup. A
+     * per-process secret would mean every marker minted by one replica reads as *forged*
+     * on every other one, and as forged again after a restart — turning the strongest
+     * signal here into a machine for accusing ordinary visitors.
+     */
+    secrets: readonly string[];
+    /** Cookie name. Default `__bh_m`. */
+    cookieName?: string;
+    /** How long a marker stands. Default 12 hours. */
+    ttlMs?: number;
+    sameSite?: "Strict" | "Lax" | "None";
+    /** Set the `Secure` attribute. Default true; turn it off only for local HTTP. */
+    secure?: boolean;
+    domain?: string;
+    /**
+     * How many markers to track network fan-out for. Default 20000; 0 turns it off.
+     *
+     * Bounded rather than shared, on purpose. The rest of this library keeps its
+     * behavioural series in process — see `state.ts` — because a store round trip on the
+     * request path buys precision nobody asked for at a cost everybody pays. Across
+     * replicas each one therefore sees its own share of a client's addresses, which
+     * *understates* fan-out and so errs towards saying nothing.
+     */
+    maxTrackedMarkers?: number;
+    /**
+     * Estimated networks at which the count saturates. Default 96.
+     *
+     * The networks are not remembered individually — see `noteNetwork`. Raising this does
+     * not cost memory; it only extends the range over which the estimate stays useful.
+     */
+    maxNetworksPerMarker?: number;
+    /**
+     * How many verified markers to remember, so a session is not re-verified per request.
+     * Default 5000; 0 turns the cache off.
+     */
+    maxVerifiedMarkers?: number;
+}
+/** What this request's marker turned out to be. */
+export interface MarkerObservation {
+    reading: MarkerReading;
+    /**
+     * How the identity claimed now differs from the identity claimed when the marker was
+     * issued. Present only for a marker that verified, because a drift measured against
+     * an unsigned claim measures nothing.
+     */
+    drift: ShapeDrift | undefined;
+    /** This request's shape, so the issuing path does not compute it twice. */
+    shape: IdentityShape;
+    /**
+     * Distinct networks this marker has now been presented from, counted in this process.
+     *
+     * `0` when the marker did not verify or tracking is off. Saturates at
+     * `maxNetworksPerMarker`, because the question it answers — "is one client moving
+     * across a pool of addresses" — is already answered long before the number is large.
+     */
+    networks: number;
+}
+export declare class MarkerProbe {
+    readonly cookieName: string;
+    private readonly secrets;
+    private readonly ttlMs;
+    private readonly cookieOptions;
+    private readonly clock;
+    /**
+     * Marker id to a 128-bit sketch of the networks it has been presented from.
+     *
+     * A `Set` of network strings is the obvious structure and measured at **55.6 MB** with
+     * both caps full — twenty thousand markers each seen from a few dozen networks — which
+     * is far too much to hand somebody for switching on a detector. The question being
+     * asked is only ever "has this marker come from more than about sixteen networks", and
+     * a bitmap answers that in sixteen bytes by linear counting: hash each network to a
+     * bit, then estimate the distinct count from how many bits are set.
+     *
+     * The estimate carries a few percent of error in **either** direction — measured, 16
+     * real networks read as 17 and 32 read as 33 — so the threshold it feeds is a soft
+     * boundary rather than a hard one. That is honest for this signal in particular, which
+     * cannot separate a proxy pool from a heavily mobile person at any resolution, and is
+     * why it is capped at `moderate` and never denies anybody by itself.
+     */
+    private readonly fanout;
+    private readonly maxNetworks;
+    /**
+     * Markers already verified, by the exact cookie value that verified.
+     *
+     * A browsing session sends one identical cookie on every request, and verifying it is
+     * an HMAC — which measured at roughly twenty microseconds, nearly doubling the cost of
+     * an assessment to re-establish a fact that had not changed. The cache is only ever
+     * populated with *successes*: caching failures would let anyone flood it with unique
+     * junk, and a failure is cheap to reach anyway.
+     *
+     * Expiry is still checked on every hit, so a cached marker stops being accepted at the
+     * moment it should. The key is the whole signed value, so a cache hit is only possible
+     * for a string that already carried a valid signature.
+     */
+    private readonly verified;
+    constructor(options: MarkerProbeOptions & {
+        clock: Clock;
+    });
+    /** Reads the marker this request carried, and measures it against the request. */
+    observe(facts: RequestFacts, ua: ParsedUserAgent): MarkerObservation;
+    /**
+     * Whether this response should carry a marker.
+     *
+     * Only when the client is not already holding a good one. An ordinary visitor is
+     * therefore issued a cookie once and then browses with uncached-by-`Set-Cookie`
+     * responses never again; a client that discards cookies is issued one every time,
+     * which is itself the observation `marker-persistence` is built on.
+     */
+    shouldIssue(observation: MarkerObservation): boolean;
+    /** Verifies a presented marker, reusing an earlier verification of the same value. */
+    private read;
+    /**
+     * Files this presentation under the marker's own id and returns how many distinct
+     * networks it has now come from.
+     *
+     * A `/24` rather than an address, because a single visitor's address changes for
+     * ordinary reasons all day — a phone moving between cells, a router relearning a
+     * lease — while the network it sits behind usually does not. Counting addresses would
+     * report every commuter.
+     */
+    private noteNetwork;
+    /** The `Set-Cookie` handing this client a marker bound to the identity it just claimed. */
+    issue(observation: MarkerObservation): string;
+}

package/dist/probe/marker.d.ts

@@ -0,0 +1,119 @@
+import type { RequestFacts } from "../types.js";
+import type { ParsedUserAgent } from "../internal/ua.js";
+/**
+ * A marker the client carries, so its requests can be read as a series.
+ *
+ * Everything else in this library correlates requests by **actor key**, which is
+ * derived from the address. That is the only join available without asking the client
+ * to hold anything, and it is wrong in both directions: a household, an office and a
+ * mobile carrier put thousands of unrelated people behind one key, while a single
+ * scraper on a proxy pool spreads one operator across thousands of keys. The
+ * consequence was a detector this library could not write — `identity-rotation`, the
+ * client that arrives as Chrome, then as curl, then as Googlebot. From headers alone
+ * it is indistinguishable from three people sharing an address, so it was left unbuilt.
+ *
+ * A signed cookie closes exactly that gap. It is minted by this server, carries an
+ * HMAC only this server can produce, and comes back only from the client that received
+ * it. Two requests bearing the same marker are the same browser profile — not the same
+ * address, not the same network, the same *client* — and that is what makes a change of
+ * claimed identity between them evidence rather than speculation.
+ *
+ * **What is in it, and what is deliberately not.** Tokens here are signed and never
+ * encrypted, so the client can read every claim and nothing secret may go in one. The
+ * marker holds a random id, the usual validity window, and three short hashes standing
+ * for the identity the client claimed when it was issued. The hashes are of data the
+ * client sent us in the first place, so they tell it nothing it did not already know,
+ * and hashing them keeps the cookie from being a readable fingerprint echoed back on
+ * every response.
+ *
+ * **Why the identity is stored coarsely.** A browser that updates from version 130 to
+ * 131 has not changed identity, and a detector that says otherwise would report every
+ * visitor in the week after a Chrome release. So the version is excluded and the three
+ * parts are kept apart rather than hashed together, because *which* part changed is the
+ * difference between a strong signal and a benign one: a browser family that changes
+ * from Chrome to curl has no innocent reading, while a platform that changes from
+ * iPhone to Mac is what "Request desktop site" does to a real person's phone.
+ */
+/** Claims inside a marker. `sub` is the marker's own id, not an actor: see below. */
+export interface MarkerClaims {
+    v: 1;
+    /**
+     * The marker's own random id.
+     *
+     * Deliberately *not* the actor. Binding a marker to an address-derived key would
+     * invalidate it the moment a phone moved between wifi and cellular, which is the
+     * ordinary behaviour of the visitors this is supposed to leave alone — and it would
+     * throw away the property that makes the marker worth having, that it identifies a
+     * client across exactly those changes.
+     */
+    sub: string;
+    iat: number;
+    exp: number;
+    /** Browser family, coarse. */
+    b: string;
+    /** Operating system or platform, coarse. */
+    o: string;
+    /** Primary language subtag. */
+    l: string;
+}
+/** How the claimed identity looked, reduced to the parts worth comparing. */
+export interface IdentityShape {
+    b: string;
+    o: string;
+    l: string;
+}
+/** Which parts of a claimed identity differ between two requests. */
+export interface ShapeDrift {
+    browser: boolean;
+    platform: boolean;
+    language: boolean;
+}
+/**
+ * The claimed identity, reduced to three coarse parts.
+ *
+ * Stored as short plain text rather than hashed, which is both cheaper and more honest.
+ * An earlier version ran each part through an HMAC so the cookie would not carry a
+ * "readable fingerprint" — but there is no privacy in hashing a value the client wrote
+ * itself and sent to us, in a token the client can already read. It bought nothing and
+ * cost three HMACs on every single request, which more than doubled the price of an
+ * assessment. Now it costs a `slice`.
+ *
+ * The version is deliberately absent. A browser updating from 130 to 131 has not changed
+ * identity, and a detector that says otherwise reports every visitor in the week after a
+ * Chrome release.
+ */
+export declare function identityShape(facts: RequestFacts, ua: ParsedUserAgent): IdentityShape;
+/** Which parts changed. All three false means the client looks the same as it did. */
+export declare function driftBetween(issued: IdentityShape, now: IdentityShape): ShapeDrift;
+export declare function newMarker(shape: IdentityShape, ttlMs: number, now: number): MarkerClaims;
+/** The `Set-Cookie` that hands a client its marker. */
+export declare function markerCookie(name: string, claims: MarkerClaims, secrets: readonly string[], options: MarkerCookieOptions): string;
+export interface MarkerCookieOptions {
+    sameSite?: "Strict" | "Lax" | "None";
+    secure?: boolean;
+    domain?: string;
+}
+export type MarkerReading = 
+/** No marker was presented. */
+{
+    kind: "absent";
+}
+/** Presented, signed by us, still valid. */
+ | {
+    kind: "valid";
+    claims: MarkerClaims;
+}
+/** Presented and past its expiry — ordinary, and not evidence of anything. */
+ | {
+    kind: "expired";
+}
+/**
+ * Presented, and not something this server signed.
+ *
+ * The interesting case. A browser does not edit its own cookies, so a marker that
+ * fails its signature was changed by whoever is holding it.
+ */
+ | {
+    kind: "forged";
+};
+export declare function readMarker(value: string | undefined, secrets: readonly string[], now: number): MarkerReading;

package/dist/site/index.d.ts

@@ -0,0 +1,122 @@
+import type { Clock } from "../internal/clock.js";
+/**
+ * What this site normally looks like.
+ *
+ * Everything else in this library judges a client against a fixed idea of what clients
+ * do. This judges one against *the rest of the traffic*, which answers questions the
+ * fixed rules cannot. A path nobody else has ever asked for is a wordlist entry without
+ * needing to be in anybody's wordlist. A client whose requests are almost all misses
+ * matters only relative to a site's own miss rate, which on a site mid-migration is
+ * enormous and innocent. And an enumeration split across five hundred addresses is
+ * invisible to every per-actor threshold by construction, but perfectly visible in the
+ * union of what those addresses asked for.
+ *
+ * **It is off by default and it stays quiet for a long time after being switched on.**
+ * Both of those are load-bearing. A baseline is a claim about normal, and a claim about
+ * normal drawn from four hundred requests is not one — a quiet site at three in the
+ * morning would otherwise produce "this path is unique to this client" for every path,
+ * because every path is. So nothing is reported until `warmupRequests` have been seen,
+ * and the failure mode of the whole module is silence.
+ *
+ * **It is kept in process, like the rest of the behavioural state.** Across replicas
+ * each one therefore sees its own share of the traffic. That understates every count
+ * here, and understating is the direction that costs a missed detection rather than an
+ * accusation — which is the correct way round for a module whose mistakes would
+ * otherwise land on everybody at once.
+ */
+export interface SiteProfileOptions {
+    /**
+     * Requests to observe before anything may be reported. Default 5000.
+     *
+     * The single most important number here. Below it every question this module answers
+     * has the same wrong answer — everything looks rare, because everything is.
+     */
+    warmupRequests?: number;
+    /** Distinct paths remembered. Default 50000. */
+    maxPaths?: number;
+    /** Numeric path shapes tracked for distributed walks. Default 256. */
+    maxTemplates?: number;
+    /** Actors remembered per template before the count saturates. Default 64. */
+    maxActorsPerTemplate?: number;
+    /** How long an observation counts for. Default one hour. */
+    windowMs?: number;
+    /** Newly-appeared paths watched for a surge. Default 2048; 0 turns it off. */
+    maxWatchedPaths?: number;
+    /** Clients remembered per watched path before the count saturates. Default 64. */
+    maxActorsPerPath?: number;
+}
+/** What has happened to a newly-appeared path since it appeared. */
+export interface PathSurge {
+    clients: number;
+    ageMs: number;
+    answered: number;
+    misses: number;
+}
+/** What the site knows about one numeric path shape right now. */
+export interface WalkSpread {
+    actors: number;
+    /**
+     * Distinct ids requested, estimated.
+     *
+     * Exact while the bitmap is at full resolution, and `buckets * scale` once it has
+     * coarsened — which is the right estimate for the thing this exists to find, because a
+     * walk that covers its range touches every id in every bucket it touches. It
+     * *overestimates* for sparse traffic at a coarse scale, so it is never used alone: the
+     * detector requires coverage and the revisit ratio to agree.
+     */
+    ids: number;
+    /** Buckets of the range that were touched. Exact distinct count while `scale` is 1. */
+    buckets: number;
+    /** Ids per bucket. 1 until the range outgrows the bitmap. */
+    scale: number;
+    visits: number;
+    /**
+     * Fraction of the range between the lowest and highest id that was touched.
+     *
+     * Always a number, and never zero: the range is measured from the lowest id seen to the
+     * highest, so it is at least one bucket wide and that bucket was touched by definition.
+     * A single id is a range of one, fully covered.
+     */
+    coverage: number;
+}
+export declare class SiteProfile {
+    private readonly options;
+    private readonly paths;
+    private readonly walks;
+    private readonly watched;
+    private readonly clock;
+    private observed;
+    private misses;
+    private answered;
+    constructor(options: SiteProfileOptions & {
+        clock: Clock;
+    });
+    /**
+     * Whether enough traffic has been seen for any of this to mean anything.
+     *
+     * Every reader checks this. A profile that answers during warmup is worse than one
+     * that does not exist, because it answers confidently and wrongly.
+     */
+    get warm(): boolean;
+    get requestsObserved(): number;
+    /** The share of answered requests that were misses, or `undefined` before warmup. */
+    get missRate(): number | undefined;
+    /** Files a request. Called once per assessed request, before the detectors run. */
+    record(path: string, actorKey: string): void;
+    /** Files what the application answered, for the site's miss rate and each watched path. */
+    recordOutcome(path: string, status: number): void;
+    /** What has happened to a path since it first appeared. `undefined` if not watched. */
+    surgeOf(path: string): PathSurge | undefined;
+    /**
+     * How many times the site has served this path, to anybody.
+     *
+     * `undefined` before warmup, and `0` for a path this process has not seen — which is
+     * not the same as one the site does not have, and is why the detector reading this
+     * needs a great many of them before it says anything.
+     */
+    timesSeen(path: string): number | undefined;
+    /** Files one step of a numeric walk against the shape it belongs to. */
+    recordWalk(template: string, id: number, actorKey: string): void;
+    /** What the whole site has done with one numeric shape. `undefined` before warmup. */
+    spreadOf(template: string): WalkSpread | undefined;
+}