@osqd/bothandlerjs 0.5.0 → 0.6.0

package/dist/index.d.ts

@@ -26,8 +26,8 @@ export type { BotHandlerConfig, ResolvedConfig, ProxyConfig } from "./config.js"
 export { combineEvidence, noisyOr, sortEvidence, weightOf } from "./evidence.js";
 export type { CombineOptions, CombinedEvidence } from "./evidence.js";
 export { generateRobotsTxt, robotsFromRules, agentFor } from "./robots.js";
-export { fetchCrawlerRanges, refreshCrawlerRanges, startCrawlerRangeRefresh, PUBLISHED_CRAWLER_RANGES } from "./crawler-ranges.js";
-export type { PublishedRangeSource, RefreshOptions, RefreshResult, ScheduleOptions } from "./crawler-ranges.js";
+export { PUBLISHED_CRAWLER_RANGES, fetchAddressList, fetchCrawlerRanges, refreshCrawlerRanges, startCrawlerRangeRefresh } from "./crawler-ranges.js";
+export type { AddressListOptions, AddressListSource, PublishedRangeSource, RefreshOptions, RefreshResult, ScheduleOptions } from "./crawler-ranges.js";
 export type { RobotsOptions, RobotsFromRulesResult } from "./robots.js";
 export { TrafficAudit, DEFAULT_CHECKS } from "./audit.js";
 export type { AnomalySeverity, AuditCheck, AuditContext, AuditOptions, AuditWindow, TrafficAnomaly } from "./audit.js";

package/dist/index.js

@@ -122,16 +122,20 @@ function hashString(value) {
   }
   return hash >>> 0;
 }
-var TIMESTAMP_RING, PATH_CAP, UA_CAP, MAX_TRACKED_ARRIVALS, MAX_TRACKED_PATHS, MAX_TRACKED_USER_AGENTS, ActorState, ActorRegistry;
+var TIMESTAMP_RING, PATH_CAP, QUERY_CAP, METHOD_CAP, WALK_CAP, UA_CAP, MAX_TRACKED_ARRIVALS, MAX_TRACKED_PATHS, MAX_TRACKED_QUERIES, MAX_TRACKED_USER_AGENTS, ActorState, ActorRegistry;
 var init_state = __esm({
   "src/state.ts"() {
     "use strict";
     init_lru();
     TIMESTAMP_RING = 32;
     PATH_CAP = 64;
+    QUERY_CAP = 64;
+    METHOD_CAP = 12;
+    WALK_CAP = 4;
     UA_CAP = 4;
     MAX_TRACKED_ARRIVALS = TIMESTAMP_RING;
     MAX_TRACKED_PATHS = PATH_CAP;
+    MAX_TRACKED_QUERIES = QUERY_CAP;
     MAX_TRACKED_USER_AGENTS = UA_CAP;
     ActorState = class {
       key;
@@ -173,6 +177,45 @@ var init_state = __esm({
       paths = /* @__PURE__ */ new Set();
       pathsOverflowed = false;
       pathsSaturatedAtTotal = 0;
+      /**
+       * Distinct *parameterised* requests: the path together with its query.
+       *
+       * Counted apart from `paths` because the two answer different questions and a scraper
+       * lives in the gap between them. `/products?page=1` through `?page=200` is one path and
+       * two hundred requests, so breadth reads it as somebody rereading a single page — which
+       * is exactly what enumerating a catalogue looks like from the path alone.
+       */
+      queries = /* @__PURE__ */ new Set();
+      queriesOverflowed = false;
+      /**
+       * Which HTTP methods this actor has used.
+       *
+       * A browser navigating issues GET. Something that has issued nothing but HEAD across a
+       * long visit is checking what exists rather than reading it, and that is a fact about
+       * the actor rather than about any one of its requests — which is why it is kept here.
+       */
+      methods = /* @__PURE__ */ new Set();
+      /**
+       * Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
+       *
+       * Three numbers per shape, deliberately — a count, a lowest and a highest — rather than
+       * the ids themselves. What separates enumeration from reading is not which ids were
+       * asked for but whether they *cover a range*: thirty requests spanning thirty
+       * consecutive ids is a walk, and thirty scattered across a hundred thousand is somebody
+       * following links. Both are answerable from a count and a span, and only the count and
+       * the span survive an actor asking for ten thousand of them.
+       */
+      walks = /* @__PURE__ */ new Map();
+      /**
+       * What the application answered, for the requests anybody bothered to tell us about.
+       *
+       * The engine decides *before* the response exists, so this arrives afterwards and only
+       * when the adapter reports it. Kept as two counters rather than a list because the one
+       * question worth asking is a ratio: an actor whose requests are almost all misses is
+       * looking for something rather than reading anything.
+       */
+      responsesSeen = 0;
+      missesSeen = 0;
       userAgents = /* @__PURE__ */ new Set();
       constructor(key, now) {
         this.key = key;
@@ -192,6 +235,15 @@ var init_state = __esm({
           this.pathsOverflowed = true;
           this.pathsSaturatedAtTotal = this.total;
         }
+        const keys = Object.keys(facts.query).sort();
+        if (keys.length > 0) {
+          const signature = `${facts.path}?${keys.map((key) => `${key}=${facts.query[key] ?? ""}`).join("&")}`;
+          const queryHash = hashString(signature);
+          if (this.queries.size < QUERY_CAP) this.queries.add(queryHash);
+          else if (!this.queries.has(queryHash)) this.queriesOverflowed = true;
+        }
+        if (this.methods.size < METHOD_CAP) this.methods.add(facts.method);
+        this.noteWalk(facts.path);
         const ua = facts.headers["user-agent"];
         if (ua !== void 0 && this.userAgents.size < UA_CAP) this.userAgents.add(ua);
       }
@@ -202,6 +254,79 @@ var init_state = __esm({
       get pathsSaturated() {
         return this.pathsOverflowed;
       }
+      /** Distinct path-and-query combinations seen. Saturates at {@link QUERY_CAP}. */
+      get distinctQueries() {
+        return this.queries.size;
+      }
+      get queriesSaturated() {
+        return this.queriesOverflowed;
+      }
+      /**
+       * Records what the application answered. Called after the response, if at all.
+       *
+       * 404 and 410 only. A 403 is usually this library's own doing and counting it would
+       * make the detector that reads this argue with itself; a 500 is the site's problem and
+       * says nothing about the client.
+       */
+      recordOutcome(status) {
+        this.responsesSeen++;
+        if (status === 404 || status === 410) this.missesSeen++;
+      }
+      /** Responses reported for this actor. Zero unless something is reporting them. */
+      get responses() {
+        return this.responsesSeen;
+      }
+      /** Of those, how many were 404 or 410. */
+      get misses() {
+        return this.missesSeen;
+      }
+      /**
+       * Files a request under the shape of its path, if that path carries a number.
+       *
+       * The last numeric segment is the one taken to be the identifier: in `/api/v2/orders/42`
+       * the version is part of the shape and the order id is what is being walked.
+       */
+      noteWalk(path) {
+        const segments = path.split("/");
+        let value;
+        let template = "";
+        for (const segment of segments) {
+          if (segment !== "" && /^\d+$/.test(segment)) {
+            const parsed = Number(segment);
+            if (Number.isSafeInteger(parsed) && parsed <= 1e7) value = parsed;
+            template += "/#";
+          } else if (segment !== "") {
+            template += `/${segment}`;
+          }
+        }
+        if (value === void 0) return;
+        const existing = this.walks.get(template);
+        if (existing !== void 0) {
+          existing.count++;
+          if (value < existing.min) existing.min = value;
+          if (value > existing.max) existing.max = value;
+          return;
+        }
+        if (this.walks.size < WALK_CAP) this.walks.set(template, { count: 1, min: value, max: value });
+      }
+      /**
+       * The path shape this actor has walked hardest, with how far it reached.
+       *
+       * `span` is inclusive of both ends, so a walk of 1 to 30 spans 30. Comparing the count
+       * against it is what separates covering a range from visiting a few points in one.
+       */
+      densestWalk() {
+        let best;
+        for (const [template, walk] of this.walks) {
+          const span = walk.max - walk.min + 1;
+          if (best === void 0 || walk.count > best.count) best = { template, count: walk.count, span };
+        }
+        return best;
+      }
+      /** Every HTTP method this actor has used, in first-seen order. */
+      get methodsSeen() {
+        return [...this.methods];
+      }
       /**
        * Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
        * still can. Over that many requests the distinct count is exact, so it is the only
@@ -290,6 +415,12 @@ var init_state = __esm({
           key: this.key,
           requests: this.total,
           distinctPaths: this.distinctPaths,
+          distinctQueries: this.distinctQueries,
+          methodsSeen: this.methodsSeen,
+          walk: this.densestWalk(),
+          responses: this.responses,
+          misses: this.misses,
+          queriesSaturated: this.queriesSaturated,
           firstSeen: this.firstSeen,
           lastSeen: this.lastSeen,
           sinceLastMs: this.sinceLast(now),
@@ -1883,7 +2014,7 @@ function compileSignatures(signatures = BOT_SIGNATURES) {
 function indexSignatures(signatures = BOT_SIGNATURES) {
   return new Map(signatures.map((signature) => [signature.id, signature]));
 }
-var BOT_CATEGORIES, SEARCH, AI, SEO, SOCIAL, MONITORING, ARCHIVE, FEED, SECURITY, LIBRARY, HEADLESS, EMBEDDED, ADVERTISING, COMMERCE, ACADEMIC, ACCESSIBILITY, BOT_SIGNATURES, BENIGN_CATEGORIES;
+var BOT_CATEGORIES, SEARCH, AI, SEO, SOCIAL, MONITORING, ARCHIVE, FEED, SECURITY, LIBRARY, HEADLESS, EMBEDDED, ADVERTISING, COMMERCE, ACADEMIC, EMAIL_SECURITY, ACCESSIBILITY, BOT_SIGNATURES, BENIGN_CATEGORIES;
 var init_known_bots = __esm({
   "src/detectors/known-bots.ts"() {
     "use strict";
@@ -1904,6 +2035,7 @@ var init_known_bots = __esm({
       "commerce",
       "accessibility",
       "academic",
+      "email-security",
       "other"
     ];
     SEARCH = [
@@ -1928,6 +2060,13 @@ var init_known_bots = __esm({
       { id: "yisouspider", name: "Shenma (Yisou) Spider", tokens: ["yisouspider"], category: "search", benign: true, robotsAgent: "YisouSpider", verification: { kind: "none" } },
       { id: "yahoo-slurp", name: "Yahoo! Slurp", tokens: ["yahoo! slurp"], category: "search", benign: true, robotsAgent: "Slurp", verification: { kind: "fcrdns", domains: ["crawl.yahoo.net", "yahoo.com"] } },
       { id: "mail-ru", name: "Mail.Ru bot", tokens: ["mail.ru_bot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "brave-search", name: "Brave Search", tokens: ["bravesearchbot"], category: "search", benign: true, robotsAgent: "BraveSearchBot", verification: { kind: "none" } },
+      { id: "ecosia", name: "Ecosia", tokens: ["ecosiabot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "startpage", name: "Startpage", tokens: ["startpagebot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "daum", name: "Daumoa", tokens: ["daumoa"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "stract", name: "Stract", tokens: ["stractbot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "rightdao", name: "RightDao", tokens: ["rightdaobot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "gigablast", name: "Gigablast", tokens: ["gigablastopensource"], category: "search", benign: true, verification: { kind: "none" } },
       { id: "exabot", name: "Exabot (Exalead)", tokens: ["exabot"], category: "search", benign: true, verification: { kind: "none" } },
       { id: "kagibot", name: "Kagi", tokens: ["kagibot"], category: "search", benign: true, robotsAgent: "KagiBot", verification: { kind: "none" } }
     ];
@@ -1998,11 +2137,15 @@ var init_known_bots = __esm({
       { id: "pinterestbot", name: "Pinterestbot", tokens: ["pinterest/", "pinterestbot"], category: "social", benign: true, verification: { kind: "fcrdns", domains: ["pinterest.com"] } },
       { id: "mastodon", name: "Mastodon / Fediverse", tokens: ["mastodon/", "pleroma", "misskey/", "akkoma"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "embedly", name: "Embedly", tokens: ["embedly"], category: "social", benign: true, verification: { kind: "none" } },
-      { id: "bluesky", name: "Bluesky card fetcher", tokens: ["bluesky cardyb", "cardyb/"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "bluesky", name: "Bluesky card fetcher", tokens: ["bluesky cardyb", "cardyb/", "blueskybot"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "iframely", name: "Iframely", tokens: ["iframely"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "skype-preview", name: "Skype URI preview", tokens: ["skypeuripreview"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "vk-share", name: "VK / Odnoklassniki preview", tokens: ["vkshare", "odklbot"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "discourse-onebox", name: "Discourse Onebox", tokens: ["discourse forum onebox"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "microsoft-preview", name: "Microsoft Teams preview", tokens: ["microsoftpreview"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "zoom-preview", name: "Zoom link preview", tokens: ["zoombot"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "signal-preview", name: "Signal link preview", tokens: ["signalbot"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "matrix-synapse", name: "Matrix (Synapse) preview", tokens: ["synapse/"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "yahoo-preview", name: "Yahoo Link Preview", tokens: ["yahoo link preview"], category: "social", benign: true, verification: { kind: "none" } }
     ];
     MONITORING = [
@@ -2042,6 +2185,10 @@ var init_known_bots = __esm({
       { id: "smartnews", name: "SmartNews", tokens: ["smartnewsbot"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "flipboard", name: "Flipboard", tokens: ["flipboardproxy"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "podcast-index", name: "Podcast Index", tokens: ["podcastindexbot"], category: "feed", benign: true, verification: { kind: "none" } },
+      // Spotify's podcast fetcher sends `Spotify/1.0` — and so does the Spotify desktop app,
+      // with a person driving it. There is no token that separates them, so this one is left
+      // unnamed rather than named wrongly: the corpus proved the point immediately by blocking
+      // a human under `protect-auth`, `indexers-only` and `under-attack` at once.
       { id: "freshrss", name: "FreshRSS", tokens: ["freshrss"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "netnewswire", name: "NetNewsWire", tokens: ["netnewswire"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "overcast", name: "Overcast", tokens: ["overcast/"], category: "feed", benign: true, verification: { kind: "none" } },
@@ -2168,7 +2315,23 @@ var init_known_bots = __esm({
     ];
     ADVERTISING = [
       { id: "adsbot-google", name: "AdsBot-Google", tokens: ["adsbot-google", "mediapartners-google", "adsbot"], category: "advertising", benign: true, verification: { kind: "fcrdns", domains: ["googlebot.com", "google.com"] } },
-      { id: "criteo", name: "Criteo", tokens: ["criteobot"], category: "advertising", benign: true, verification: { kind: "none" } }
+      { id: "criteo", name: "Criteo", tokens: ["criteobot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      // Verification and contextual classification: they read a page to decide whether an ad
+      // may appear beside it, or what the page is about. A publisher usually wants these and a
+      // site with no advertising has no reason to.
+      { id: "doubleverify", name: "DoubleVerify", tokens: ["doubleverifybot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "ias", name: "Integral Ad Science", tokens: ["ias crawler"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "moat", name: "Moat", tokens: ["moatbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "comscore", name: "comScore (Proximic)", tokens: ["proximic"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "grapeshot", name: "Grapeshot", tokens: ["grapeshotcrawler"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "peer39", name: "Peer39", tokens: ["peer39bot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "taboola", name: "Taboola", tokens: ["taboolabot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "outbrain", name: "Outbrain", tokens: ["outbrainbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "pubmatic", name: "PubMatic", tokens: ["pubmaticbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "thetradedesk", name: "The Trade Desk", tokens: ["ttd-content"], category: "advertising", benign: true, verification: { kind: "none" } },
+      // Competitive ad intelligence rather than verification: it collects what everyone else
+      // is running. Named, and left for the operator to decide about.
+      { id: "adbeat", name: "Adbeat", tokens: ["adbeat_bot"], category: "advertising", benign: false, verification: { kind: "none" } }
     ];
     COMMERCE = [
       { id: "idealo", name: "idealo", tokens: ["idealo-bot"], category: "commerce", benign: false, verification: { kind: "none" } },
@@ -2184,6 +2347,12 @@ var init_known_bots = __esm({
       { id: "openalex", name: "OpenAlex", tokens: ["openalexbot"], category: "academic", benign: true, verification: { kind: "none" } },
       { id: "webis", name: "Webis research crawler", tokens: ["webisbot"], category: "academic", benign: true, verification: { kind: "none" } }
     ];
+    EMAIL_SECURITY = [
+      { id: "proofpoint", name: "Proofpoint URL Defense", tokens: ["proofpointurldefensebot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "mimecast", name: "Mimecast URL Protect", tokens: ["mimecasturlprotectbot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "barracuda", name: "Barracuda Link Protect", tokens: ["barracudalinkprotectbot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "cisco-esa", name: "Cisco Secure Email", tokens: ["ciscosecureemailbot"], category: "email-security", benign: true, verification: { kind: "none" } }
+    ];
     ACCESSIBILITY = [
       { id: "siteimprove", name: "Siteimprove", tokens: ["siteimprovebot"], category: "accessibility", benign: true, robotsAgent: "SiteimproveBot", verification: { kind: "none" } }
     ];
@@ -2202,7 +2371,8 @@ var init_known_bots = __esm({
       ...EMBEDDED,
       ...COMMERCE,
       ...ACADEMIC,
-      ...ACCESSIBILITY
+      ...ACCESSIBILITY,
+      ...EMAIL_SECURITY
     ]);
     BENIGN_CATEGORIES = /* @__PURE__ */ new Set(["search", "social", "monitoring", "feed", "archive", "advertising"]);
   }
@@ -2786,6 +2956,166 @@ var init_crawl_breadth = __esm({
   }
 });
 
+// src/detectors/parameter-sweep.ts
+function parameterSweepDetector(options = {}) {
+  const threshold = options.threshold ?? 25;
+  const variantsPerPath = options.variantsPerPath ?? 8;
+  const minRequests = options.minRequests ?? 20;
+  if (threshold > MAX_TRACKED_QUERIES) {
+    throw new RangeError(
+      `parameterSweepDetector threshold ${threshold} can never be reached: an actor's distinct-query count saturates at ${MAX_TRACKED_QUERIES}. Use ${MAX_TRACKED_QUERIES} or fewer.`
+    );
+  }
+  return {
+    id: "parameter-sweep",
+    description: "Counts distinct query strings against the paths they sit on, to catch enumeration that leaves the path unchanged",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const { distinctPaths, distinctQueries, queriesSaturated, total } = ctx.state;
+      if (total < minRequests || distinctQueries < threshold) return void 0;
+      const spread = distinctQueries / Math.max(1, distinctPaths);
+      if (spread < variantsPerPath) return void 0;
+      return {
+        detector: "parameter-sweep",
+        summary: queriesSaturated ? `at least ${distinctQueries} distinct query strings across only ${distinctPaths} path(s)` : `${distinctQueries} distinct query strings across only ${distinctPaths} path(s) in ${total} requests`,
+        direction: "bot",
+        certainty: "weak",
+        // Saturation means the count stopped being able to grow, so the real spread is
+        // wider than the one reported — the same argument breadth makes for itself.
+        weight: queriesSaturated ? 0.25 : 0.15,
+        botClass: "scraper",
+        metadata: {
+          distinctQueries,
+          distinctPaths,
+          variantsPerPath: Number(spread.toFixed(1)),
+          totalRequests: total,
+          saturated: queriesSaturated
+        }
+      };
+    }
+  };
+}
+var init_parameter_sweep = __esm({
+  "src/detectors/parameter-sweep.ts"() {
+    "use strict";
+    init_state();
+  }
+});
+
+// src/detectors/transport-coherence.ts
+function transportCoherenceDetector(options = {}) {
+  const checkLegacyHttp = options.legacyHttp ?? true;
+  const minHeadRequests = options.minHeadRequests ?? 8;
+  return {
+    id: "transport-coherence",
+    description: "Reads the HTTP version and the methods across a visit against the client the request claims to be",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      if (!claimsBrowser(ctx.ua)) return void 0;
+      const results = [];
+      const version = ctx.facts.httpVersion;
+      if (checkLegacyHttp && version !== void 0 && LEGACY_VERSIONS.has(version)) {
+        results.push({
+          detector: "transport-coherence",
+          summary: `Client claims to be a browser but negotiated HTTP/${version}, which no shipping browser has offered in over a decade`,
+          direction: "bot",
+          certainty: "moderate",
+          botClass: "impersonator",
+          // One downgrading proxy in front of the application does this to every request
+          // that passes through it, so this must count once rather than once per reason.
+          family: "legacy-transport",
+          metadata: { httpVersion: version, browser: ctx.ua.browser }
+        });
+      }
+      const methods = ctx.state.methodsSeen;
+      if (ctx.state.total >= minHeadRequests && methods.length > 0 && methods.every((method) => method === "HEAD")) {
+        results.push({
+          detector: "transport-coherence",
+          summary: `Client claims to be a browser but has issued nothing but HEAD across ${ctx.state.total} requests`,
+          direction: "bot",
+          certainty: "moderate",
+          botClass: "scraper",
+          metadata: { requests: ctx.state.total, browser: ctx.ua.browser }
+        });
+      }
+      return results.length > 0 ? results : void 0;
+    }
+  };
+}
+var LEGACY_VERSIONS;
+var init_transport_coherence = __esm({
+  "src/detectors/transport-coherence.ts"() {
+    "use strict";
+    init_ua();
+    LEGACY_VERSIONS = /* @__PURE__ */ new Set(["0.9", "1.0"]);
+  }
+});
+
+// src/detectors/probe-volume.ts
+function probeVolumeDetector(options = {}) {
+  const minResponses = options.minResponses ?? 20;
+  const missRatio = options.missRatio ?? 0.8;
+  return {
+    id: "probe-volume",
+    description: "Reads the share of an actor's requests that were answered 404 or 410, where the application reports them",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const { responses, misses } = ctx.state;
+      if (responses < minResponses) return void 0;
+      const ratio = misses / responses;
+      if (ratio < missRatio) return void 0;
+      return {
+        detector: "probe-volume",
+        summary: `${misses} of this client's last ${responses} requests were answered "not found" (${(ratio * 100).toFixed(0)}%)`,
+        direction: "bot",
+        certainty: "moderate",
+        botClass: "scanner",
+        metadata: { responses, misses, missRatio: Number(ratio.toFixed(3)) }
+      };
+    }
+  };
+}
+var init_probe_volume = __esm({
+  "src/detectors/probe-volume.ts"() {
+    "use strict";
+  }
+});
+
+// src/detectors/id-enumeration.ts
+function idEnumerationDetector(options = {}) {
+  const minRequests = options.minRequests ?? 30;
+  const density = options.density ?? 0.9;
+  return {
+    id: "id-enumeration",
+    description: "Reports an actor covering a contiguous range of numeric identifiers under one path shape",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const walk = ctx.state.densestWalk();
+      if (walk === void 0 || walk.count < minRequests) return void 0;
+      if (walk.span < minRequests) return void 0;
+      const covered = Math.min(1, walk.count / walk.span);
+      if (covered < density) return void 0;
+      return {
+        detector: "id-enumeration",
+        summary: `${walk.count} requests to ${walk.template} covering ${(covered * 100).toFixed(0)}% of a ${walk.span}-wide range of ids`,
+        direction: "bot",
+        certainty: "moderate",
+        botClass: "scraper",
+        metadata: { template: walk.template, requests: walk.count, span: walk.span, coverage: Number(covered.toFixed(3)) }
+      };
+    }
+  };
+}
+var init_id_enumeration = __esm({
+  "src/detectors/id-enumeration.ts"() {
+    "use strict";
+  }
+});
+
 // src/detectors/crawler-verification.ts
 function crawlerVerificationDetector(options = {}) {
   const missingPtrIsForgery = options.treatMissingPtrAsForgery ?? true;
@@ -4163,11 +4493,15 @@ function defaultDetectors(options = {}) {
     clientHintsDetector(),
     fetchMetadataDetector(),
     acceptSignatureDetector(),
+    transportCoherenceDetector(),
     headerOrderDetector(),
     // Behaviour across requests.
     rateAnomalyDetector(),
     cadenceDetector(),
     crawlBreadthDetector(),
+    parameterSweepDetector(),
+    probeVolumeDetector(),
+    idEnumerationDetector(),
     sessionIntegrityDetector(),
     // The other side of the argument: what a real browsing session looks like.
     browsingCoherenceDetector(),
@@ -4183,6 +4517,10 @@ var init_detectors = __esm({
     init_cadence();
     init_client_hints();
     init_crawl_breadth();
+    init_parameter_sweep();
+    init_transport_coherence();
+    init_probe_volume();
+    init_id_enumeration();
     init_crawler_verification();
     init_fetch_metadata();
     init_header_integrity();
@@ -4205,6 +4543,10 @@ var init_detectors = __esm({
     init_rate_anomaly();
     init_cadence();
     init_crawl_breadth();
+    init_parameter_sweep();
+    init_transport_coherence();
+    init_probe_volume();
+    init_id_enumeration();
     init_session_integrity();
     init_identity_rotation();
     init_trap();
@@ -5032,6 +5374,11 @@ function assessmentFromEntry(entry) {
       key: entry.actor,
       requests: entry.actorStats.requests,
       distinctPaths: entry.actorStats.distinctPaths,
+      distinctQueries: 0,
+      queriesSaturated: false,
+      methodsSeen: ["GET"],
+      responses: 0,
+      misses: 0,
       firstSeen: entry.actorStats.firstSeen,
       lastSeen: entry.at,
       ...entry.actorStats.sinceLastMs !== void 0 ? { sinceLastMs: entry.actorStats.sinceLastMs } : {},
@@ -5300,7 +5647,10 @@ button[disabled] { opacity: .5; cursor: default; }
 .tab[aria-selected="true"] { color: var(--ink); border-bottom-color: var(--s1); }
 
 /* --- layout ------------------------------------------------------------- */
-main { padding: 18px 20px 64px; max-width: 1680px; margin: 0 auto; }
+/* The top padding is the gap under the sticky header. At 18px the counter row sat almost
+   against the header's border and read as part of it; the tiles carry their own border, so
+   two lines were meeting with nothing between them. */
+main { padding: 28px 20px 64px; max-width: 1680px; margin: 0 auto; }
 .stack { display: grid; gap: 16px; }
 /* Everything above the feed is drawn by script once the first snapshot arrives, which
    inserts a block of content above what is already laid out. The browser's scroll
@@ -9293,6 +9643,24 @@ var BotHandler = class {
     this.warn(`Actor "${key}" was cleared as human at runtime${attribute(context)}, until ${new Date(until).toISOString()}.`);
     this.events.emit("actor-change", { key, action: "clear", until, by: context.by });
   }
+  /**
+   * Tells the engine what the application answered.
+   *
+   * The one thing detection cannot see for itself. Every verdict here is reached *before*
+   * the response exists — that is what makes it useful, since it can shape the response —
+   * and so the status is knowledge only the application holds. Handed back, it closes the
+   * oldest gap in reading a scanner: an actor whose requests are almost all misses is
+   * looking for something rather than reading anything, and no amount of header analysis
+   * shows that.
+   *
+   * Optional, and silent when the actor has already been forgotten. Nothing about
+   * detection depends on it being called; supplying it sharpens `probe-volume` and
+   * nothing else. The bundled Node adapter wires it up for you.
+   */
+  recordOutcome(facts, status) {
+    if (!Number.isFinite(status)) return;
+    this.registry.peek(this.actorKeyFor(facts))?.recordOutcome(status);
+  }
   /** Convenience for `updateRanges("crawler:<id>", …)`, matching a signature id. */
   updateCrawlerRanges(signatureId, entries, context = {}) {
     this.updateRanges(`crawler:${signatureId}`, entries, context);
@@ -9682,6 +10050,11 @@ var BotHandler = class {
         key: actorKey,
         requests: 0,
         distinctPaths: 0,
+        distinctQueries: 0,
+        queriesSaturated: false,
+        methodsSeen: ["GET"],
+        responses: 0,
+        misses: 0,
         firstSeen: facts.timestamp,
         lastSeen: facts.timestamp,
         priorConfirmations: 0,
@@ -9753,6 +10126,12 @@ var MAX_V6_BLOCK = 19;
 var MAX_PREFIXES = 1e4;
 var MAX_BYTES = 4 * 1024 * 1024;
 async function fetchCrawlerRanges(source, options = {}) {
+  return fetchPrefixes(source, options, { maxPrefixes: MAX_PREFIXES, subject: `a crawler's address list`, id: source.id });
+}
+async function fetchAddressList(source, options = {}) {
+  return fetchPrefixes(source, options, { maxPrefixes: options.maxPrefixes ?? 1e5, subject: "an address list", id: source.id });
+}
+async function fetchPrefixes(source, options, limits) {
   const url = new URL(source.url);
   if (url.protocol !== "https:") throw new ConfigError(`Crawler ranges must be published over HTTPS. "${source.url}" is not.`);
   const fetcher = options.fetch ?? globalThis.fetch;
@@ -9766,7 +10145,7 @@ async function fetchCrawlerRanges(source, options = {}) {
   const text = await response.text();
   if (text.length > MAX_BYTES) throw new Error(`the list is ${Math.round(text.length / 1024)} kB, which is not a list of prefixes`);
   const prefixes = text.trimStart().startsWith("{") ? fromJson(text) : fromLines(text);
-  return validate(prefixes, source.id);
+  return validate(prefixes, limits);
 }
 function fromJson(text) {
   const document = JSON.parse(text);
@@ -9779,24 +10158,24 @@ function fromJson(text) {
   return out;
 }
 function fromLines(text) {
-  return text.split("\n").map((line) => line.split("#")[0]?.trim() ?? "").filter((line) => line !== "");
+  return text.split("\n").map((line) => (line.split("#")[0] ?? "").split(";")[0]?.trim() ?? "").filter((line) => line !== "");
 }
-function validate(prefixes, id) {
+function validate(prefixes, limits) {
   if (prefixes.length === 0) throw new Error("the list is empty");
-  if (prefixes.length > MAX_PREFIXES) throw new Error(`${prefixes.length} prefixes is not a crawler's address list`);
+  if (prefixes.length > limits.maxPrefixes) throw new Error(`${prefixes.length} prefixes is not ${limits.subject}`);
   const accepted = [];
   for (const prefix of prefixes) {
     const cidr = parseCidr(prefix);
     if (cidr === void 0 || cidr === null) continue;
     const isV4 = cidr.bytes.length === 4;
     if (cidr.prefix < (isV4 ? MAX_V4_BLOCK : MAX_V6_BLOCK)) {
-      throw new Error(`"${prefix}" covers more of the internet than any crawler owns \u2014 refusing the whole list rather than verifying strangers`);
+      throw new Error(`"${prefix}" covers more of the internet than any published list should \u2014 refusing the whole list rather than acting on it`);
     }
     accepted.push(prefix);
   }
   if (accepted.length === 0) throw new Error(`nothing in the list parsed as an address or CIDR (first entry: "${prefixes[0]}")`);
   const set = new IpRangeSet(accepted);
-  if (set.size === 0) throw new Error(`nothing in the list loaded for ${id}`);
+  if (set.size === 0) throw new Error(`nothing in the list loaded for ${limits.id}`);
   return accepted;
 }
 async function refreshCrawlerRanges(handler, options = {}) {
@@ -10109,6 +10488,7 @@ export {
   defineHandler,
   evidence,
   executeAction,
+  fetchAddressList,
   fetchCrawlerRanges,
   fetchMetadataDetector,
   formatIp,
@@ -10117,6 +10497,7 @@ export {
   headerIntegrityDetector,
   headerOrderDetector,
   headerOrderFingerprint,
+  idEnumerationDetector,
   identityRotationDetector,
   independentStrongSignals,
   indexSignatures,
@@ -10132,6 +10513,7 @@ export {
   noisyOr,
   normalizeIp,
   notifyJsNotifier,
+  parameterSweepDetector,
   parseAcceptLanguage,
   parseCidr,
   parseCookies,
@@ -10141,6 +10523,7 @@ export {
   pickTranslation,
   probeShapeFor,
   probeSignatureDetector,
+  probeVolumeDetector,
   protectApi,
   protectAuth,
   protectContent,
@@ -10169,6 +10552,7 @@ export {
   systemClock,
   tlsFingerprintDetector,
   toPrometheus,
+  transportCoherenceDetector,
   trapDetector,
   trapRobotsEntries,
   uaCoherenceDetector,