@osqd/bothandlerjs 0.5.0 → 0.6.0

package/dist/cli.cjs

@@ -145,16 +145,20 @@ function hashString(value) {
   }
   return hash >>> 0;
 }
-var TIMESTAMP_RING, PATH_CAP, UA_CAP, MAX_TRACKED_ARRIVALS, MAX_TRACKED_PATHS, ActorState, ActorRegistry;
+var TIMESTAMP_RING, PATH_CAP, QUERY_CAP, METHOD_CAP, WALK_CAP, UA_CAP, MAX_TRACKED_ARRIVALS, MAX_TRACKED_PATHS, MAX_TRACKED_QUERIES, ActorState, ActorRegistry;
 var init_state = __esm({
   "src/state.ts"() {
     "use strict";
     init_lru();
     TIMESTAMP_RING = 32;
     PATH_CAP = 64;
+    QUERY_CAP = 64;
+    METHOD_CAP = 12;
+    WALK_CAP = 4;
     UA_CAP = 4;
     MAX_TRACKED_ARRIVALS = TIMESTAMP_RING;
     MAX_TRACKED_PATHS = PATH_CAP;
+    MAX_TRACKED_QUERIES = QUERY_CAP;
     ActorState = class {
       key;
       firstSeen;
@@ -195,6 +199,45 @@ var init_state = __esm({
       paths = /* @__PURE__ */ new Set();
       pathsOverflowed = false;
       pathsSaturatedAtTotal = 0;
+      /**
+       * Distinct *parameterised* requests: the path together with its query.
+       *
+       * Counted apart from `paths` because the two answer different questions and a scraper
+       * lives in the gap between them. `/products?page=1` through `?page=200` is one path and
+       * two hundred requests, so breadth reads it as somebody rereading a single page — which
+       * is exactly what enumerating a catalogue looks like from the path alone.
+       */
+      queries = /* @__PURE__ */ new Set();
+      queriesOverflowed = false;
+      /**
+       * Which HTTP methods this actor has used.
+       *
+       * A browser navigating issues GET. Something that has issued nothing but HEAD across a
+       * long visit is checking what exists rather than reading it, and that is a fact about
+       * the actor rather than about any one of its requests — which is why it is kept here.
+       */
+      methods = /* @__PURE__ */ new Set();
+      /**
+       * Numeric walks in progress, by path shape: `/user/#` against the ids requested under it.
+       *
+       * Three numbers per shape, deliberately — a count, a lowest and a highest — rather than
+       * the ids themselves. What separates enumeration from reading is not which ids were
+       * asked for but whether they *cover a range*: thirty requests spanning thirty
+       * consecutive ids is a walk, and thirty scattered across a hundred thousand is somebody
+       * following links. Both are answerable from a count and a span, and only the count and
+       * the span survive an actor asking for ten thousand of them.
+       */
+      walks = /* @__PURE__ */ new Map();
+      /**
+       * What the application answered, for the requests anybody bothered to tell us about.
+       *
+       * The engine decides *before* the response exists, so this arrives afterwards and only
+       * when the adapter reports it. Kept as two counters rather than a list because the one
+       * question worth asking is a ratio: an actor whose requests are almost all misses is
+       * looking for something rather than reading anything.
+       */
+      responsesSeen = 0;
+      missesSeen = 0;
       userAgents = /* @__PURE__ */ new Set();
       constructor(key, now) {
         this.key = key;
@@ -214,6 +257,15 @@ var init_state = __esm({
           this.pathsOverflowed = true;
           this.pathsSaturatedAtTotal = this.total;
         }
+        const keys = Object.keys(facts.query).sort();
+        if (keys.length > 0) {
+          const signature = `${facts.path}?${keys.map((key) => `${key}=${facts.query[key] ?? ""}`).join("&")}`;
+          const queryHash = hashString(signature);
+          if (this.queries.size < QUERY_CAP) this.queries.add(queryHash);
+          else if (!this.queries.has(queryHash)) this.queriesOverflowed = true;
+        }
+        if (this.methods.size < METHOD_CAP) this.methods.add(facts.method);
+        this.noteWalk(facts.path);
         const ua = facts.headers["user-agent"];
         if (ua !== void 0 && this.userAgents.size < UA_CAP) this.userAgents.add(ua);
       }
@@ -224,6 +276,79 @@ var init_state = __esm({
       get pathsSaturated() {
         return this.pathsOverflowed;
       }
+      /** Distinct path-and-query combinations seen. Saturates at {@link QUERY_CAP}. */
+      get distinctQueries() {
+        return this.queries.size;
+      }
+      get queriesSaturated() {
+        return this.queriesOverflowed;
+      }
+      /**
+       * Records what the application answered. Called after the response, if at all.
+       *
+       * 404 and 410 only. A 403 is usually this library's own doing and counting it would
+       * make the detector that reads this argue with itself; a 500 is the site's problem and
+       * says nothing about the client.
+       */
+      recordOutcome(status) {
+        this.responsesSeen++;
+        if (status === 404 || status === 410) this.missesSeen++;
+      }
+      /** Responses reported for this actor. Zero unless something is reporting them. */
+      get responses() {
+        return this.responsesSeen;
+      }
+      /** Of those, how many were 404 or 410. */
+      get misses() {
+        return this.missesSeen;
+      }
+      /**
+       * Files a request under the shape of its path, if that path carries a number.
+       *
+       * The last numeric segment is the one taken to be the identifier: in `/api/v2/orders/42`
+       * the version is part of the shape and the order id is what is being walked.
+       */
+      noteWalk(path) {
+        const segments = path.split("/");
+        let value;
+        let template = "";
+        for (const segment of segments) {
+          if (segment !== "" && /^\d+$/.test(segment)) {
+            const parsed = Number(segment);
+            if (Number.isSafeInteger(parsed) && parsed <= 1e7) value = parsed;
+            template += "/#";
+          } else if (segment !== "") {
+            template += `/${segment}`;
+          }
+        }
+        if (value === void 0) return;
+        const existing = this.walks.get(template);
+        if (existing !== void 0) {
+          existing.count++;
+          if (value < existing.min) existing.min = value;
+          if (value > existing.max) existing.max = value;
+          return;
+        }
+        if (this.walks.size < WALK_CAP) this.walks.set(template, { count: 1, min: value, max: value });
+      }
+      /**
+       * The path shape this actor has walked hardest, with how far it reached.
+       *
+       * `span` is inclusive of both ends, so a walk of 1 to 30 spans 30. Comparing the count
+       * against it is what separates covering a range from visiting a few points in one.
+       */
+      densestWalk() {
+        let best;
+        for (const [template, walk] of this.walks) {
+          const span = walk.max - walk.min + 1;
+          if (best === void 0 || walk.count > best.count) best = { template, count: walk.count, span };
+        }
+        return best;
+      }
+      /** Every HTTP method this actor has used, in first-seen order. */
+      get methodsSeen() {
+        return [...this.methods];
+      }
       /**
        * Requests seen when {@link distinctPaths} stopped being able to grow, or 0 if it
        * still can. Over that many requests the distinct count is exact, so it is the only
@@ -312,6 +437,12 @@ var init_state = __esm({
           key: this.key,
           requests: this.total,
           distinctPaths: this.distinctPaths,
+          distinctQueries: this.distinctQueries,
+          methodsSeen: this.methodsSeen,
+          walk: this.densestWalk(),
+          responses: this.responses,
+          misses: this.misses,
+          queriesSaturated: this.queriesSaturated,
           firstSeen: this.firstSeen,
           lastSeen: this.lastSeen,
           sinceLastMs: this.sinceLast(now),
@@ -3374,7 +3505,7 @@ function compileSignatures(signatures = BOT_SIGNATURES) {
   }
   return new MultiPatternMatcher(entries);
 }
-var BOT_CATEGORIES, SEARCH, AI, SEO, SOCIAL, MONITORING, ARCHIVE, FEED, SECURITY, LIBRARY, HEADLESS, EMBEDDED, ADVERTISING, COMMERCE, ACADEMIC, ACCESSIBILITY, BOT_SIGNATURES;
+var BOT_CATEGORIES, SEARCH, AI, SEO, SOCIAL, MONITORING, ARCHIVE, FEED, SECURITY, LIBRARY, HEADLESS, EMBEDDED, ADVERTISING, COMMERCE, ACADEMIC, EMAIL_SECURITY, ACCESSIBILITY, BOT_SIGNATURES;
 var init_known_bots = __esm({
   "src/detectors/known-bots.ts"() {
     "use strict";
@@ -3395,6 +3526,7 @@ var init_known_bots = __esm({
       "commerce",
       "accessibility",
       "academic",
+      "email-security",
       "other"
     ];
     SEARCH = [
@@ -3419,6 +3551,13 @@ var init_known_bots = __esm({
       { id: "yisouspider", name: "Shenma (Yisou) Spider", tokens: ["yisouspider"], category: "search", benign: true, robotsAgent: "YisouSpider", verification: { kind: "none" } },
       { id: "yahoo-slurp", name: "Yahoo! Slurp", tokens: ["yahoo! slurp"], category: "search", benign: true, robotsAgent: "Slurp", verification: { kind: "fcrdns", domains: ["crawl.yahoo.net", "yahoo.com"] } },
       { id: "mail-ru", name: "Mail.Ru bot", tokens: ["mail.ru_bot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "brave-search", name: "Brave Search", tokens: ["bravesearchbot"], category: "search", benign: true, robotsAgent: "BraveSearchBot", verification: { kind: "none" } },
+      { id: "ecosia", name: "Ecosia", tokens: ["ecosiabot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "startpage", name: "Startpage", tokens: ["startpagebot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "daum", name: "Daumoa", tokens: ["daumoa"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "stract", name: "Stract", tokens: ["stractbot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "rightdao", name: "RightDao", tokens: ["rightdaobot"], category: "search", benign: true, verification: { kind: "none" } },
+      { id: "gigablast", name: "Gigablast", tokens: ["gigablastopensource"], category: "search", benign: true, verification: { kind: "none" } },
       { id: "exabot", name: "Exabot (Exalead)", tokens: ["exabot"], category: "search", benign: true, verification: { kind: "none" } },
       { id: "kagibot", name: "Kagi", tokens: ["kagibot"], category: "search", benign: true, robotsAgent: "KagiBot", verification: { kind: "none" } }
     ];
@@ -3489,11 +3628,15 @@ var init_known_bots = __esm({
       { id: "pinterestbot", name: "Pinterestbot", tokens: ["pinterest/", "pinterestbot"], category: "social", benign: true, verification: { kind: "fcrdns", domains: ["pinterest.com"] } },
       { id: "mastodon", name: "Mastodon / Fediverse", tokens: ["mastodon/", "pleroma", "misskey/", "akkoma"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "embedly", name: "Embedly", tokens: ["embedly"], category: "social", benign: true, verification: { kind: "none" } },
-      { id: "bluesky", name: "Bluesky card fetcher", tokens: ["bluesky cardyb", "cardyb/"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "bluesky", name: "Bluesky card fetcher", tokens: ["bluesky cardyb", "cardyb/", "blueskybot"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "iframely", name: "Iframely", tokens: ["iframely"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "skype-preview", name: "Skype URI preview", tokens: ["skypeuripreview"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "vk-share", name: "VK / Odnoklassniki preview", tokens: ["vkshare", "odklbot"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "discourse-onebox", name: "Discourse Onebox", tokens: ["discourse forum onebox"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "microsoft-preview", name: "Microsoft Teams preview", tokens: ["microsoftpreview"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "zoom-preview", name: "Zoom link preview", tokens: ["zoombot"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "signal-preview", name: "Signal link preview", tokens: ["signalbot"], category: "social", benign: true, verification: { kind: "none" } },
+      { id: "matrix-synapse", name: "Matrix (Synapse) preview", tokens: ["synapse/"], category: "social", benign: true, verification: { kind: "none" } },
       { id: "yahoo-preview", name: "Yahoo Link Preview", tokens: ["yahoo link preview"], category: "social", benign: true, verification: { kind: "none" } }
     ];
     MONITORING = [
@@ -3533,6 +3676,10 @@ var init_known_bots = __esm({
       { id: "smartnews", name: "SmartNews", tokens: ["smartnewsbot"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "flipboard", name: "Flipboard", tokens: ["flipboardproxy"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "podcast-index", name: "Podcast Index", tokens: ["podcastindexbot"], category: "feed", benign: true, verification: { kind: "none" } },
+      // Spotify's podcast fetcher sends `Spotify/1.0` — and so does the Spotify desktop app,
+      // with a person driving it. There is no token that separates them, so this one is left
+      // unnamed rather than named wrongly: the corpus proved the point immediately by blocking
+      // a human under `protect-auth`, `indexers-only` and `under-attack` at once.
       { id: "freshrss", name: "FreshRSS", tokens: ["freshrss"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "netnewswire", name: "NetNewsWire", tokens: ["netnewswire"], category: "feed", benign: true, verification: { kind: "none" } },
       { id: "overcast", name: "Overcast", tokens: ["overcast/"], category: "feed", benign: true, verification: { kind: "none" } },
@@ -3659,7 +3806,23 @@ var init_known_bots = __esm({
     ];
     ADVERTISING = [
       { id: "adsbot-google", name: "AdsBot-Google", tokens: ["adsbot-google", "mediapartners-google", "adsbot"], category: "advertising", benign: true, verification: { kind: "fcrdns", domains: ["googlebot.com", "google.com"] } },
-      { id: "criteo", name: "Criteo", tokens: ["criteobot"], category: "advertising", benign: true, verification: { kind: "none" } }
+      { id: "criteo", name: "Criteo", tokens: ["criteobot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      // Verification and contextual classification: they read a page to decide whether an ad
+      // may appear beside it, or what the page is about. A publisher usually wants these and a
+      // site with no advertising has no reason to.
+      { id: "doubleverify", name: "DoubleVerify", tokens: ["doubleverifybot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "ias", name: "Integral Ad Science", tokens: ["ias crawler"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "moat", name: "Moat", tokens: ["moatbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "comscore", name: "comScore (Proximic)", tokens: ["proximic"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "grapeshot", name: "Grapeshot", tokens: ["grapeshotcrawler"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "peer39", name: "Peer39", tokens: ["peer39bot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "taboola", name: "Taboola", tokens: ["taboolabot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "outbrain", name: "Outbrain", tokens: ["outbrainbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "pubmatic", name: "PubMatic", tokens: ["pubmaticbot"], category: "advertising", benign: true, verification: { kind: "none" } },
+      { id: "thetradedesk", name: "The Trade Desk", tokens: ["ttd-content"], category: "advertising", benign: true, verification: { kind: "none" } },
+      // Competitive ad intelligence rather than verification: it collects what everyone else
+      // is running. Named, and left for the operator to decide about.
+      { id: "adbeat", name: "Adbeat", tokens: ["adbeat_bot"], category: "advertising", benign: false, verification: { kind: "none" } }
     ];
     COMMERCE = [
       { id: "idealo", name: "idealo", tokens: ["idealo-bot"], category: "commerce", benign: false, verification: { kind: "none" } },
@@ -3675,6 +3838,12 @@ var init_known_bots = __esm({
       { id: "openalex", name: "OpenAlex", tokens: ["openalexbot"], category: "academic", benign: true, verification: { kind: "none" } },
       { id: "webis", name: "Webis research crawler", tokens: ["webisbot"], category: "academic", benign: true, verification: { kind: "none" } }
     ];
+    EMAIL_SECURITY = [
+      { id: "proofpoint", name: "Proofpoint URL Defense", tokens: ["proofpointurldefensebot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "mimecast", name: "Mimecast URL Protect", tokens: ["mimecasturlprotectbot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "barracuda", name: "Barracuda Link Protect", tokens: ["barracudalinkprotectbot"], category: "email-security", benign: true, verification: { kind: "none" } },
+      { id: "cisco-esa", name: "Cisco Secure Email", tokens: ["ciscosecureemailbot"], category: "email-security", benign: true, verification: { kind: "none" } }
+    ];
     ACCESSIBILITY = [
       { id: "siteimprove", name: "Siteimprove", tokens: ["siteimprovebot"], category: "accessibility", benign: true, robotsAgent: "SiteimproveBot", verification: { kind: "none" } }
     ];
@@ -3693,7 +3862,8 @@ var init_known_bots = __esm({
       ...EMBEDDED,
       ...COMMERCE,
       ...ACADEMIC,
-      ...ACCESSIBILITY
+      ...ACCESSIBILITY,
+      ...EMAIL_SECURITY
     ]);
   }
 });
@@ -4445,6 +4615,166 @@ var init_crawl_breadth = __esm({
   }
 });
 
+// src/detectors/parameter-sweep.ts
+function parameterSweepDetector(options = {}) {
+  const threshold = options.threshold ?? 25;
+  const variantsPerPath = options.variantsPerPath ?? 8;
+  const minRequests = options.minRequests ?? 20;
+  if (threshold > MAX_TRACKED_QUERIES) {
+    throw new RangeError(
+      `parameterSweepDetector threshold ${threshold} can never be reached: an actor's distinct-query count saturates at ${MAX_TRACKED_QUERIES}. Use ${MAX_TRACKED_QUERIES} or fewer.`
+    );
+  }
+  return {
+    id: "parameter-sweep",
+    description: "Counts distinct query strings against the paths they sit on, to catch enumeration that leaves the path unchanged",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const { distinctPaths, distinctQueries, queriesSaturated, total } = ctx.state;
+      if (total < minRequests || distinctQueries < threshold) return void 0;
+      const spread = distinctQueries / Math.max(1, distinctPaths);
+      if (spread < variantsPerPath) return void 0;
+      return {
+        detector: "parameter-sweep",
+        summary: queriesSaturated ? `at least ${distinctQueries} distinct query strings across only ${distinctPaths} path(s)` : `${distinctQueries} distinct query strings across only ${distinctPaths} path(s) in ${total} requests`,
+        direction: "bot",
+        certainty: "weak",
+        // Saturation means the count stopped being able to grow, so the real spread is
+        // wider than the one reported — the same argument breadth makes for itself.
+        weight: queriesSaturated ? 0.25 : 0.15,
+        botClass: "scraper",
+        metadata: {
+          distinctQueries,
+          distinctPaths,
+          variantsPerPath: Number(spread.toFixed(1)),
+          totalRequests: total,
+          saturated: queriesSaturated
+        }
+      };
+    }
+  };
+}
+var init_parameter_sweep = __esm({
+  "src/detectors/parameter-sweep.ts"() {
+    "use strict";
+    init_state();
+  }
+});
+
+// src/detectors/transport-coherence.ts
+function transportCoherenceDetector(options = {}) {
+  const checkLegacyHttp = options.legacyHttp ?? true;
+  const minHeadRequests = options.minHeadRequests ?? 8;
+  return {
+    id: "transport-coherence",
+    description: "Reads the HTTP version and the methods across a visit against the client the request claims to be",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      if (!claimsBrowser(ctx.ua)) return void 0;
+      const results = [];
+      const version = ctx.facts.httpVersion;
+      if (checkLegacyHttp && version !== void 0 && LEGACY_VERSIONS.has(version)) {
+        results.push({
+          detector: "transport-coherence",
+          summary: `Client claims to be a browser but negotiated HTTP/${version}, which no shipping browser has offered in over a decade`,
+          direction: "bot",
+          certainty: "moderate",
+          botClass: "impersonator",
+          // One downgrading proxy in front of the application does this to every request
+          // that passes through it, so this must count once rather than once per reason.
+          family: "legacy-transport",
+          metadata: { httpVersion: version, browser: ctx.ua.browser }
+        });
+      }
+      const methods = ctx.state.methodsSeen;
+      if (ctx.state.total >= minHeadRequests && methods.length > 0 && methods.every((method) => method === "HEAD")) {
+        results.push({
+          detector: "transport-coherence",
+          summary: `Client claims to be a browser but has issued nothing but HEAD across ${ctx.state.total} requests`,
+          direction: "bot",
+          certainty: "moderate",
+          botClass: "scraper",
+          metadata: { requests: ctx.state.total, browser: ctx.ua.browser }
+        });
+      }
+      return results.length > 0 ? results : void 0;
+    }
+  };
+}
+var LEGACY_VERSIONS;
+var init_transport_coherence = __esm({
+  "src/detectors/transport-coherence.ts"() {
+    "use strict";
+    init_ua();
+    LEGACY_VERSIONS = /* @__PURE__ */ new Set(["0.9", "1.0"]);
+  }
+});
+
+// src/detectors/probe-volume.ts
+function probeVolumeDetector(options = {}) {
+  const minResponses = options.minResponses ?? 20;
+  const missRatio = options.missRatio ?? 0.8;
+  return {
+    id: "probe-volume",
+    description: "Reads the share of an actor's requests that were answered 404 or 410, where the application reports them",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const { responses, misses } = ctx.state;
+      if (responses < minResponses) return void 0;
+      const ratio = misses / responses;
+      if (ratio < missRatio) return void 0;
+      return {
+        detector: "probe-volume",
+        summary: `${misses} of this client's last ${responses} requests were answered "not found" (${(ratio * 100).toFixed(0)}%)`,
+        direction: "bot",
+        certainty: "moderate",
+        botClass: "scanner",
+        metadata: { responses, misses, missRatio: Number(ratio.toFixed(3)) }
+      };
+    }
+  };
+}
+var init_probe_volume = __esm({
+  "src/detectors/probe-volume.ts"() {
+    "use strict";
+  }
+});
+
+// src/detectors/id-enumeration.ts
+function idEnumerationDetector(options = {}) {
+  const minRequests = options.minRequests ?? 30;
+  const density = options.density ?? 0.9;
+  return {
+    id: "id-enumeration",
+    description: "Reports an actor covering a contiguous range of numeric identifiers under one path shape",
+    cost: "cheap",
+    stage: "always",
+    inspect(ctx) {
+      const walk = ctx.state.densestWalk();
+      if (walk === void 0 || walk.count < minRequests) return void 0;
+      if (walk.span < minRequests) return void 0;
+      const covered = Math.min(1, walk.count / walk.span);
+      if (covered < density) return void 0;
+      return {
+        detector: "id-enumeration",
+        summary: `${walk.count} requests to ${walk.template} covering ${(covered * 100).toFixed(0)}% of a ${walk.span}-wide range of ids`,
+        direction: "bot",
+        certainty: "moderate",
+        botClass: "scraper",
+        metadata: { template: walk.template, requests: walk.count, span: walk.span, coverage: Number(covered.toFixed(3)) }
+      };
+    }
+  };
+}
+var init_id_enumeration = __esm({
+  "src/detectors/id-enumeration.ts"() {
+    "use strict";
+  }
+});
+
 // src/detectors/crawler-verification.ts
 function crawlerVerificationDetector(options = {}) {
   const missingPtrIsForgery = options.treatMissingPtrAsForgery ?? true;
@@ -5622,11 +5952,15 @@ function defaultDetectors(options = {}) {
     clientHintsDetector(),
     fetchMetadataDetector(),
     acceptSignatureDetector(),
+    transportCoherenceDetector(),
     headerOrderDetector(),
     // Behaviour across requests.
     rateAnomalyDetector(),
     cadenceDetector(),
     crawlBreadthDetector(),
+    parameterSweepDetector(),
+    probeVolumeDetector(),
+    idEnumerationDetector(),
     sessionIntegrityDetector(),
     // The other side of the argument: what a real browsing session looks like.
     browsingCoherenceDetector(),
@@ -5642,6 +5976,10 @@ var init_detectors = __esm({
     init_cadence();
     init_client_hints();
     init_crawl_breadth();
+    init_parameter_sweep();
+    init_transport_coherence();
+    init_probe_volume();
+    init_id_enumeration();
     init_crawler_verification();
     init_fetch_metadata();
     init_header_integrity();
@@ -6493,6 +6831,11 @@ function assessmentFromEntry(entry) {
       key: entry.actor,
       requests: entry.actorStats.requests,
       distinctPaths: entry.actorStats.distinctPaths,
+      distinctQueries: 0,
+      queriesSaturated: false,
+      methodsSeen: ["GET"],
+      responses: 0,
+      misses: 0,
       firstSeen: entry.actorStats.firstSeen,
       lastSeen: entry.at,
       ...entry.actorStats.sinceLastMs !== void 0 ? { sinceLastMs: entry.actorStats.sinceLastMs } : {},
@@ -6761,7 +7104,10 @@ button[disabled] { opacity: .5; cursor: default; }
 .tab[aria-selected="true"] { color: var(--ink); border-bottom-color: var(--s1); }
 
 /* --- layout ------------------------------------------------------------- */
-main { padding: 18px 20px 64px; max-width: 1680px; margin: 0 auto; }
+/* The top padding is the gap under the sticky header. At 18px the counter row sat almost
+   against the header's border and read as part of it; the tiles carry their own border, so
+   two lines were meeting with nothing between them. */
+main { padding: 28px 20px 64px; max-width: 1680px; margin: 0 auto; }
 .stack { display: grid; gap: 16px; }
 /* Everything above the feed is drawn by script once the first snapshot arrives, which
    inserts a block of content above what is already laid out. The browser's scroll
@@ -9169,6 +9515,24 @@ var init_core = __esm({
         this.warn(`Actor "${key}" was cleared as human at runtime${attribute(context)}, until ${new Date(until).toISOString()}.`);
         this.events.emit("actor-change", { key, action: "clear", until, by: context.by });
       }
+      /**
+       * Tells the engine what the application answered.
+       *
+       * The one thing detection cannot see for itself. Every verdict here is reached *before*
+       * the response exists — that is what makes it useful, since it can shape the response —
+       * and so the status is knowledge only the application holds. Handed back, it closes the
+       * oldest gap in reading a scanner: an actor whose requests are almost all misses is
+       * looking for something rather than reading anything, and no amount of header analysis
+       * shows that.
+       *
+       * Optional, and silent when the actor has already been forgotten. Nothing about
+       * detection depends on it being called; supplying it sharpens `probe-volume` and
+       * nothing else. The bundled Node adapter wires it up for you.
+       */
+      recordOutcome(facts, status) {
+        if (!Number.isFinite(status)) return;
+        this.registry.peek(this.actorKeyFor(facts))?.recordOutcome(status);
+      }
       /** Convenience for `updateRanges("crawler:<id>", …)`, matching a signature id. */
       updateCrawlerRanges(signatureId, entries, context = {}) {
         this.updateRanges(`crawler:${signatureId}`, entries, context);
@@ -9558,6 +9922,11 @@ var init_core = __esm({
             key: actorKey,
             requests: 0,
             distinctPaths: 0,
+            distinctQueries: 0,
+            queriesSaturated: false,
+            methodsSeen: ["GET"],
+            responses: 0,
+            misses: 0,
             firstSeen: facts.timestamp,
             lastSeen: facts.timestamp,
             priorConfirmations: 0,
@@ -10368,6 +10737,89 @@ var init_adversarial = __esm({
     init_ranges();
     CHROME_UA = userAgentOf("chromeWindows");
     ADVERSARIAL_CASES = [
+      bot({
+        id: "id-harvest-contiguous",
+        title: "Every profile id in order, with a copied browser header set",
+        audience: "hostile",
+        category: "scraping",
+        provenance: "Harvesting by identifier rather than by link: the shape of an IDOR sweep and of profile collection. Distinct-path breadth reads it as somebody who visited a lot of pages, which is also what it reads when a person works through a documentation site.",
+        requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.65" }, 40, 800, (index) => `/user/${index + 1}`),
+        expect: {
+          // One `moderate` signal against a flawless header set, like the others here. What
+          // changed is that the walk is now *visible* — before this detector it was scored
+          // identically to a hundred and twenty scattered ids and to ordinary article paths.
+          verdict: "unknown",
+          detectors: ["id-enumeration"]
+        },
+        notes: "What separates this from reading is not which ids were asked for but that they cover a range: people arrive at ids through links, and links do not densely enumerate an integer interval. Held at `moderate` because products in one category often carry consecutive ids, so somebody browsing a catalogue makes a smaller version of this shape."
+      }),
+      bot({
+        id: "wordlist-scan-mostly-misses",
+        title: "A wordlist walked with a copied browser header set, almost all of it missing",
+        audience: "hostile",
+        category: "scanning",
+        provenance: "The oldest tell there is, and the one this library could not see: it decides before the response exists, which is what lets it shape the response and also what hides the status from it. A person browsing does not generate thirty misses in a row; a wordlist does almost nothing else.",
+        requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.64", status: 404 }, 30, 700, (index) => `/${["admin", "backup", "old", "test", "config", "db"][index % 6]}-${index}`),
+        expect: {
+          // One `moderate` signal against an otherwise flawless header set does not cross the
+          // line, and it should not: a site that has just moved its URLs produces the same
+          // shape from ordinary readers. Raising the ceiling so this case reads better would
+          // be tuning the detector to the test rather than to the traffic.
+          verdict: "unknown",
+          detectors: ["probe-volume"]
+        },
+        notes: "Only counts 404 and 410. A 403 is usually this library's own doing, and counting it would let a rule that challenges an actor manufacture the evidence for having challenged it; a 500 is the site's problem and says nothing about the client. Capped at `moderate` because a site that has just moved its URLs produces this from perfectly ordinary readers."
+      }),
+      bot({
+        id: "browser-claim-over-http-1-0",
+        title: "A perfect Chrome header set, arriving over HTTP/1.0",
+        audience: "hostile",
+        category: "impersonation",
+        provenance: "Most tooling lets you set headers and does not let you choose an HTTP version, so the transport is the half a copied header set does not cover. No shipping browser has offered HTTP/1.0 to a server in well over a decade.",
+        requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.62", httpVersion: "1.0" }, 30, 900, (index) => `/${["news", "about", "blog", "help", "terms"][index % 5]}`),
+        expect: {
+          // Contributes rather than concludes. On its own, against an otherwise flawless
+          // header set, one `moderate` signal does not reach the threshold — and it should
+          // not, because an intermediary can cause this. Beside anything sharper it does.
+          verdict: "unknown",
+          detectors: ["transport-coherence"]
+        },
+        notes: "Capped at `moderate` because it is not always the client's doing: a few older load balancers speak HTTP/1.0 to the origin, and behind one of those every request looks like this. That is what `transportCoherenceDetector({ legacyHttp: false })` is for, and why this may never deny anybody on its own."
+      }),
+      bot({
+        id: "head-only-visit",
+        title: "A visit made entirely of HEAD, claiming a browser",
+        audience: "unwanted-bot",
+        category: "scraping",
+        provenance: "Checking what exists without reading any of it: link checkers, availability monitors and inventory watchers all do this, and a browser navigating never does.",
+        requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.63", method: "HEAD" }, 30, 900, (index) => `/${["news", "about", "blog", "help", "terms"][index % 5]}`),
+        expect: {
+          // As above: a shape worth reporting, not worth concluding from alone.
+          verdict: "unknown",
+          detectors: ["transport-coherence"]
+        },
+        notes: "One HEAD is a browser checking a link it is about to follow, or a cache revalidating; the shape only means anything across a visit, which is why it is counted on the actor rather than on the request. A link checker is a real and mostly harmless thing to be, so this stays `moderate`."
+      }),
+      bot({
+        id: "catalogue-sweep-by-page",
+        title: "A catalogue taken a page at a time, with the path never changing",
+        audience: "unwanted-bot",
+        category: "scraping",
+        provenance: "How a catalogue is actually taken. The collector copies a browser's headers exactly and walks ?page=1..N, which leaves the path constant \u2014 so distinct-path breadth reads it as somebody rereading one page rather than as enumeration.",
+        requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.61" }, 40, 900, (index) => `/products?page=${index}`),
+        expect: {
+          // Not proven, and not even suspected at this pace. Said plainly because it is true:
+          // headers this clean leave only behaviour, behaviour is weak by construction, and a
+          // collector polite enough to space its requests stays under the line. What changed
+          // is that it no longer scores *lower* than the identical crawl expressed as distinct
+          // paths — measured at a faster pace before this detector existed, the two differed by
+          // seven points and only the path version crossed; they now score the same at every
+          // volume tried.
+          verdict: "unknown",
+          detectors: ["parameter-sweep"]
+        },
+        notes: "The counterpart to crawl-breadth rather than a replacement for it: breadth counts paths, this counts what is hung on them. Both stay weak, and both are worth having because a collector picks one shape or the other and nothing says which. Neither is a reason to deny anybody on its own."
+      }),
       // ---------------------------------------------------------------------------
       // Forged identities. The narrow case where a lie is provable.
       // ---------------------------------------------------------------------------
@@ -13310,6 +13762,7 @@ async function runCase(handler, clock, item, startedAt, provides, assertActions
     const withClearance = clearanceCookie === void 0 ? request : { ...request, headers: [...request.headers, ["Cookie", clearanceCookie]] };
     const facts = toFacts(withClearance, fallbackIp, clock.now());
     const { assessment, decision, outcome } = await handler.handle(facts);
+    if (request.status !== void 0) handler.recordOutcome(facts, request.status);
     requests.push({ assessment, decision, outcome });
   }
   const final = requests[requests.length - 1];