@thi.ng/server 0.7.0 → 0.8.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-03-09T19:21:53Z
+- **Last updated**: 2025-03-13T14:21:38Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,15 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.8.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/server@0.8.0) (2025-03-13)
+
+#### 🚀 Features
+
+- add method adapter, update `RequestCtx` ([360abda](https://github.com/thi-ng/umbrella/commit/360abda))
+  - add `ServerOpts.method` to allow for method conversion
+- add `rejectUserAgents()` interceptor ([de0c373](https://github.com/thi-ng/umbrella/commit/de0c373))
+  - add UA presets for AI bots & scrapers
+
 ## [0.7.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/server@0.7.0) (2025-03-09)
 
 #### 🚀 Features

package/README.md

@@ -80,6 +80,7 @@ for more details.
 - [`logResponse()`](https://docs.thi.ng/umbrella/server/functions/logResponse.html): Response logging
 - [`rateLimiter()`](https://docs.thi.ng/umbrella/server/functions/rateLimiter-1.html): Configurable rate limiting
 - [`referrerPolicy()`](https://docs.thi.ng/umbrella/server/functions/referrerPolicy-1.html): Policy header injection
+- [`rejectUserAgents()`](https://docs.thi.ng/umbrella/server/functions/rejectUserAgents.html): Configurable UA blocking
 - [`sessionInterceptor()`](https://docs.thi.ng/umbrella/server/functions/sessionInterceptor-1.html): User defined in-memory sessions with TTL
 - [`strictTransportSecurity()`](https://docs.thi.ng/umbrella/server/functions/strictTransportSecurity.html): Policy header injection
 
@@ -149,7 +150,7 @@ For Node.js REPL:
 const ser = await import("@thi.ng/server");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 5.15 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 6.15 KB
 
 ## Dependencies
 
@@ -203,6 +204,8 @@ const app = srv.server<AppCtx>({
     intercept: [
         // log all requests (using server's configured logger)
         srv.logRequest(),
+        // block known AI bots
+        srv.rejectUserAgents(srv.USER_AGENT_AI_BOTS),
         // lookup/create sessions (using above interceptor)
         session,
         // ensure routes with `auth` flag have a logged-in user

package/api.d.ts

@@ -1,4 +1,4 @@
-import type { Fn, Maybe, MaybePromise } from "@thi.ng/api";
+import type { Fn, Fn2, Maybe, MaybePromise } from "@thi.ng/api";
 import type { ILogger } from "@thi.ng/logger";
 import type { Route, RouteMatch } from "@thi.ng/router";
 import type { IncomingMessage } from "node:http";
@@ -67,6 +67,12 @@ export interface ServerOpts<CTX extends RequestCtx = RequestCtx> {
      * Reference: https://nodejs.org/api/net.html#class-netblocklist
      */
     blockList: BlockList;
+    /**
+     * HTTP method adapter/converter. The default implementation only converts
+     * {@link Method} `head` to `get` if the route does not provide a `head`
+     * handler.
+     */
+    method: Fn2<Method, Pick<RequestCtx, "req" | "route" | "match" | "cookies" | "query">, Method>;
 }
 export interface ServerRoute<CTX extends RequestCtx = RequestCtx> extends Route {
     handlers: Partial<Record<Method, RequestHandler<CTX>>>;
@@ -85,6 +91,8 @@ export interface RequestCtx {
     res: ServerResponse;
     route: CompiledServerRoute;
     match: RouteMatch;
+    method: Method;
+    origMethod: Method;
     path: string;
     query: Record<string, any>;
     cookies?: Record<string, string>;

package/index.d.ts

@@ -8,6 +8,7 @@ export * from "./interceptors/logging.js";
 export * from "./interceptors/measure.js";
 export * from "./interceptors/rate-limit.js";
 export * from "./interceptors/referrer-policy.js";
+export * from "./interceptors/reject-useragent.js";
 export * from "./interceptors/strict-transport.js";
 export * from "./interceptors/x-origin-opener.js";
 export * from "./interceptors/x-origin-resource.js";

package/index.js

@@ -8,6 +8,7 @@ export * from "./interceptors/logging.js";
 export * from "./interceptors/measure.js";
 export * from "./interceptors/rate-limit.js";
 export * from "./interceptors/referrer-policy.js";
+export * from "./interceptors/reject-useragent.js";
 export * from "./interceptors/strict-transport.js";
 export * from "./interceptors/x-origin-opener.js";
 export * from "./interceptors/x-origin-resource.js";

package/interceptors/reject-useragent.d.ts

@@ -0,0 +1,21 @@
+import type { Interceptor } from "../api.js";
+/**
+ * Pre-interceptor to check `User-Agent` header against given regexp. If the
+ * regexp matches, triggers a HTTP 403 response and terminates the request.
+ *
+ * @param patterns
+ */
+export declare const rejectUserAgents: (patterns: string | RegExp) => Interceptor;
+/**
+ * String defining partial regexp of known AI bot names in `User-Agent` header.
+ *
+ * Source: https://github.com/qwebltd/Useful-scripts/blob/main/Bash%20scripts%20for%20Linux/nginx-badbot-forbids.conf
+ */
+export declare const USER_AGENT_AI_BOTS = "AI2Bot|Anthropic|BrightBot|ByteDance|ByteSpider|CCBot|ChatGPT|ClaudeBot|Claude-Web|Cohere-AI|Cohere-Training-Data-Crawler|DiffBot|DuckAssistBot|FriendlyCrawler|Friendly_Crawler|Google-CloudVertexBot|GPTBot|ICC-Crawler|Img2Dataset|Kangaroo Bot|MLBot|OAI-SearchBot|PanguBot|Sentibot|VelenPublicWebCrawler|Webzio-Extended";
+/**
+ * String defining partial regexp of known crawlers & scraper names in `User-Agent` header.
+ *
+ * Source: https://github.com/qwebltd/Useful-scripts/blob/main/Bash%20scripts%20for%20Linux/nginx-rate-limiting.conf
+ */
+export declare const USER_AGENT_SCRAPERS = "008|AddSugarSpiderBot|AdsBot|AhrefsBot|AmazonBot|Arachmo|Barkrowler|BimBot|BlexBot|Boitho.com|BTBot|ConveraCrawler|DiamondBot|DotBot|Earthcom.info|EmeraldShield.com|EsperanzaBot|FacebookBot|Fast Enterprise|FindLinks|FurlBot|FyberSpider|GaisBot|GigaBot|GirafaBot|GoogleOther|Go-HTTP-Client|HL_Ftien_Spider|Holmes|HTDig|ICCrawler|Ichiro|IgdeSpyder|ImageSiftBot|IonCrawl|IRLbot|ISSCyberRiskCrawler|IssueCrawler|Jaxified Bot|JyxoBot|KoepaBot|Kototoi.org|Larbin|LDSpider|LinkWalker|LMSpider|Lwp-Trivial|L.Webis|Mabontland|Magpie-Crawler|Mail.RU_Bot|Masscan-NG|Meltwater|Meta-ExternalAgent|Mogimogi|MoreoverBot|Morning Paper|MSRBot|MVAClient|MXBot|NetResearchServer|NetSeer Crawler|NewsGator|NiceBot|NUSearch Spider|Nutch|Nymesis|OmniExplorer_Bot|OrbBot|OozBot|PageBitesHyperBot|Peer39_Crawler|PolyBot|PSBot|PycUrl|Qseero|Radian6|RampyBot|RufusBot|SandCrawler|SBIder|Scrapy|SeekBot|SemanticDiscovery|Semrush|Sensis Web Crawler|SEOChat|Shim-Crawler|SiteBot|Snappy|SurveyBot|Sqworm|SuggyBot|SynooBot|TerrawizBot|TheSuBot|Thumbnail.cz|TimpiBot|TinEye|TruwoGPS|TurnItInBot|TweetedTimes Bot|UrlFileBot|Vagabondo|Vortex|Voyager|VYU2|WebCollage|Wf84|WomlpeFactory|Xaldon_WebSpider|Yacy|YasakliBot";
+//# sourceMappingURL=reject-useragent.d.ts.map

package/interceptors/reject-useragent.js

@@ -0,0 +1,21 @@
+import { isString } from "@thi.ng/checks";
+const rejectUserAgents = (patterns) => {
+  const regexp = isString(patterns) ? new RegExp(patterns) : patterns;
+  return {
+    pre: (ctx) => {
+      const ua = ctx.req.headers["user-agent"];
+      if (ua && regexp.test(ua)) {
+        ctx.res.forbidden();
+        return false;
+      }
+      return true;
+    }
+  };
+};
+const USER_AGENT_AI_BOTS = "AI2Bot|Anthropic|BrightBot|ByteDance|ByteSpider|CCBot|ChatGPT|ClaudeBot|Claude-Web|Cohere-AI|Cohere-Training-Data-Crawler|DiffBot|DuckAssistBot|FriendlyCrawler|Friendly_Crawler|Google-CloudVertexBot|GPTBot|ICC-Crawler|Img2Dataset|Kangaroo Bot|MLBot|OAI-SearchBot|PanguBot|Sentibot|VelenPublicWebCrawler|Webzio-Extended";
+const USER_AGENT_SCRAPERS = "008|AddSugarSpiderBot|AdsBot|AhrefsBot|AmazonBot|Arachmo|Barkrowler|BimBot|BlexBot|Boitho.com|BTBot|ConveraCrawler|DiamondBot|DotBot|Earthcom.info|EmeraldShield.com|EsperanzaBot|FacebookBot|Fast Enterprise|FindLinks|FurlBot|FyberSpider|GaisBot|GigaBot|GirafaBot|GoogleOther|Go-HTTP-Client|HL_Ftien_Spider|Holmes|HTDig|ICCrawler|Ichiro|IgdeSpyder|ImageSiftBot|IonCrawl|IRLbot|ISSCyberRiskCrawler|IssueCrawler|Jaxified Bot|JyxoBot|KoepaBot|Kototoi.org|Larbin|LDSpider|LinkWalker|LMSpider|Lwp-Trivial|L.Webis|Mabontland|Magpie-Crawler|Mail.RU_Bot|Masscan-NG|Meltwater|Meta-ExternalAgent|Mogimogi|MoreoverBot|Morning Paper|MSRBot|MVAClient|MXBot|NetResearchServer|NetSeer Crawler|NewsGator|NiceBot|NUSearch Spider|Nutch|Nymesis|OmniExplorer_Bot|OrbBot|OozBot|PageBitesHyperBot|Peer39_Crawler|PolyBot|PSBot|PycUrl|Qseero|Radian6|RampyBot|RufusBot|SandCrawler|SBIder|Scrapy|SeekBot|SemanticDiscovery|Semrush|Sensis Web Crawler|SEOChat|Shim-Crawler|SiteBot|Snappy|SurveyBot|Sqworm|SuggyBot|SynooBot|TerrawizBot|TheSuBot|Thumbnail.cz|TimpiBot|TinEye|TruwoGPS|TurnItInBot|TweetedTimes Bot|UrlFileBot|Vagabondo|Vortex|Voyager|VYU2|WebCollage|Wf84|WomlpeFactory|Xaldon_WebSpider|Yacy|YasakliBot";
+export {
+  USER_AGENT_AI_BOTS,
+  USER_AGENT_SCRAPERS,
+  rejectUserAgents
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/server",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Minimal HTTP server with declarative routing, static file serving and freely extensible via pre/post interceptors",
   "type": "module",
   "module": "./index.js",
@@ -45,7 +45,7 @@
     "@thi.ng/checks": "^3.7.2",
     "@thi.ng/errors": "^2.5.28",
     "@thi.ng/file-io": "^2.1.31",
-    "@thi.ng/leaky-bucket": "^0.1.0",
+    "@thi.ng/leaky-bucket": "^0.2.0",
     "@thi.ng/logger": "^3.1.3",
     "@thi.ng/mime": "^2.7.4",
     "@thi.ng/paths": "^5.2.5",
@@ -123,6 +123,9 @@
     "./interceptors/referrer-policy": {
       "default": "./interceptors/referrer-policy.js"
     },
+    "./interceptors/reject-useragent": {
+      "default": "./interceptors/reject-useragent.js"
+    },
     "./interceptors/strict-transport": {
       "default": "./interceptors/strict-transport.js"
     },
@@ -164,5 +167,5 @@
     "status": "alpha",
     "year": 2024
   },
-  "gitHead": "55987d581e4985b1c3091ba6be3f9b53a0c5eeea\n"
+  "gitHead": "c508e27f87e59a812bbbb838317f8f710bd77c0c\n"
 }

package/server.d.ts

@@ -10,6 +10,7 @@ export declare class Server<CTX extends RequestCtx = RequestCtx> {
     server: http.Server<typeof http.IncomingMessage, typeof ServerResponse>;
     host: string;
     protected augmentCtx: Fn<RequestCtx, CTX>;
+    protected methodAdapter: ServerOpts["method"];
     constructor(opts?: Partial<ServerOpts<CTX>>);
     start(): Promise<boolean>;
     stop(): Promise<boolean>;
@@ -26,15 +27,86 @@ export declare const server: <CTX extends RequestCtx>(opts?: Partial<ServerOpts<
  * for various commonly used HTTP statuses/errors.
  */
 export declare class ServerResponse extends http.ServerResponse<http.IncomingMessage> {
+    /**
+     * Writes a HTTP 204 header (plus given `headers`) and ends the response.
+     *
+     * @param headers
+     */
     noContent(headers?: http.OutgoingHttpHeaders): void;
+    /**
+     * Writes a HTTP 302 header to redirect to given URL, add given additional
+     * `headers` and ends the response.
+     *
+     * @remarks
+     * Also see {@link ServerResponse.seeOther}.
+     *
+     * @param headers
+     */
     redirectTo(location: string, headers?: http.OutgoingHttpHeaders): void;
+    /**
+     * Writes a HTTP 303 header to redirect to given URL, add given additional
+     * `headers` and ends the response.
+     *
+     * @remarks
+     * Also see {@link ServerResponse.redirectTo}.
+     *
+     * @param headers
+     */
     seeOther(location: string, headers?: http.OutgoingHttpHeaders): void;
+    /**
+     * Writes a HTTP 304 header (plus given `headers`) and ends the response.
+     *
+     * @param headers
+     */
     unmodified(headers?: http.OutgoingHttpHeaders): void;
+    /**
+     * Writes a HTTP 400 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
+    badRequest(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 401 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     unauthorized(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 403 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     forbidden(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 404 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     missing(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 405 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     notAllowed(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 406 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     notAcceptable(headers?: http.OutgoingHttpHeaders, body?: any): void;
+    /**
+     * Writes a HTTP 429 header (plus given `headers`) and ends the response
+     * (with optional `body`).
+     *
+     * @param headers
+     */
     rateLimit(headers?: http.OutgoingHttpHeaders, body?: any): void;
     /**
      * HTTP 444. Indicates the server has returned no information to the client and closed

package/server.js

@@ -20,6 +20,7 @@ class Server {
     this.opts = opts;
     this.logger = opts.logger ?? new ConsoleLogger("server");
     this.host = opts.host ?? "localhost";
+    this.methodAdapter = opts.method ?? ((method, { route }) => method === "head" && !route.handlers.head && route.handlers.get ? (console.log("adapted head"), "get") : method);
     if (isIPv6(this.host)) this.host = normalizeIPv6Address(this.host);
     this.augmentCtx = opts.context ?? identity;
     const routes = [
@@ -44,6 +45,7 @@ class Server {
   server;
   host;
   augmentCtx;
+  methodAdapter;
   async start() {
     const {
       ssl,
@@ -101,15 +103,22 @@ class Server {
       const match = this.router.route(path);
       if (match.id === MISSING) return res.missing();
       const route = this.router.routeForID(match.id).spec;
-      let method = req.method.toLowerCase();
+      const rawCookies = req.headers["cookie"] || req.headers["set-cookie"]?.join(";");
+      const cookies = rawCookies ? parseCoookies(rawCookies) : {};
+      const query = parseSearchParams(url.searchParams);
+      const origMethod = req.method.toLowerCase();
+      const method = this.methodAdapter(origMethod, {
+        req,
+        route,
+        match,
+        query,
+        cookies
+      });
       if (method === "options" && !route.handlers.options) {
         return res.noContent({
           allow: Object.keys(route.handlers).map(upper).join(", ")
         });
       }
-      const rawCookies = req.headers["cookie"] || req.headers["set-cookie"]?.join(";");
-      const cookies = rawCookies ? parseCoookies(rawCookies) : {};
-      const query = parseSearchParams(url.searchParams);
       const ctx = this.augmentCtx({
         // @ts-ignore
         server: this,
@@ -120,11 +129,10 @@ class Server {
         query,
         cookies,
         route,
-        match
+        match,
+        method,
+        origMethod
       });
-      if (method === "head" && !route.handlers.head && route.handlers.get) {
-        method = "get";
-      }
       const handler = route.handlers[method];
       if (handler) {
         this.runHandler(handler, ctx);
@@ -232,33 +240,106 @@ class Server {
 }
 const server = (opts) => new Server(opts);
 class ServerResponse extends http.ServerResponse {
+  /**
+   * Writes a HTTP 204 header (plus given `headers`) and ends the response.
+   *
+   * @param headers
+   */
   noContent(headers) {
     this.writeHead(204, headers).end();
   }
+  /**
+   * Writes a HTTP 302 header to redirect to given URL, add given additional
+   * `headers` and ends the response.
+   *
+   * @remarks
+   * Also see {@link ServerResponse.seeOther}.
+   *
+   * @param headers
+   */
   redirectTo(location, headers) {
     this.writeHead(302, { ...headers, location }).end();
   }
+  /**
+   * Writes a HTTP 303 header to redirect to given URL, add given additional
+   * `headers` and ends the response.
+   *
+   * @remarks
+   * Also see {@link ServerResponse.redirectTo}.
+   *
+   * @param headers
+   */
   seeOther(location, headers) {
     this.writeHead(303, { ...headers, location }).end();
   }
+  /**
+   * Writes a HTTP 304 header (plus given `headers`) and ends the response.
+   *
+   * @param headers
+   */
   unmodified(headers) {
     this.writeHead(304, headers).end();
   }
+  /**
+   * Writes a HTTP 400 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
+  badRequest(headers, body) {
+    this.writeHead(400, headers).end(body);
+  }
+  /**
+   * Writes a HTTP 401 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   unauthorized(headers, body) {
     this.writeHead(401, headers).end(body);
   }
+  /**
+   * Writes a HTTP 403 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   forbidden(headers, body) {
     this.writeHead(403, headers).end(body);
   }
+  /**
+   * Writes a HTTP 404 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   missing(headers, body) {
     this.writeHead(404, headers).end(body);
   }
+  /**
+   * Writes a HTTP 405 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   notAllowed(headers, body) {
     this.writeHead(405, headers).end(body);
   }
+  /**
+   * Writes a HTTP 406 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   notAcceptable(headers, body) {
     this.writeHead(406, headers).end(body);
   }
+  /**
+   * Writes a HTTP 429 header (plus given `headers`) and ends the response
+   * (with optional `body`).
+   *
+   * @param headers
+   */
   rateLimit(headers, body) {
     this.writeHead(429, headers).end(body);
   }