@naturalcycles/backend-lib 9.75.0 → 9.76.0

package/dist/server/basicAuthMiddleware.js

@@ -14,6 +14,7 @@ export function basicAuthMiddleware(cfg) {
         res
             .set('WWW-Authenticate', `Basic${realm ? ` realm="${realm}"` : ''}`)
             .status(401)
+            .type('text/plain')
             .send('Unauthorized');
     };
 }

package/dist/server/request.util.d.ts

@@ -9,5 +9,19 @@ import type { BackendRequest } from './server.model.js';
  *
  * If stripPrefix (e.g `/api/v2`) is provided, and the path starts with it (like path.startsWith(stripPrefix)),
  * it will be stripped from the beginning of the path.
+ *
+ * The result is injection-safe: `req.baseUrl` and `req.path` come from the request as-is
+ * (not even url-decoded), so every character outside of a conservative allowlist is replaced with `_`.
+ * `req.route.path` is the server-defined route pattern, so it's used as-is.
+ *
+ * Use `getRequestEndpointUnsafe` if the exact path is needed.
  */
 export declare function getRequestEndpoint(req: BackendRequest, stripPrefix?: string): string;
+/**
+ * Same as `getRequestEndpoint`, but without sanitization:
+ * for requests that didn't match a route (e.g 404s) the path is returned exactly as the client sent it.
+ *
+ * Use at your own risk: the result must be escaped before it's embedded in html
+ * (or any other context where it could be interpreted).
+ */
+export declare function getRequestEndpointUnsafe(req: BackendRequest, stripPrefix?: string): string;

package/dist/server/request.util.js

@@ -8,9 +8,28 @@
  *
  * If stripPrefix (e.g `/api/v2`) is provided, and the path starts with it (like path.startsWith(stripPrefix)),
  * it will be stripped from the beginning of the path.
+ *
+ * The result is injection-safe: `req.baseUrl` and `req.path` come from the request as-is
+ * (not even url-decoded), so every character outside of a conservative allowlist is replaced with `_`.
+ * `req.route.path` is the server-defined route pattern, so it's used as-is.
+ *
+ * Use `getRequestEndpointUnsafe` if the exact path is needed.
  */
 export function getRequestEndpoint(req, stripPrefix) {
-    let path = (req.baseUrl + (req.route?.path || req.path)).toLowerCase();
+    return buildRequestEndpoint(req, stripPrefix, sanitizePath);
+}
+/**
+ * Same as `getRequestEndpoint`, but without sanitization:
+ * for requests that didn't match a route (e.g 404s) the path is returned exactly as the client sent it.
+ *
+ * Use at your own risk: the result must be escaped before it's embedded in html
+ * (or any other context where it could be interpreted).
+ */
+export function getRequestEndpointUnsafe(req, stripPrefix) {
+    return buildRequestEndpoint(req, stripPrefix, identity);
+}
+function buildRequestEndpoint(req, stripPrefix, sanitize) {
+    let path = (sanitize(req.baseUrl) + (req.route?.path || sanitize(req.path))).toLowerCase();
     if (path.length > 1 && path.endsWith('/')) {
         path = path.slice(0, path.length - 1);
     }
@@ -19,3 +38,11 @@ export function getRequestEndpoint(req, stripPrefix) {
     }
     return [req.method, path].join(' ');
 }
+// Letters, digits and the url punctuation that is inert in html, js, json and logs.
+// Everything else (< > & " ' ( ) ` \ control chars, etc) is replaced, not encoded:
+// the exact bytes are not worth preserving here, `req.url` still has them.
+const UNSAFE_PATH_CHAR = /[^\w\-./:@%!$*+,;=~]/gu;
+function sanitizePath(path) {
+    return path.replace(UNSAFE_PATH_CHAR, '_');
+}
+const identity = (path) => path;

package/dist/server/requestTimeoutMiddleware.d.ts

@@ -13,6 +13,13 @@ export interface RequestTimeoutMiddlewareCfg {
      * @default 'Request timed out'
      */
     httpErrorMessage?: string;
+    /**
+     * Upper bound for the `?requestTimeout=N` query override.
+     * Higher values are clamped to it, so a client cannot hold a request open for arbitrarily long.
+     *
+     * @default 600
+     */
+    maxTimeoutSeconds?: NumberOfSeconds;
 }
 export declare function requestTimeoutMiddleware(cfg?: RequestTimeoutMiddlewareCfg): BackendRequestHandler;
 export interface CustomRequestTimeoutMiddlewareCfg {
@@ -25,7 +32,7 @@ export interface CustomRequestTimeoutMiddlewareCfg {
  * Example:
  *
  * router.get('/', customRequestTimeoutMiddleware(
- *   (req, res) => res.status(409).send('my custom message!'),
+ *   (req, res) => res.status(409).type('text/plain').send('my custom message!'),
  *   { timeoutSeconds: 30 },
  * )
  */

package/dist/server/requestTimeoutMiddleware.js

@@ -6,7 +6,7 @@ import { getRequestEndpoint } from './request.util.js';
 const code = 'REQUEST_TIMEOUT';
 const REQUEST_TIMEOUT_QUERY_KEY = 'requestTimeout';
 export function requestTimeoutMiddleware(cfg = {}) {
-    const { timeoutSeconds: defTimeoutSeconds, backendResponseStatusCode, httpErrorMessage, } = {
+    const { timeoutSeconds: defTimeoutSeconds, backendResponseStatusCode, httpErrorMessage, maxTimeoutSeconds, } = {
         // Considerations about the default value of the timeout.
         // Ideally the default value here would be HIGHER than the default timeout for getGot (in nodejs-lib),
         // so, cross-service communication has a chance to fail SOONER than server times out,
@@ -14,12 +14,11 @@ export function requestTimeoutMiddleware(cfg = {}) {
         timeoutSeconds: 120,
         backendResponseStatusCode: 503,
         httpErrorMessage: 'Request timed out',
+        maxTimeoutSeconds: 600,
         ...cfg,
     };
     return function requestTimeoutHandler(req, res, next) {
-        const timeoutSeconds = req.query[REQUEST_TIMEOUT_QUERY_KEY]
-            ? Number.parseInt(req.query[REQUEST_TIMEOUT_QUERY_KEY])
-            : defTimeoutSeconds;
+        const timeoutSeconds = getTimeoutSeconds(req, defTimeoutSeconds, maxTimeoutSeconds);
         // If requestTimeout was previously set - cancel it first
         // Then set the new requestTimeout and handler
         if (req.requestTimeout)
@@ -39,11 +38,26 @@ export function requestTimeoutMiddleware(cfg = {}) {
         next();
     };
 }
+/**
+ * `?requestTimeout=N` query param lets the client override the timeout of its request
+ * (used e.g by Cloud Tasks for long-running tasks).
+ * Being client-controlled, it is validated: non-numeric or non-positive values are ignored
+ * (fall back to the default), values above `maxTimeoutSeconds` are clamped.
+ */
+function getTimeoutSeconds(req, defTimeoutSeconds, maxTimeoutSeconds) {
+    const raw = req.query[REQUEST_TIMEOUT_QUERY_KEY];
+    if (typeof raw !== 'string')
+        return defTimeoutSeconds;
+    const requested = Number.parseInt(raw, 10);
+    if (Number.isNaN(requested) || requested <= 0)
+        return defTimeoutSeconds;
+    return Math.min(requested, maxTimeoutSeconds);
+}
 /**
  * Example:
  *
  * router.get('/', customRequestTimeoutMiddleware(
- *   (req, res) => res.status(409).send('my custom message!'),
+ *   (req, res) => res.status(409).type('text/plain').send('my custom message!'),
  *   { timeoutSeconds: 30 },
  * )
  */

package/dist/server/serverStatsMiddleware.js

@@ -5,6 +5,7 @@ import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _percentile } from '@naturalcycles/js-lib/math/math.util.js';
 import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js';
 import { _get, _mapValues } from '@naturalcycles/js-lib/object/object.util.js';
+import { htmlEscape } from '@naturalcycles/js-lib/string/escape.js';
 import { _stringMapEntries, _stringMapValues } from '@naturalcycles/js-lib/types';
 import { onFinished } from '../onFinished.js';
 import { getRequestEndpoint } from './request.util.js';
@@ -61,7 +62,7 @@ export const serverStatsHTMLHandler = (req, res) => {
         }).map(([endpoint, stat]) => {
             return [
                 '<tr>',
-                `<td><pre>${endpoint}</pre></td>`,
+                `<td><pre>${htmlEscape(endpoint)}</pre></td>`,
                 `<td align="right"><pre>${stat.total}</pre></td>`,
                 // oxlint-disable-next-line @typescript-eslint/no-base-to-string
                 ...families.map(f => `<td align="right"><pre>${stat[f]}</pre></td>`),
@@ -71,7 +72,7 @@ export const serverStatsHTMLHandler = (req, res) => {
         }),
         '</table>',
     ].join('\n');
-    res.send(html);
+    res.type('html').send(html);
 };
 /**
  * Unlocks serverStatsHTMLHandler
@@ -83,7 +84,12 @@ export function serverStatsMiddleware() {
         onFinished(res, () => {
             const now = Date.now();
             const latency = now - started;
-            const endpoint = getRequestEndpoint(req);
+            // Only requests that matched a route are recorded by their route pattern.
+            // Unmatched requests (404s, requests rejected by router-level middleware) carry an
+            // arbitrary client-controlled path: recording them per-path would let any client
+            // grow the map unboundedly and plant arbitrary strings into it.
+            // So they're bucketed per mount point instead.
+            const endpoint = req.route ? getRequestEndpoint(req) : getUnmatchedEndpoint(req);
             serverStatsMap[endpoint] ||= {
                 stack: new NumberStack(SIZE),
                 '2xx': 0,
@@ -102,6 +108,13 @@ export function serverStatsMiddleware() {
         next();
     };
 }
+/**
+ * Returns e.g `GET /*` or `GET /api/v3/admin/*`.
+ * `req.baseUrl` is the mount point that matched, so it's bounded by the app's routing config.
+ */
+function getUnmatchedEndpoint(req) {
+    return `${req.method} ${req.baseUrl.toLowerCase()}/*`;
+}
 function cleanupServerStats() {
     if (Object.keys(serverStatsMap).length <= MAX_ENDPOINTS)
         return;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/backend-lib",
   "type": "module",
-  "version": "9.75.0",
+  "version": "9.76.0",
   "dependencies": {
     "@naturalcycles/db-lib": "^10",
     "@naturalcycles/js-lib": "^15",

package/src/admin/login.html

@@ -63,7 +63,8 @@
       })
 
       const qs = Object.fromEntries(new URLSearchParams(location.search))
-      const { autoLogin, logout, returnUrl, adminTokenKey = 'admin_token' } = qs
+      const { autoLogin, logout, adminTokenKey = 'admin_token' } = qs
+      const returnUrl = toSafeReturnUrl(qs.returnUrl)
       // console.log(qs)
 
       const app = createApp({
@@ -137,6 +138,20 @@
         alert('Error\n ' + JSON.stringify(err, null, 2))
       }
 
+      /**
+       * `returnUrl` is client-controlled. Only http(s) urls are allowed:
+       * anything else (e.g `javascript:`) would otherwise be executed by `location.href = returnUrl`
+       * in the context of this page, where the token is exposed.
+       */
+      function toSafeReturnUrl(url) {
+        if (!url) return
+        try {
+          const { protocol } = new URL(url, location.href)
+          if (protocol === 'http:' || protocol === 'https:') return url
+        } catch {}
+        console.warn('ignoring invalid returnUrl', url)
+      }
+
       async function postToken(token) {
         await fetch(`/admin/login`, {
           method: 'post',

package/src/server/basicAuthMiddleware.ts

@@ -31,6 +31,7 @@ export function basicAuthMiddleware(cfg: BasicAuthMiddlewareCfg): BackendRequest
     res
       .set('WWW-Authenticate', `Basic${realm ? ` realm="${realm}"` : ''}`)
       .status(401)
+      .type('text/plain')
       .send('Unauthorized')
   }
 }

package/src/server/request.util.ts

@@ -10,9 +10,34 @@ import type { BackendRequest } from './server.model.js'
  *
  * If stripPrefix (e.g `/api/v2`) is provided, and the path starts with it (like path.startsWith(stripPrefix)),
  * it will be stripped from the beginning of the path.
+ *
+ * The result is injection-safe: `req.baseUrl` and `req.path` come from the request as-is
+ * (not even url-decoded), so every character outside of a conservative allowlist is replaced with `_`.
+ * `req.route.path` is the server-defined route pattern, so it's used as-is.
+ *
+ * Use `getRequestEndpointUnsafe` if the exact path is needed.
  */
 export function getRequestEndpoint(req: BackendRequest, stripPrefix?: string): string {
-  let path = (req.baseUrl + (req.route?.path || req.path)).toLowerCase()
+  return buildRequestEndpoint(req, stripPrefix, sanitizePath)
+}
+
+/**
+ * Same as `getRequestEndpoint`, but without sanitization:
+ * for requests that didn't match a route (e.g 404s) the path is returned exactly as the client sent it.
+ *
+ * Use at your own risk: the result must be escaped before it's embedded in html
+ * (or any other context where it could be interpreted).
+ */
+export function getRequestEndpointUnsafe(req: BackendRequest, stripPrefix?: string): string {
+  return buildRequestEndpoint(req, stripPrefix, identity)
+}
+
+function buildRequestEndpoint(
+  req: BackendRequest,
+  stripPrefix: string | undefined,
+  sanitize: (path: string) => string,
+): string {
+  let path = (sanitize(req.baseUrl) + (req.route?.path || sanitize(req.path))).toLowerCase()
   if (path.length > 1 && path.endsWith('/')) {
     path = path.slice(0, path.length - 1)
   }
@@ -23,3 +48,14 @@ export function getRequestEndpoint(req: BackendRequest, stripPrefix?: string): s
 
   return [req.method, path].join(' ')
 }
+
+// Letters, digits and the url punctuation that is inert in html, js, json and logs.
+// Everything else (< > & " ' ( ) ` \ control chars, etc) is replaced, not encoded:
+// the exact bytes are not worth preserving here, `req.url` still has them.
+const UNSAFE_PATH_CHAR = /[^\w\-./:@%!$*+,;=~]/gu
+
+function sanitizePath(path: string): string {
+  return path.replace(UNSAFE_PATH_CHAR, '_')
+}
+
+const identity = (path: string): string => path

package/src/server/requestTimeoutMiddleware.ts

@@ -21,6 +21,14 @@ export interface RequestTimeoutMiddlewareCfg {
    * @default 'Request timed out'
    */
   httpErrorMessage?: string
+
+  /**
+   * Upper bound for the `?requestTimeout=N` query override.
+   * Higher values are clamped to it, so a client cannot hold a request open for arbitrarily long.
+   *
+   * @default 600
+   */
+  maxTimeoutSeconds?: NumberOfSeconds
 }
 
 const code = 'REQUEST_TIMEOUT'
@@ -33,6 +41,7 @@ export function requestTimeoutMiddleware(
     timeoutSeconds: defTimeoutSeconds,
     backendResponseStatusCode,
     httpErrorMessage,
+    maxTimeoutSeconds,
   } = {
     // Considerations about the default value of the timeout.
     // Ideally the default value here would be HIGHER than the default timeout for getGot (in nodejs-lib),
@@ -41,13 +50,12 @@ export function requestTimeoutMiddleware(
     timeoutSeconds: 120,
     backendResponseStatusCode: 503,
     httpErrorMessage: 'Request timed out',
+    maxTimeoutSeconds: 600,
     ...cfg,
   }
 
   return function requestTimeoutHandler(req, res, next) {
-    const timeoutSeconds = req.query[REQUEST_TIMEOUT_QUERY_KEY]
-      ? Number.parseInt(req.query[REQUEST_TIMEOUT_QUERY_KEY] as string)
-      : defTimeoutSeconds
+    const timeoutSeconds = getTimeoutSeconds(req, defTimeoutSeconds, maxTimeoutSeconds)
 
     // If requestTimeout was previously set - cancel it first
     // Then set the new requestTimeout and handler
@@ -76,6 +84,26 @@ export function requestTimeoutMiddleware(
   }
 }
 
+/**
+ * `?requestTimeout=N` query param lets the client override the timeout of its request
+ * (used e.g by Cloud Tasks for long-running tasks).
+ * Being client-controlled, it is validated: non-numeric or non-positive values are ignored
+ * (fall back to the default), values above `maxTimeoutSeconds` are clamped.
+ */
+function getTimeoutSeconds(
+  req: BackendRequest,
+  defTimeoutSeconds: NumberOfSeconds,
+  maxTimeoutSeconds: NumberOfSeconds,
+): NumberOfSeconds {
+  const raw = req.query[REQUEST_TIMEOUT_QUERY_KEY]
+  if (typeof raw !== 'string') return defTimeoutSeconds
+
+  const requested = Number.parseInt(raw, 10)
+  if (Number.isNaN(requested) || requested <= 0) return defTimeoutSeconds
+
+  return Math.min(requested, maxTimeoutSeconds)
+}
+
 export interface CustomRequestTimeoutMiddlewareCfg {
   /**
    * @default 120
@@ -87,7 +115,7 @@ export interface CustomRequestTimeoutMiddlewareCfg {
  * Example:
  *
  * router.get('/', customRequestTimeoutMiddleware(
- *   (req, res) => res.status(409).send('my custom message!'),
+ *   (req, res) => res.status(409).type('text/plain').send('my custom message!'),
  *   { timeoutSeconds: 30 },
  * )
  */

package/src/server/serverStatsMiddleware.ts

@@ -5,11 +5,12 @@ import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { _percentile } from '@naturalcycles/js-lib/math/math.util.js'
 import { NumberStack } from '@naturalcycles/js-lib/math/stack.util.js'
 import { _get, _mapValues } from '@naturalcycles/js-lib/object/object.util.js'
+import { htmlEscape } from '@naturalcycles/js-lib/string/escape.js'
 import { _stringMapEntries, _stringMapValues } from '@naturalcycles/js-lib/types'
 import type { StringMap } from '@naturalcycles/js-lib/types'
 import { onFinished } from '../onFinished.js'
 import { getRequestEndpoint } from './request.util.js'
-import type { BackendRequestHandler } from './server.model.js'
+import type { BackendRequest, BackendRequestHandler } from './server.model.js'
 
 const { GAE_INSTANCE } = process.env
 
@@ -86,7 +87,7 @@ export const serverStatsHTMLHandler: BackendRequestHandler = (req, res) => {
     }).map(([endpoint, stat]) => {
       return [
         '<tr>',
-        `<td><pre>${endpoint}</pre></td>`,
+        `<td><pre>${htmlEscape(endpoint)}</pre></td>`,
         `<td align="right"><pre>${stat.total}</pre></td>`,
         // oxlint-disable-next-line @typescript-eslint/no-base-to-string
         ...families.map(f => `<td align="right"><pre>${stat[f]}</pre></td>`),
@@ -97,7 +98,7 @@ export const serverStatsHTMLHandler: BackendRequestHandler = (req, res) => {
     '</table>',
   ].join('\n')
 
-  res.send(html)
+  res.type('html').send(html)
 }
 
 /**
@@ -113,7 +114,12 @@ export function serverStatsMiddleware(): BackendRequestHandler {
       const now = Date.now()
       const latency = now - started
 
-      const endpoint = getRequestEndpoint(req)
+      // Only requests that matched a route are recorded by their route pattern.
+      // Unmatched requests (404s, requests rejected by router-level middleware) carry an
+      // arbitrary client-controlled path: recording them per-path would let any client
+      // grow the map unboundedly and plant arbitrary strings into it.
+      // So they're bucketed per mount point instead.
+      const endpoint = req.route ? getRequestEndpoint(req) : getUnmatchedEndpoint(req)
 
       serverStatsMap[endpoint] ||= {
         stack: new NumberStack(SIZE),
@@ -137,6 +143,14 @@ export function serverStatsMiddleware(): BackendRequestHandler {
   }
 }
 
+/**
+ * Returns e.g `GET /*` or `GET /api/v3/admin/*`.
+ * `req.baseUrl` is the mount point that matched, so it's bounded by the app's routing config.
+ */
+function getUnmatchedEndpoint(req: BackendRequest): string {
+  return `${req.method} ${req.baseUrl.toLowerCase()}/*`
+}
+
 function cleanupServerStats(): void {
   if (Object.keys(serverStatsMap).length <= MAX_ENDPOINTS) return