@harshankur/viewcounter 3.4.0 → 3.5.0

package/.env.example

@@ -127,6 +127,17 @@ RATE_LIMIT_MAX=100
 # Set 0 to disable (sensible for a single-tenant deployment).
 APP_RATE_LIMIT_MAX=1000
 
+# An app's own figures, where the general ones above do not fit it: a list of
+# appId:number. A site that records far more per visitor than the others (every
+# step inside a single-page app, say) gets the room it needs without loosening
+# the limits for every other app. An app listed in RATE_LIMIT_MAX_BY_APP is
+# counted on its own per address, so its traffic does not use up the address's
+# budget for other apps. Zero means no limit, here as in the general figures
+# above. A malformed entry, or an app listed twice, stops the server from
+# starting; a figure for an app that is not configured is warned about.
+#RATE_LIMIT_MAX_BY_APP=homepage:600
+#APP_RATE_LIMIT_MAX_BY_APP=homepage:5000
+
 # How long a visitor counts as "the same visitor" for deduplication, in hours.
 # Also the rotation period for the visitor hash: after this window the same
 # person hashes differently, so their visits cannot be linked across windows.

package/README.md

@@ -3,7 +3,7 @@
 [![Documentation](https://img.shields.io/badge/docs-viewcounter.harshankur.com-blueviolet)](https://viewcounter.harshankur.com)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![Test Suite](https://github.com/harshankur/viewcounter/actions/workflows/test.yml/badge.svg)](https://github.com/harshankur/viewcounter/actions/workflows/test.yml)
-[![Tests](https://img.shields.io/badge/tests-1134%20passing-success)](TEST_REPORT.md)
+[![Tests](https://img.shields.io/badge/tests-1164%20passing-success)](TEST_REPORT.md)
 [![npm](https://img.shields.io/npm/v/@harshankur/viewcounter?logo=npm)](https://www.npmjs.com/package/@harshankur/viewcounter)
 [![provenance](https://img.shields.io/badge/provenance-signed-brightgreen?logo=github)](https://www.npmjs.com/package/@harshankur/viewcounter#provenance)
 
@@ -190,7 +190,7 @@ than running on a guessable default:
 - `ADMIN_PASSWORD`: turns on the [admin UI](#admin-ui) at `/admin`. At least 16 characters. Unset means the admin UI does not exist.
 
 **Optional**: `DB_MODE`, `PORT`, `LOG_LEVEL`, `RATE_LIMIT_WINDOW_MS`,
-`RATE_LIMIT_MAX`, `UNIQUE_VISITOR_WINDOW_HOURS`, `ALLOWED_DEVICE_SIZES`,
+`RATE_LIMIT_MAX`, `RATE_LIMIT_MAX_BY_APP`, `APP_RATE_LIMIT_MAX_BY_APP`, `UNIQUE_VISITOR_WINDOW_HOURS`, `ALLOWED_DEVICE_SIZES`,
 `TRASH_RETENTION_DAYS`, `VIEW_LOG_RETENTION_DAYS`, `ADMIN_SESSION_IDLE_TIMEOUT`,
 `ADMIN_SESSION_MAX_AGE`, `GEOIP_CITY_DB`.
 
@@ -867,6 +867,26 @@ Two independent limits apply to writes:
   everyone else on the instance depends on. Keyed on `appId` alone, so it cannot
   be bypassed by rotating addresses. Set `0` to disable for single-tenant use.
 
+Either can be set for one app where the general figure does not fit it, as a
+list of `appId:number`:
+
+- `RATE_LIMIT_MAX_BY_APP=homepage:600`: tracking requests a minute from one
+  address to that app. Such an app is counted on its own, so its traffic does
+  not use up the address's budget for your other apps, nor the other way round.
+- `APP_RATE_LIMIT_MAX_BY_APP=homepage:5000`: that app's whole budget.
+
+Use these for a site that records far more per visitor than the others, such
+as a single-page app that counts every step inside it.
+
+- **Zero means no limit**, in all four settings: as a general figure it
+  switches that limit off, as an app's own figure it lifts it for that app.
+- **Only tracking requests** (`/registerView`, `/event`, `/engage`) are limited
+  as an app, and always as the app they are stored under. A read or admin call
+  is limited by the general figure whatever it names.
+- **Mistakes are not silent.** A malformed entry, or an app listed twice, stops
+  the server from starting. A figure for an app that is not configured is
+  logged as a warning at start, since it would otherwise never apply.
+
 Each limit is applied twice, as two separate budgets of that size: one for
 engagement reports (`/engage`) and one for everything else. A page being read
 reports every half minute, so each open, active tab costs two reports a minute;
@@ -930,7 +950,10 @@ app.use('/analytics', createAnalyticsRouter({
     privacy: { visitorSecret: process.env.VISITOR_SECRET },
     server: {
       uniqueVisitorWindowHours: 24,
-      // omit to disable the per-app write budget
+      // perAppMax: the per-app write budget (omit to disable it).
+      // max: per address, applied by the router to engagement reports only;
+      //   every other request is yours to limit, as below.
+      // perAppMaxByApp / maxByApp: { appId: number } for apps with their own figures.
       rateLimit: { windowMs: 60000, perAppMax: 1000 },
     },
   },
@@ -940,8 +963,10 @@ app.use('/analytics', createAnalyticsRouter({
 Endpoints then live under the prefix: `POST /analytics/event`,
 `GET /analytics/stats/blog`, and so on.
 
-Two things the host application owns in this mode, because the router does not
-install them itself: `helmet()` and the CORS allowlist, and `trust proxy`. Set
+Three things the host application owns in this mode, because the router does
+not install them itself: `helmet()` and the CORS allowlist, a per-address rate
+limit (the router limits only engagement reports per address, and only when
+`rateLimit.max` is given), and `trust proxy`. Set
 `app.set('trust proxy', <hop count>)`, never `true`, or callers can forge
 their own IP through `X-Forwarded-For`.
 

package/config/index.js

@@ -12,7 +12,7 @@ const {
     SCOPE_ALL,
     SERVER,
 } = require('../constants');
-const { filterValidAppIds } = require('../utils/appIdUtils');
+const { filterValidAppIds, isValidAppId } = require('../utils/appIdUtils');
 const { parseDuration, formatDuration } = require('../utils/durationUtils');
 const { getError, logWarning, ErrorType, WarningType } = require('../utils/errorUtils');
 const { LogLevel } = require('../utils/logger');
@@ -47,6 +47,30 @@ function parseList(raw) {
         .filter(Boolean);
 }
 
+/**
+ * Parse per-app figures, `appId:number` separated by commas
+ * (`homepage:600,shop:0`), into a map. A malformed entry is an error: a
+ * limit silently not applied is worse than a refusal to start.
+ * @param {string|undefined} raw
+ * @param {string} field the setting's name, for the error
+ * @returns {Record<string, number>}
+ */
+function parseAppNumbers(raw, field) {
+    const figures = {};
+    for (const entry of parseList(raw)) {
+        const [appId, value, ...rest] = entry.split(':').map((part) => part.trim());
+        if (rest.length || !isValidAppId(appId) || !/^\d+$/.test(value || '')) {
+            throw getError(ErrorType.CONFIG_INVALID_VALUE, { field, reason: `'${entry}' is not appId:number, such as homepage:600` });
+        }
+        // Two figures for one app is a mistake, and guessing which was meant is not ours to do.
+        if (Object.hasOwn(figures, appId)) {
+            throw getError(ErrorType.CONFIG_INVALID_VALUE, { field, reason: `'${appId}' is listed twice` });
+        }
+        figures[appId] = Number(value);
+    }
+    return figures;
+}
+
 /** Parse an integer env var, falling back when absent or unparseable. */
 function parseIntOr(raw, fallback) {
     const parsed = Number.parseInt(raw, 10);
@@ -130,6 +154,9 @@ class Config {
                 // Per-app ceiling on writes, so one tenant cannot exhaust the
                 // budget the others depend on.
                 perAppMax: parseIntOr(this.env.APP_RATE_LIMIT_MAX, SERVER.DEFAULT_APP_RATE_LIMIT_MAX),
+                // An app's own figures, where the general ones do not fit it.
+                maxByApp: parseAppNumbers(this.env.RATE_LIMIT_MAX_BY_APP, 'RATE_LIMIT_MAX_BY_APP'),
+                perAppMaxByApp: parseAppNumbers(this.env.APP_RATE_LIMIT_MAX_BY_APP, 'APP_RATE_LIMIT_MAX_BY_APP'),
             },
             uniqueVisitorWindowHours: parseIntOr(
                 this.env.UNIQUE_VISITOR_WINDOW_HOURS,
@@ -391,6 +418,7 @@ class Config {
         void this.privacy.visitorSecret;
 
         this.validateAdmin();
+        this.warnAboutUnknownRateLimitApps();
 
         if (!this.server.isProduction) {
             this.warnAboutDevelopmentDefaults();
@@ -457,6 +485,21 @@ class Config {
     }
 
     /** Surface the same problems as warnings outside production. */
+    /**
+     * A per-app rate limit figure for an app this configuration does not
+     * list is most likely a typo, and would otherwise never apply without a
+     * word. It is a warning, not an error: an app may also be registered
+     * while the server runs.
+     */
+    warnAboutUnknownRateLimitApps() {
+        const { maxByApp = {}, perAppMaxByApp = {} } = this.server.rateLimit;
+        for (const [field, figures] of [['RATE_LIMIT_MAX_BY_APP', maxByApp], ['APP_RATE_LIMIT_MAX_BY_APP', perAppMaxByApp]]) {
+            for (const appId of Object.keys(figures)) {
+                if (!this.allowed.appId.includes(appId)) logWarning(WarningType.RATE_LIMIT_UNKNOWN_APP, { field, appId });
+            }
+        }
+    }
+
     warnAboutDevelopmentDefaults() {
         if (Object.keys(this.auth.readKeyScopes).length === 0) {
             logWarning(WarningType.READ_API_UNPROTECTED);

package/index.js

@@ -7,7 +7,7 @@ const config = require('./config');
 const DatabaseManager = require('./db/DatabaseManager');
 const logger = require('./utils/logger');
 const { buildCorsOptions, countRefusedPreflights } = require('./middleware/security');
-const { createAnalyticsRouter, buildPerIpLimiters, trackingSourceFor } = require('./routes/analytics');
+const { createAnalyticsRouter, buildPerIpLimiter, trackingSourceFor } = require('./routes/analytics');
 const { createAdminRouter } = require('./routes/admin');
 const { startRetention } = require('./db/retention');
 const { createDbSessionStore } = require('./db/adminSessionStore');
@@ -80,7 +80,8 @@ function createApp() {
 
     // A tracking request turned away here is counted in the tracking log like
     // any other refusal (in memory, written in batches).
-    app.use(buildPerIpLimiters(config.server.rateLimit, (req) => {
+    // (Engagement reports have a limiter of their own, on their route.)
+    app.use(buildPerIpLimiter(config.server.rateLimit, (req) => {
         if (trackingSourceFor(req.path)) router.countRejection(req, REJECTION_REASON.RATE_LIMITED, { detail: 'ip' });
     }));
 

package/middleware/security.js

@@ -70,6 +70,23 @@ function requestOrigin(req) {
     }
 }
 
+/**
+ * The app a tracking request is for, read from the one place its route
+ * validates and stores: the query of a GET, the body of anything else.
+ *
+ * Everything that decides by app before validation (the origin check, the
+ * rate limits) must read it here. Reading "query, or else body" let a POST
+ * name one app in its query, be checked and limited as that app, and then be
+ * stored under the other app named in its body.
+ *
+ * @param {import('express').Request} req
+ * @returns {string} the appId, or '' when absent or not a string
+ */
+function requestedAppId(req) {
+    const appId = req.method === 'GET' ? req.query?.appId : req.body?.appId;
+    return typeof appId === 'string' ? appId : '';
+}
+
 /**
  * Bind writes for an appId to the site origins registered for it.
  *
@@ -92,8 +109,8 @@ function requireRegisteredOrigin(allowed, { onReject = () => {} } = {}) {
     const origins = allowed?.origins || {};
 
     return (req, res, next) => {
-        const appId = req.query.appId || req.body?.appId;
-        const registered = origins[appId];
+        const appId = requestedAppId(req);
+        const registered = Object.hasOwn(origins, appId) ? origins[appId] : undefined;
 
         if (!Array.isArray(registered) || registered.length === 0) {
             return next();
@@ -128,6 +145,7 @@ module.exports = {
     buildCorsOptions,
     countRefusedPreflights,
     requireRegisteredOrigin,
+    requestedAppId,
     requestOrigin,
     noStore,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@harshankur/viewcounter",
   "description": "A middleware backend server that registers views to my db server when requested to register a view from my other projects.",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "main": "index.js",
   "engines": {
     "node": ">=24"

package/routes/analytics.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const path = require('path');
 const express = require('express');
 const rateLimit = require('express-rate-limit');
+const { ipKeyGenerator } = rateLimit;
 const geoip = require('geoip-country');
 
 const {
@@ -21,7 +22,7 @@ const PrivacyUtils = require('../utils/privacyUtils');
 const logger = require('../utils/logger');
 const { getClientIp, isValidIP, normalizeIp } = require('../utils/ipUtils');
 const { requireReadApiKey, requireAppScope, requireAdminApiKey, appsInScope } = require('../middleware/auth');
-const { requireRegisteredOrigin, requestOrigin, noStore } = require('../middleware/security');
+const { requireRegisteredOrigin, requestedAppId, requestOrigin, noStore } = require('../middleware/security');
 const { createRejectionCounter } = require('../db/rejectionCounter');
 const { hostnameOf, primaryLanguage, utmTags } = require('../utils/visitorContext');
 const {
@@ -132,21 +133,24 @@ function intQuery(req, name, fallback) {
  * @returns {import('express').RequestHandler}
  */
 function buildPerAppLimiter(rateLimitConfig, onLimit = () => {}) {
-    const { perAppMax, windowMs } = rateLimitConfig || {};
-    // Zero disables it, for single-tenant deployments where the per-IP limit
-    // is the only bound that means anything.
-    if (!perAppMax || perAppMax <= 0) return (req, res, next) => next();
+    const { perAppMax, windowMs, perAppMaxByApp = {} } = rateLimitConfig || {};
+    // An app's own figure wins over the general one; zero means no ceiling.
+    const limitOf = (req) => (Object.hasOwn(perAppMaxByApp, appOf(req)) ? perAppMaxByApp[appOf(req)] : perAppMax) || 0;
+    // Nothing set anywhere disables it, for single-tenant deployments where
+    // the per-IP limit is the only bound that means anything.
+    if (!(perAppMax > 0) && !Object.values(perAppMaxByApp).some((value) => value > 0)) return (req, res, next) => next();
 
     return rateLimit({
         windowMs,
-        limit: perAppMax,
+        limit: limitOf,
+        skip: (req) => limitOf(req) <= 0,
         standardHeaders: true,
         legacyHeaders: false,
         message: { message: 'This app has exceeded its request budget, please try again later.' },
         // A request with no appId lands in one shared bucket rather than
         // falling back to the IP, which would reintroduce the address-rotation
         // bypass this limiter exists to be immune to.
-        keyGenerator: (req) => String(req.query?.appId || req.body?.appId || '__unattributed__'),
+        keyGenerator: (req) => appOf(req) || '__unattributed__',
         validate: { keyGeneratorIpFallback: false },
         handler: (req, res, next, options) => {
             onLimit(req);
@@ -159,34 +163,71 @@ function buildPerAppLimiter(rateLimitConfig, onLimit = () => {}) {
 const isEngagement = (req) => trackingSourceFor(req.path) === VIEW_LOG_SOURCE.ENGAGE;
 
 /**
- * The per-IP limiters: one budget for engagement reports, one for everything
- * else, each of `max` requests per window.
+ * The app a request is limited as: for a tracking request, the app its route
+ * goes on to validate and store under; for anything else, none. A read or
+ * admin call cannot name an app to borrow that app's figures.
+ */
+const appOf = (req) => (trackingSourceFor(req.path) ? requestedAppId(req) : '');
+
+/**
+ * A per-IP limiter: `max` requests per window from one address, or the app's
+ * own figure (`maxByApp`) for tracking requests to an app that has one. Zero
+ * means no limit, as the general figure or as an app's own.
  *
- * They are separate because a page being read reports its engagement every
+ * There are two of these, with separate budgets: one for engagement reports
+ * and one for everything else. A page being read reports its engagement every
  * half minute. On one shared budget, a few dozen readers behind one address
  * (an office, a campus) would use it up with those reports alone, and the page
  * views of everyone at that address would be refused. Apart, reports can only
  * ever crowd out other reports.
  *
- * @param {{ max: number, windowMs: number }} rateLimitConfig
+ * An app with its own figure is counted on its own, per address: a site that
+ * records far more per visitor than the others (every step inside a
+ * single-page app, say) gets the room it needs without loosening the limit
+ * for every other app, and without using up the address's budget for them.
+ *
+ * @param {{ max: number, windowMs: number, maxByApp?: Record<string, number> }} rateLimitConfig
  * @param {(req: import('express').Request) => void} [onLimit] called for a refused request
- * @returns {import('express').RequestHandler[]}
+ * @param {{ engagement?: boolean }} [options] true for the limiter of engagement
+ *     reports, which is mounted on that route so it can know the app; false
+ *     (the default) for the limiter of everything else, mounted app-wide
+ * @returns {import('express').RequestHandler}
  */
-function buildPerIpLimiters(rateLimitConfig, onLimit = () => {}) {
-    const { max, windowMs } = rateLimitConfig || {};
-    const limiter = (skip) => rateLimit({
+function buildPerIpLimiter(rateLimitConfig, onLimit = () => {}, { engagement = false } = {}) {
+    const { max = 0, windowMs, maxByApp = {} } = rateLimitConfig || {};
+    const ownFigure = (req) => (Object.hasOwn(maxByApp, appOf(req)) ? maxByApp[appOf(req)] : null);
+    const pass = (req, res, next) => next();
+    const shared = {
         windowMs,
-        limit: max,
         message: { message: 'Too many requests, please try again later.' },
         standardHeaders: true,
         legacyHeaders: false,
-        skip,
         handler: (req, res, next, options) => {
             onLimit(req);
             res.status(options.statusCode).json(options.message);
         },
-    });
-    return [limiter(isEngagement), limiter((req) => !isEngagement(req))];
+    };
+
+    // The general budget keeps the library's own key (the address) and, with
+    // it, the library's checks for a misconfigured proxy.
+    const general = max > 0 ? rateLimit({ ...shared, limit: max }) : pass;
+    // Apps with a figure of their own: one counter per address and app.
+    const own = Object.values(maxByApp).some((value) => value > 0)
+        ? rateLimit({
+            ...shared,
+            limit: (req) => ownFigure(req),
+            keyGenerator: (req) => `${ipKeyGenerator(req.ip)}|${appOf(req)}`,
+        })
+        : pass;
+
+    return (req, res, next) => {
+        // The reports' limiter is the one on their route: POST /engage.
+        const isReport = isEngagement(req) && req.method === 'POST';
+        if (isReport !== engagement) return next();
+        const figure = ownFigure(req);
+        if (figure === null) return general(req, res, next);
+        return figure > 0 ? own(req, res, next) : next();
+    };
 }
 
 /**
@@ -271,6 +312,11 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo =
     // views refused because of the reports those readers' pages send.
     const limitEngagePerApp = buildPerAppLimiter(config.server?.rateLimit,
         (req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'app' }));
+    // Per address too. This one sits on the route, not app-wide with the
+    // other, so it runs after the beacon's body is read and knows the app.
+    const readBeacon = express.text({ type: () => true, limit: TRACKING.ENGAGE_BODY_BYTES });
+    const limitEngagePerIp = buildPerIpLimiter(config.server?.rateLimit,
+        (req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'ip' }), { engagement: true });
     const trackingValidation = handleTrackingValidation(reject);
 
     /** Views from these are counted in the tracking log and never stored. */
@@ -466,8 +512,12 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo =
      * far it was scrolled, sent by the tracker when the page is hidden or left.
      */
     router.post('/engage',
-        express.text({ type: () => true, limit: TRACKING.ENGAGE_BODY_BYTES }),
+        // A body that cannot be read (too large, say) still counts against
+        // the address before it is refused: the app is unknown, so by the
+        // general figure.
+        (req, res, next) => readBeacon(req, res, (error) => (error ? limitEngagePerIp(req, res, () => next(error)) : next())),
         parseBeaconBody,
+        limitEngagePerIp,
         limitEngagePerApp,
         requireOrigin,
         validateEngage(config.allowed),
@@ -680,4 +730,4 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo =
     return router;
 }
 
-module.exports = { createAnalyticsRouter, buildPerIpLimiters, handleRouteError, logContext, withRequestId, trackingSourceFor };
+module.exports = { createAnalyticsRouter, buildPerIpLimiter, handleRouteError, logContext, withRequestId, trackingSourceFor };

package/utils/errorUtils.js

@@ -49,6 +49,7 @@ const WarningType = {
     ADMIN_LOG_WRITE_FAILED: 'ADMIN_LOG_WRITE_FAILED',
     TRASH_PURGE_FAILED: 'TRASH_PURGE_FAILED',
     SALT_PRUNE_FAILED: 'SALT_PRUNE_FAILED',
+    RATE_LIMIT_UNKNOWN_APP: 'RATE_LIMIT_UNKNOWN_APP',
 };
 
 /**
@@ -125,6 +126,8 @@ const WARNING_MESSAGES = {
         `Automatic view log pruning failed: ${info?.cause}`,
     [WarningType.TRASH_PURGE_FAILED]: (info) =>
         `Automatic trash purge failed for '${info?.appId}': ${info?.cause}`,
+    [WarningType.RATE_LIMIT_UNKNOWN_APP]: (info) =>
+        `${info?.field} names '${info?.appId}', which is not one of the configured apps: its figure applies only if an app of that name is registered later`,
     [WarningType.SALT_PRUNE_FAILED]: (info) =>
         `Could not delete the visitor salts of ended windows (it is tried again shortly): ${info?.cause}`,
 };