@harshankur/viewcounter 3.0.1 → 3.2.0

package/middleware/adminValidation.js

@@ -0,0 +1,319 @@
+/**
+ * Boundary validation for the admin API (CODE_STANDARDS.md §6).
+ *
+ * The admin is authenticated, but authenticated is not the same as trusted
+ * input: a stolen session, a buggy client, or a crafted request all arrive
+ * here. Every value is bounded, every enum-like value is checked against an
+ * allowlist, and a batch can never exceed ADMIN.MAX_BATCH_IDS rows.
+ */
+
+const { body, param, query, validationResult } = require('express-validator');
+
+const {
+    ADMIN,
+    ADMIN_ACTION,
+    ADMIN_RANGE,
+    ADMIN_ERROR_CODE,
+    ADMIN_SORT_COLUMNS,
+    EDITABLE_FIELDS,
+    FIELD_MAX_LENGTH,
+    HTTP_STATUS,
+    MODIFIED_FILTER,
+    PAYLOAD_LIMITS,
+    SORT_ORDER,
+    UUID_PATTERN,
+    VIEW_LOG_SOURCE,
+    TRACKING_OUTCOME,
+    VIEW_STATUS,
+} = require('../constants');
+const ReferrerParser = require('../utils/referrerParser');
+const { FILTER_COLUMNS } = require('../db/analysis');
+const { jsonByteLength } = require('../utils/stringUtils');
+
+const within = (values) => (value) => Object.values(values).includes(value);
+
+/** appId path parameter: must be a currently allowed app. */
+const adminAppIdParam = (allowed) =>
+    param('appId')
+        .custom((value) => allowed.appId.includes(value)).withMessage('Unknown appId');
+
+/** Optional appId filter on a log listing. */
+const adminAppIdFilter = (allowed) =>
+    query('appId')
+        .optional()
+        .custom((value) => allowed.appId.includes(value)).withMessage('Unknown appId');
+
+const pageQuery = () =>
+    query('page')
+        .optional()
+        .isInt({ min: 1, max: ADMIN.PAGE_MAX }).withMessage(`page must be an integer between 1 and ${ADMIN.PAGE_MAX}`);
+
+const pageSizeQuery = () =>
+    query('pageSize')
+        .optional()
+        .custom((value) => ADMIN.PAGE_SIZES.includes(Number(value)))
+        .withMessage(`pageSize must be one of: ${ADMIN.PAGE_SIZES.join(', ')}`);
+
+/**
+ * `ids`: a non-empty array of distinct UUIDs, at most MAX_BATCH_IDS long.
+ * Checked as a whole before any element is inspected, so an enormous array is
+ * rejected on its length rather than walked.
+ */
+const idsBody = () =>
+    body('ids')
+        .isArray({ min: 1, max: ADMIN.MAX_BATCH_IDS })
+        .withMessage(`ids must be an array of 1 to ${ADMIN.MAX_BATCH_IDS} view IDs`)
+        .bail()
+        .custom((ids) => ids.every((id) => typeof id === 'string' && UUID_PATTERN.test(id)))
+        .withMessage('every id must be a UUID')
+        .bail()
+        .custom((ids) => new Set(ids).size === ids.length)
+        .withMessage('ids must not repeat');
+
+const validateLogin = () => [
+    body('password')
+        .isString().withMessage('password is required')
+        .isLength({ min: 1, max: ADMIN.MAX_PASSWORD_INPUT_LENGTH })
+        .withMessage('password is required'),
+];
+
+/** The longest breakdown value a `where` filter can hold: a referrer or page path. */
+const WHERE_VALUE_MAX_LENGTH = Math.max(FIELD_MAX_LENGTH.REFERRER, FIELD_MAX_LENGTH.PAGE_PATH);
+/** Every dimension at its longest, as JSON, with room for escaping. */
+const WHERE_MAX_LENGTH = Object.keys(FILTER_COLUMNS).length * (WHERE_VALUE_MAX_LENGTH + 32) * 2;
+
+/**
+ * Read `where`: breakdown filters as a JSON object of dimension name to
+ * value, with null for rows that have none, such as {"country":"DE"}.
+ * @param {unknown} raw
+ * @returns {{ ok: true, filters: Record<string, string|null> } | { ok: false, error: string }}
+ */
+function readWhere(raw) {
+    if (raw === undefined || raw === '') return { ok: true, filters: {} };
+    if (typeof raw !== 'string' || raw.length > WHERE_MAX_LENGTH) return { ok: false, error: 'where must be a short JSON object' };
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    } catch {
+        return { ok: false, error: 'where must be a JSON object' };
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return { ok: false, error: 'where must be a JSON object' };
+    const filters = {};
+    for (const [dim, value] of Object.entries(parsed)) {
+        if (!Object.hasOwn(FILTER_COLUMNS, dim)) return { ok: false, error: 'where names a dimension that cannot be filtered' };
+        if (value !== null && (typeof value !== 'string' || value.length > WHERE_VALUE_MAX_LENGTH)) {
+            return { ok: false, error: `each where value must be null or a string of at most ${WHERE_VALUE_MAX_LENGTH} characters` };
+        }
+        filters[dim] = value;
+    }
+    return { ok: true, filters };
+}
+
+/**
+ * The breakdown filters of a request that passed validation.
+ * @param {unknown} raw
+ * @returns {Record<string, string|null>}
+ */
+function parseWhere(raw) {
+    const result = readWhere(raw);
+    return result.ok ? result.filters : {};
+}
+
+/** Filters shared by a listing and its analysis, for one app or all. */
+const filterQueries = () => [
+    query('status').optional().custom(within(VIEW_STATUS)).withMessage('Invalid status'),
+    query('modified').optional().custom(within(MODIFIED_FILTER)).withMessage('Invalid modified filter'),
+    query('range').optional().custom(within(ADMIN_RANGE)).withMessage('Invalid range'),
+    query('eventType')
+        .optional()
+        .isString().withMessage('eventType must be a string')
+        .isLength({ max: FIELD_MAX_LENGTH.EVENT_TYPE })
+        .withMessage(`eventType must be at most ${FIELD_MAX_LENGTH.EVENT_TYPE} characters`),
+    query('search')
+        .optional()
+        .isString().withMessage('search must be a string')
+        .isLength({ max: ADMIN.SEARCH_MAX_LENGTH })
+        .withMessage(`search must be at most ${ADMIN.SEARCH_MAX_LENGTH} characters`),
+    query('where')
+        .optional()
+        .custom((value) => readWhere(value).ok)
+        .withMessage((value) => readWhere(value).error),
+];
+
+/** A listing of one app (with `allowed`) or of every app (without). */
+const validateViewListing = (allowed) => [
+    ...(allowed ? [adminAppIdParam(allowed)] : []),
+    ...filterQueries(),
+    query('sort').optional().custom((value) => Object.hasOwn(ADMIN_SORT_COLUMNS, value)).withMessage('Invalid sort'),
+    query('order').optional().custom(within(SORT_ORDER)).withMessage('Invalid order'),
+    pageQuery(),
+    pageSizeQuery(),
+];
+
+/** The analysis of one app (with `allowed`) or of every app (without). */
+const validateAnalysis = (allowed) => [
+    ...(allowed ? [adminAppIdParam(allowed)] : []),
+    ...filterQueries(),
+];
+
+/** The event types of one app (with `allowed`) or of every app, in a status. */
+const validateEventTypes = (allowed) => [
+    ...(allowed ? [adminAppIdParam(allowed)] : []),
+    query('status').optional().custom(within(VIEW_STATUS)).withMessage('Invalid status'),
+];
+
+/** Right now takes no filters, only the app for the per-app route. */
+const validateRealtime = (allowed) => (allowed ? [adminAppIdParam(allowed)] : []);
+
+
+/**
+ * Validate one field of an edit and return the column value to store.
+ * @returns {{ ok: true, value: unknown } | { ok: false, error: string }}
+ */
+function checkField(field, value, deviceSizes) {
+    const optionalText = (max) => {
+        if (value === null || value === '') return { ok: true, value: null };
+        if (typeof value !== 'string') return { ok: false, error: `${field} must be a string or null` };
+        if (value.length > max) return { ok: false, error: `${field} must be at most ${max} characters` };
+        return { ok: true, value };
+    };
+
+    switch (field) {
+        case 'pagePath':
+            return optionalText(FIELD_MAX_LENGTH.PAGE_PATH);
+        case 'pageTitle':
+            return optionalText(FIELD_MAX_LENGTH.PAGE_TITLE);
+        case 'referrer':
+            return optionalText(FIELD_MAX_LENGTH.REFERRER);
+        case 'deviceSize':
+            return deviceSizes.includes(value)
+                ? { ok: true, value }
+                : { ok: false, error: `deviceSize must be one of: ${deviceSizes.join(', ')}` };
+        case 'eventType':
+            if (typeof value !== 'string' || value.trim().length === 0) {
+                return { ok: false, error: 'eventType must be a non-empty string' };
+            }
+            if (value.length > FIELD_MAX_LENGTH.EVENT_TYPE) {
+                return { ok: false, error: `eventType must be at most ${FIELD_MAX_LENGTH.EVENT_TYPE} characters` };
+            }
+            return { ok: true, value };
+        case 'eventData':
+            if (value === null) return { ok: true, value: null };
+            if (typeof value !== 'object') return { ok: false, error: 'eventData must be an object, an array, or null' };
+            if (jsonByteLength(value) > PAYLOAD_LIMITS.MAX_EVENT_DATA_BYTES) {
+                return { ok: false, error: `eventData must serialize to at most ${PAYLOAD_LIMITS.MAX_EVENT_DATA_BYTES} bytes` };
+            }
+            return { ok: true, value: JSON.stringify(value) };
+        default:
+            return { ok: false, error: `${field} is not an editable field` };
+    }
+}
+
+/**
+ * Turn an edit's `changes` object into column values.
+ *
+ * Only EDITABLE_FIELDS may appear. A change to `referrer` also re-derives
+ * `referrer_domain` and `source_type` with the same parser the write path
+ * uses, and stores the referrer the same way (origin and path only). Whether
+ * it is on the row's own site, and so internal, depends on each row's site,
+ * which the repository applies per row (AdminRepository.updateContent).
+ *
+ * @param {unknown} changes
+ * @param {string[]} deviceSizes
+ * @returns {{ ok: true, columns: Record<string, unknown>, fields: string[] } | { ok: false, error: string }}
+ */
+function resolveChanges(changes, deviceSizes) {
+    if (!changes || typeof changes !== 'object' || Array.isArray(changes)) {
+        return { ok: false, error: 'changes must be an object' };
+    }
+    const fields = Object.keys(changes);
+    if (fields.length === 0) return { ok: false, error: 'changes must name at least one field' };
+
+    const columns = {};
+    for (const field of fields) {
+        if (!Object.hasOwn(EDITABLE_FIELDS, field)) {
+            return { ok: false, error: `${field} is not an editable field` };
+        }
+        const result = checkField(field, changes[field], deviceSizes);
+        if (!result.ok) return result;
+        columns[EDITABLE_FIELDS[field]] = result.value;
+    }
+
+    if (Object.hasOwn(changes, 'referrer')) {
+        const parsed = ReferrerParser.parse(columns.referrer);
+        columns.referrer = parsed.referrer;
+        columns.referrer_domain = parsed.referrerDomain;
+        columns.source_type = parsed.sourceType;
+    }
+
+    return { ok: true, columns, fields };
+}
+
+const validateEdit = (allowed) => [
+    adminAppIdParam(allowed),
+    idsBody(),
+    body('changes')
+        .custom((changes, { req }) => {
+            const result = resolveChanges(changes, allowed.deviceSize);
+            req.resolvedChanges = result;
+            return result.ok;
+        })
+        .withMessage((value, { req }) => req.resolvedChanges?.error),
+];
+
+const validateNote = (allowed) => [
+    adminAppIdParam(allowed),
+    idsBody(),
+    body('note')
+        .custom((note) => note === null || typeof note === 'string').withMessage('note must be a string or null')
+        .bail()
+        .custom((note) => note === null || note.length <= FIELD_MAX_LENGTH.NOTE)
+        .withMessage(`note must be at most ${FIELD_MAX_LENGTH.NOTE} characters`),
+];
+
+const validateBatch = (allowed) => [adminAppIdParam(allowed), idsBody()];
+
+const validateAdminLogListing = (allowed) => [
+    pageQuery(),
+    pageSizeQuery(),
+    adminAppIdFilter(allowed),
+    query('action').optional().custom(within(ADMIN_ACTION)).withMessage('Invalid action'),
+];
+
+const validateTrackingLogListing = (allowed) => [
+    pageQuery(),
+    pageSizeQuery(),
+    adminAppIdFilter(allowed),
+    query('source').optional().custom(within(VIEW_LOG_SOURCE)).withMessage('Invalid source'),
+    query('outcome').optional().custom(within(TRACKING_OUTCOME)).withMessage('Invalid outcome'),
+];
+
+const validateTrackingSummary = (allowed) => [adminAppIdFilter(allowed)];
+
+/** Admin-shaped validation failure: a stable code plus per-field detail. */
+function handleAdminValidation(req, res, next) {
+    const errors = validationResult(req);
+    if (errors.isEmpty()) return next();
+    return res.status(HTTP_STATUS.UNPROCESSABLE_ENTITY).json({
+        code: ADMIN_ERROR_CODE.VALIDATION_FAILED,
+        errors: errors.array().map((error) => ({ field: error.path, message: error.msg })),
+    });
+}
+
+module.exports = {
+    parseWhere,
+    validateLogin,
+    validateViewListing,
+    validateAnalysis,
+    validateEventTypes,
+    validateRealtime,
+    validateEdit,
+    validateNote,
+    validateBatch,
+    validateAdminLogListing,
+    validateTrackingLogListing,
+    validateTrackingSummary,
+    handleAdminValidation,
+    resolveChanges,
+    checkField,
+};

package/middleware/auth.js

@@ -3,8 +3,8 @@
  *
  * Two distinct steps, deliberately separate:
  *
- *   requireReadApiKey  — *authentication*: is this a key we issued?
- *   requireAppScope    — *authorization*: may THIS key read THIS app?
+ *   requireReadApiKey   *authentication*: is this a key we issued?
+ *   requireAppScope     *authorization*: may THIS key read THIS app?
  *
  * The second step is what makes the service multi-tenant. Without it a valid
  * key read every tenant's analytics, because the appId allowlist only ever

package/middleware/security.js

@@ -10,7 +10,7 @@ const { HTTP_STATUS } = require('../constants');
  * agent-instructions SECURITY.md §9: `cors()` with no options is never the
  * default. The wildcard mattered more here than the usual "no credentials, so
  * it's harmless" reasoning suggests, because the read endpoints served real
- * data — `*` made them script-readable from any origin, not merely reachable.
+ * data: `*` made them script-readable from any origin, not merely reachable.
  *
  * @param {string[]} allowedOrigins
  * @returns {import('cors').CorsOptions}
@@ -32,6 +32,26 @@ function buildCorsOptions(allowedOrigins) {
     };
 }
 
+/**
+ * Before a browser sends JSON to another site, it asks (a CORS preflight). A
+ * site missing from CORS_ORIGINS is refused there, and the request itself
+ * never arrives, so nothing else could count it. This counts the refusal, for
+ * the tracking-log paths `isTrackingPath` names, so a misconfigured
+ * CORS_ORIGINS shows up in the tracking log. Place it before `cors()`.
+ *
+ * @param {string[]} allowedOrigins as given to buildCorsOptions
+ * @param {{ isTrackingPath: (path: string) => boolean, onRefused: (req: import('express').Request) => void }} hooks
+ * @returns {import('express').RequestHandler}
+ */
+function countRefusedPreflights(allowedOrigins, { isTrackingPath, onRefused }) {
+    const allowlist = new Set(allowedOrigins);
+    return (req, res, next) => {
+        const origin = req.get('origin');
+        if (req.method === 'OPTIONS' && origin && !allowlist.has(origin) && isTrackingPath(req.path)) onRefused(req);
+        next();
+    };
+}
+
 /**
  * Extract the requesting origin, falling back to the referrer's origin.
  * @returns {string|null}
@@ -64,9 +84,11 @@ function requestOrigin(req) {
  * every appId still in that state.
  *
  * @param {{ origins: Record<string, string[]> }} allowed
+ * @param {{ onReject?: (req: import('express').Request, appId: string) => void }} [options]
+ *   told about each refusal, so the tracking log can count it
  * @returns {import('express').RequestHandler}
  */
-function requireRegisteredOrigin(allowed) {
+function requireRegisteredOrigin(allowed, { onReject = () => {} } = {}) {
     const origins = allowed?.origins || {};
 
     return (req, res, next) => {
@@ -82,6 +104,7 @@ function requireRegisteredOrigin(allowed) {
             return next();
         }
 
+        onReject(req, appId);
         return res.status(HTTP_STATUS.FORBIDDEN).json({
             message: 'Request origin is not registered for this appId',
         });
@@ -103,6 +126,7 @@ function noStore(req, res, next) {
 
 module.exports = {
     buildCorsOptions,
+    countRefusedPreflights,
     requireRegisteredOrigin,
     requestOrigin,
     noStore,

package/middleware/validation.js

@@ -5,8 +5,12 @@ const {
     HTTP_STATUS,
     PAYLOAD_LIMITS,
     QUERY_LIMITS,
+    REJECTION_REASON,
+    TRACKING,
     TREND_PERIODS,
+    UUID_PATTERN,
 } = require('../constants');
+const { UTM_PARAMETERS } = require('../utils/visitorContext');
 const { jsonByteLength } = require('../utils/stringUtils');
 const { isValidAppId } = require('../utils/appIdUtils');
 
@@ -60,7 +64,7 @@ const boundedBody = (name, max) =>
  *
  * Deliberately no `.toInt()` sanitizer: under Express 5 `req.query` is a
  * getter-only property, so a sanitizer appears to work but never writes the
- * coerced value back — the handler would still receive a string and bind it
+ * coerced value back: the handler would still receive a string and bind it
  * into `LIMIT ?`, which MySQL rejects. Handlers coerce explicitly instead,
  * after this validator has established the value is a valid integer in range.
  */
@@ -83,6 +87,24 @@ const validateRegisterView = (allowedValues) => [
     boundedQuery('title', FIELD_MAX_LENGTH.PAGE_TITLE),
     boundedQuery('referrer', FIELD_MAX_LENGTH.REFERRER),
     boundedQuery('sessionId', FIELD_MAX_LENGTH.SESSION_ID),
+    ...UTM_PARAMETERS.map((name) => boundedQuery(name, FIELD_MAX_LENGTH.UTM)),
+];
+
+/**
+ * Validate an engagement beacon: how long a recorded view's page was visible
+ * and how far it was scrolled.
+ */
+const validateEngage = (allowedValues) => [
+    body('appId')
+        .notEmpty().withMessage('appId is required')
+        .custom((value) => allowedValues.appId.includes(value)).withMessage('Invalid appId'),
+    body('id')
+        .isString().withMessage('id must be a string')
+        .matches(UUID_PATTERN).withMessage('id must be a view ID'),
+    body('ms')
+        .isInt({ min: 0, max: TRACKING.MAX_ENGAGED_MS }).withMessage(`ms must be an integer between 0 and ${TRACKING.MAX_ENGAGED_MS}`),
+    body('scroll')
+        .isInt({ min: 0, max: 100 }).withMessage('scroll must be an integer between 0 and 100'),
 ];
 
 /**
@@ -165,7 +187,7 @@ const validateSessionRequest = (allowedValues) => [
  * Validate an app-provisioning request.
  *
  * `appId` here becomes a table identifier, so it is checked against the strict
- * pattern rather than an allowlist — there is no allowlist yet, that is the
+ * pattern rather than an allowlist: there is no allowlist yet, that is the
  * point of the call.
  */
 const validateAppRegistration = () => [
@@ -199,14 +221,40 @@ const handleValidationErrors = (req, res, next) => {
     return next();
 };
 
+/**
+ * The same 422 as handleValidationErrors, for a tracking endpoint: the refusal
+ * is also reported, so the tracking log can say why a site's views are not
+ * arriving. An appId that was given but is not allowed is its own reason,
+ * since a misspelled one is the commonest cause; anything else, including a
+ * missing appId, names the first field that failed.
+ *
+ * @param {(req: import('express').Request, reason: string, details: { appId?: string, detail?: string }) => void} onReject
+ * @returns {import('express').RequestHandler}
+ */
+const handleTrackingValidation = (onReject) => (req, res, next) => {
+    const errors = validationResult(req);
+    if (errors.isEmpty()) return next();
+
+    const failed = errors.array();
+    const appIdError = failed.find((error) => error.path === 'appId');
+    if (appIdError && typeof appIdError.value === 'string' && appIdError.value !== '') {
+        onReject(req, REJECTION_REASON.UNKNOWN_APP, { appId: appIdError.value });
+    } else {
+        onReject(req, REJECTION_REASON.INVALID_REQUEST, { detail: (appIdError || failed[0]).path });
+    }
+    return handleValidationErrors(req, res, next);
+};
+
 module.exports = {
     validateAppRegistration,
     validateRegisterView,
     validateEvent,
+    validateEngage,
     validateStatsRequest,
     validateTrendsRequest,
     validateListRequest,
     validateViewsRequest,
     validateSessionRequest,
     handleValidationErrors,
+    handleTrackingValidation,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@harshankur/viewcounter",
   "description": "A middleware backend server that registers views to my db server when requested to register a view from my other projects.",
-  "version": "3.0.1",
+  "version": "3.2.0",
   "main": "index.js",
   "engines": {
     "node": ">=24"
@@ -9,6 +9,8 @@
   "files": [
     "index.js",
     "constants.js",
+    "admin/",
+    "tracker/",
     "config/",
     "db/",
     "middleware/",
@@ -27,8 +29,11 @@
     "start": "node index.js",
     "setup": "node scripts/setup.js",
     "assets": "node scripts/generate-brand-assets.js",
+    "map": "node scripts/generate-world-map.js",
+    "admin:demo": "node tests/ui/server.js 4173 --demo",
     "lint": "eslint .",
-    "test": "npm run lint && jest --coverage --verbose && node scripts/generate-test-report.js",
+    "test": "npm run lint && jest --coverage --verbose && playwright test && node scripts/generate-test-report.js",
+    "test:ui": "playwright test",
     "test:watch": "jest --watch",
     "test:ci": "npm run lint && jest --coverage --ci",
     "test:persist": "PERSIST_TEST_DB=true jest --coverage --verbose && node scripts/generate-test-report.js",
@@ -38,10 +43,15 @@
   },
   "devDependencies": {
     "@eslint/js": "^9.39.5",
+    "@playwright/test": "^1.63.0",
+    "d3-geo": "^3.1.1",
     "eslint": "^9.39.5",
+    "i18n-iso-countries": "^7.14.0",
     "jest": "^30.4.2",
     "jest-html-reporter": "^4.4.0",
-    "supertest": "^7.2.2"
+    "supertest": "^7.2.2",
+    "topojson-client": "^3.1.0",
+    "world-atlas": "^2.0.2"
   },
   "repository": {
     "type": "git",
@@ -65,17 +75,17 @@
     "cors": "^2.8.6",
     "dotenv": "^17.4.2",
     "express": "^5.2.1",
-    "express-rate-limit": "^8.6.0",
+    "express-rate-limit": "^8.7.0",
     "express-validator": "^7.3.2",
-    "geoip-country": "^5.0.202607180103",
+    "geoip-country": "^5.0.202609230144",
     "helmet": "^8.3.0",
-    "mysql2": "^3.23.0",
-    "ua-parser-js": "^2.0.10",
+    "isbot": "^5.2.2",
+    "maxmind": "^5.0.7",
+    "mysql2": "^3.24.4",
+    "ua-parser-js": "^1.0.41",
     "url-parse": "^1.5.10"
   },
   "overrides": {
-    "geoip-country": {
-      "ip-address": "^10.2.0"
-    }
+    "ip-address": "^10.7.2"
   }
 }