@harshankur/viewcounter 3.1.0 → 3.3.0

package/db/DatabaseManager.js

@@ -9,6 +9,7 @@ const {
     QUERY_LIMITS,
     SERVER,
     TOP_N_RESULTS,
+    TRACKING,
     TREND_PERIOD,
     VIEW_LOG_SOURCE,
 } = require('../constants');
@@ -37,7 +38,7 @@ const AdminRepository = require('./AdminRepository');
  * Columns returned for a session lookup.
  *
  * Deliberately explicit rather than `SELECT *`. The previous wildcard returned
- * `visitor_hash` — the pseudonymous visitor identifier itself — to any caller
+ * `visitor_hash` (the pseudonymous visitor identifier itself) to any caller
  * of the sessions endpoint.
  */
 const SESSION_COLUMNS = [
@@ -65,7 +66,7 @@ const SESSION_COLUMNS = [
  *
  * `id` is a generated UUID and is the row's identity; `app_id` is a uniqueness
  * *constraint*, not an identity (CODE_STANDARDS.md §8). The distinction matters
- * the first time an app is renamed, or deleted and a later one reuses the name —
+ * the first time an app is renamed, or deleted and a later one reuses the name:
  * with the natural key as the primary key, anything referencing the old row
  * would silently re-point at the new one.
  */
@@ -82,7 +83,7 @@ const APP_REGISTRY_DDL = `
  * DDL for one app's event table.
  *
  * `appId` is interpolated because MySQL cannot bind an identifier as a
- * parameter. Every caller must have passed it through `isValidAppId` first —
+ * parameter. Every caller must have passed it through `isValidAppId` first;
  * the assertion below is the backstop, not the primary gate.
  *
  * @param {string} appId
@@ -141,7 +142,7 @@ function appTableDDL(appId) {
  * `mysql2/promise`'s pool emits the RAW callback-style connection on its
  * `connection` event, not the promise-wrapped one. Its `query()` returns a
  * `Query`, and mysql2 deliberately makes `.then()`/`.catch()` on a `Query`
- * throw — so treating it as a promise crashes the process on the very first
+ * throw, so treating it as a promise crashes the process on the very first
  * database connection. The callback form is the correct API for that object.
  *
  * Failure is swallowed on purpose: MariaDB and MySQL < 5.7.8 have no
@@ -347,7 +348,7 @@ class DatabaseManager {
     /**
      * Provision a new app: validate, create its table, record it.
      *
-     * Idempotent — re-registering an existing app is a no-op rather than an
+     * Idempotent: re-registering an existing app is a no-op rather than an
      * error, so a retried provisioning call cannot fail halfway.
      *
      * @param {string} appId
@@ -439,6 +440,15 @@ class DatabaseManager {
             sessionId,
             eventType = EVENT_TYPE.PAGEVIEW,
             eventData,
+            hostname = null,
+            language = null,
+            utmSource = null,
+            utmMedium = null,
+            utmCampaign = null,
+            utmTerm = null,
+            utmContent = null,
+            region = null,
+            city = null,
             uniqueWindowHours = SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS,
             userAgent = '',
             visitorSecret,
@@ -480,8 +490,10 @@ class DatabaseManager {
                 page_path, page_title,
                 referrer, referrer_domain, source_type,
                 browser, browser_version, os, os_version, device_type,
-                session_id, event_type, event_data, is_unique
-            ) VALUES (?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+                session_id, event_type, event_data, is_unique,
+                hostname, language, utm_source, utm_medium, utm_campaign, utm_term, utm_content,
+                region, city
+            ) VALUES (?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
             [
                 publicId,
                 truncate(maskedIp, FIELD_MAX_LENGTH.MASKED_IP),
@@ -502,6 +514,15 @@ class DatabaseManager {
                 storedEventType,
                 eventData ? JSON.stringify(eventData) : null,
                 isUnique,
+                truncate(hostname, FIELD_MAX_LENGTH.HOSTNAME),
+                truncate(language, FIELD_MAX_LENGTH.LANGUAGE),
+                truncate(utmSource, FIELD_MAX_LENGTH.UTM),
+                truncate(utmMedium, FIELD_MAX_LENGTH.UTM),
+                truncate(utmCampaign, FIELD_MAX_LENGTH.UTM),
+                truncate(utmTerm, FIELD_MAX_LENGTH.UTM),
+                truncate(utmContent, FIELD_MAX_LENGTH.UTM),
+                truncate(region, FIELD_MAX_LENGTH.REGION),
+                truncate(city, FIELD_MAX_LENGTH.CITY),
             ]
         );
 
@@ -513,6 +534,7 @@ class DatabaseManager {
             viewId: publicId,
             eventType: storedEventType,
             isUnique: isUnique === 1,
+            hostname,
         });
 
         return {
@@ -523,6 +545,34 @@ class DatabaseManager {
         };
     }
 
+    /**
+     * Record how long a view's page was visible and how far it was scrolled.
+     *
+     * A page reports this when it is hidden or left, and every half minute
+     * while it is being read, each time with its running total, so the larger
+     * value always wins. Each report also marks the view as seen just now,
+     * which is what keeps its visitor in "right now" between page views. Only a live view from the last
+     * TRACKING.ENGAGE_WINDOW_HOURS is updated: an old or trashed view keeps
+     * what it had.
+     *
+     * @param {string} appId already validated
+     * @param {{ viewId: string, engagedMs: number, scrollDepth: number }} engagement
+     * @returns {Promise<boolean>} whether a view was updated
+     */
+    async addEngagement(appId, { viewId, engagedMs, scrollDepth }) {
+        this.assertReady();
+        const [result] = await this.pool.query(
+            `UPDATE \`${appId}\`
+             SET engaged_ms = GREATEST(COALESCE(engaged_ms, 0), ?),
+                 scroll_depth = GREATEST(COALESCE(scroll_depth, 0), ?),
+                 last_seen_at = NOW()
+             WHERE public_id = ? AND ${LIVE_ROW}
+               AND timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)`,
+            [engagedMs, scrollDepth, viewId, TRACKING.ENGAGE_WINDOW_HOURS]
+        );
+        return result.affectedRows > 0;
+    }
+
     /**
      * Register a view (backward compatible wrapper)
      */

package/db/LogRepository.js

@@ -7,7 +7,14 @@
 
 const crypto = require('crypto');
 
-const { ADMIN, ADMIN_LOG_TABLE, VIEW_LOG_TABLE } = require('../constants');
+const {
+    ADMIN,
+    ADMIN_LOG_TABLE,
+    REJECTION_REASON,
+    TRACKING_OUTCOME,
+    TRACKING_REJECTIONS_TABLE,
+    VIEW_LOG_TABLE,
+} = require('../constants');
 const { logWarning, WarningType } = require('../utils/errorUtils');
 
 /**
@@ -20,9 +27,29 @@ const ADMIN_LOG_COLUMNS = [
 ].join(', ');
 
 const VIEW_LOG_COLUMNS = [
-    'id', 'created_at', 'app_id', 'source', 'view_id', 'event_type', 'is_unique',
+    'id', 'created_at', 'app_id', 'source', 'view_id', 'event_type', 'is_unique', 'hostname',
 ].join(', ');
 
+/**
+ * The tracking log is two streams read as one: accepted views, one row each
+ * in the view log, and requests that were not stored, counted per minute.
+ * Both branches select the same columns so they can be merged and paged
+ * together, newest first.
+ */
+const ACCEPTED_BRANCH = `
+    SELECT created_at AS at, id AS entry_id, app_id, source,
+        IF(is_unique = 1, '${TRACKING_OUTCOME.RECORDED}', '${TRACKING_OUTCOME.REPEAT}') AS outcome,
+        CAST(NULL AS CHAR) AS reason, CAST(NULL AS CHAR) AS detail, hostname,
+        view_id, event_type, 1 AS requests
+    FROM \`${VIEW_LOG_TABLE}\``;
+const REJECTED_BRANCH = `
+    SELECT minute AS at, CONCAT_WS('|', minute, source, reason, app_id, detail, hostname) AS entry_id,
+        NULLIF(app_id, '') AS app_id, source,
+        IF(reason = '${REJECTION_REASON.BOT}', '${TRACKING_OUTCOME.BOT}', '${TRACKING_OUTCOME.REJECTED}') AS outcome,
+        reason, NULLIF(detail, '') AS detail, NULLIF(hostname, '') AS hostname,
+        CAST(NULL AS CHAR) AS view_id, CAST(NULL AS CHAR) AS event_type, requests
+    FROM \`${TRACKING_REJECTIONS_TABLE}\``;
+
 /** mysql2 returns JSON columns parsed; tolerate a string from other drivers. */
 function parseJson(value) {
     if (value === null || value === undefined) return null;
@@ -101,13 +128,13 @@ class LogRepository {
      *   isUnique: boolean }} entry
      * @returns {Promise<boolean>}
      */
-    async writeViewLog({ appId, source, viewId, eventType, isUnique }) {
+    async writeViewLog({ appId, source, viewId, eventType, isUnique, hostname = null }) {
         try {
             await this.pool.query(
                 `INSERT INTO \`${VIEW_LOG_TABLE}\`
-                    (id, created_at, app_id, source, view_id, event_type, is_unique)
-                 VALUES (?, NOW(3), ?, ?, ?, ?, ?)`,
-                [crypto.randomUUID(), appId, source, viewId, eventType, isUnique ? 1 : 0]
+                    (id, created_at, app_id, source, view_id, event_type, is_unique, hostname)
+                 VALUES (?, NOW(3), ?, ?, ?, ?, ?, ?)`,
+                [crypto.randomUUID(), appId, source, viewId, eventType, isUnique ? 1 : 0, hostname]
             );
             return true;
         } catch (cause) {
@@ -125,16 +152,51 @@ class LogRepository {
      * @returns {Promise<number>} entries removed
      */
     async pruneViewLog(days) {
-        let total = 0;
-        for (;;) {
-            const [result] = await this.pool.query(
-                `DELETE FROM \`${VIEW_LOG_TABLE}\` WHERE created_at < DATE_SUB(NOW(3), INTERVAL ? DAY)
-                 ORDER BY created_at LIMIT ?`,
-                [days, ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE]
+        const prune = async (table, column) => {
+            let total = 0;
+            for (;;) {
+                const [result] = await this.pool.query(
+                    `DELETE FROM \`${table}\` WHERE \`${column}\` < DATE_SUB(NOW(3), INTERVAL ? DAY)
+                     ORDER BY \`${column}\` LIMIT ?`,
+                    [days, ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE]
+                );
+                const removed = Number(result?.affectedRows || 0);
+                total += removed;
+                if (removed < ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE) return total;
+            }
+        };
+        return await prune(VIEW_LOG_TABLE, 'created_at') + await prune(TRACKING_REJECTIONS_TABLE, 'minute');
+    }
+
+    /**
+     * Add counted rejections, merging into any count already stored for the
+     * same minute and key. Never throws: losing a count must not fail the
+     * request that produced it, or the flush that carries many.
+     *
+     * @param {{ minute: Date, source: string, reason: string, appId?: string,
+     *   detail?: string, hostname?: string, requests: number }[]} rows
+     * @returns {Promise<boolean>}
+     */
+    async recordRejections(rows) {
+        if (rows.length === 0) return true;
+        try {
+            await this.pool.query(
+                `INSERT INTO \`${TRACKING_REJECTIONS_TABLE}\`
+                    (minute, source, reason, app_id, detail, hostname, requests)
+                 VALUES ${rows.map(() => '(FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)').join(', ')}
+                 ON DUPLICATE KEY UPDATE requests = requests + VALUES(requests)`,
+                // The minute as seconds since the epoch, placed in the database's
+                // own time zone like every NOW() it is compared with, whatever
+                // zone this process runs in.
+                rows.flatMap((row) => [
+                    Math.floor(new Date(row.minute).getTime() / 1000), row.source, row.reason,
+                    row.appId || '', row.detail || '', row.hostname || '', row.requests,
+                ])
             );
-            const removed = Number(result?.affectedRows || 0);
-            total += removed;
-            if (removed < ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE) return total;
+            return true;
+        } catch (cause) {
+            logWarning(WarningType.TRACKING_LOG_WRITE_FAILED, { cause: cause.message });
+            return false;
         }
     }
 
@@ -176,37 +238,112 @@ class LogRepository {
     }
 
     /**
-     * @param {{ page: number, pageSize: number, appId?: string, source?: string }} query
+     * The tracking log: every tracking request and what became of it, newest
+     * first. Accepted views are listed one by one; bots and rejections as one
+     * entry per minute and key, with how many requests it stands for.
+     *
+     * @param {{ page: number, pageSize: number, appId?: string, source?: string, outcome?: string }} query
      * @returns {Promise<{ entries: object[], total: number }>}
      */
-    async listViewLog({ page, pageSize, appId, source }) {
-        const where = [];
+    async listTrackingLog({ page, pageSize, appId, source, outcome }) {
+        const accepted = { where: [], params: [] };
+        const rejected = { where: [], params: [] };
+        for (const branch of [accepted, rejected]) {
+            if (appId) { branch.where.push('app_id = ?'); branch.params.push(appId); }
+            if (source) { branch.where.push('source = ?'); branch.params.push(source); }
+        }
+        if (outcome === TRACKING_OUTCOME.RECORDED) accepted.where.push('is_unique = 1');
+        if (outcome === TRACKING_OUTCOME.REPEAT) accepted.where.push('is_unique = 0');
+        if (outcome === TRACKING_OUTCOME.BOT) { rejected.where.push('reason = ?'); rejected.params.push(REJECTION_REASON.BOT); }
+        if (outcome === TRACKING_OUTCOME.REJECTED) { rejected.where.push('reason <> ?'); rejected.params.push(REJECTION_REASON.BOT); }
+
+        const wantAccepted = !outcome || outcome === TRACKING_OUTCOME.RECORDED || outcome === TRACKING_OUTCOME.REPEAT;
+        const wantRejected = !outcome || outcome === TRACKING_OUTCOME.BOT || outcome === TRACKING_OUTCOME.REJECTED;
+        const clause = (branch) => (branch.where.length ? ` WHERE ${branch.where.join(' AND ')}` : '');
+
+        // Each table gives at most the rows the page could need, in the same
+        // order as the whole, so a page never sorts either table in full; the
+        // view log reads its created_at index backwards to do it.
+        const offset = (page - 1) * pageSize;
+        const ORDER = 'ORDER BY at DESC, entry_id DESC';
+        const branches = [];
         const params = [];
-        if (appId) { where.push('app_id = ?'); params.push(appId); }
-        if (source) { where.push('source = ?'); params.push(source); }
-        const clause = where.length ? `WHERE ${where.join(' AND ')}` : '';
+        for (const [want, branch, sql] of [[wantAccepted, accepted, ACCEPTED_BRANCH], [wantRejected, rejected, REJECTED_BRANCH]]) {
+            if (!want) continue;
+            branches.push(`(${sql}${clause(branch)} ${ORDER} LIMIT ?)`);
+            params.push(...branch.params, offset + pageSize);
+        }
 
         const [rows] = await this.pool.query(
-            `SELECT ${VIEW_LOG_COLUMNS} FROM \`${VIEW_LOG_TABLE}\` ${clause}
-             ORDER BY created_at DESC, id ASC LIMIT ? OFFSET ?`,
-            [...params, pageSize, (page - 1) * pageSize]
-        );
-        const [count] = await this.pool.query(
-            `SELECT COUNT(*) AS count FROM \`${VIEW_LOG_TABLE}\` ${clause}`,
-            params
+            `SELECT * FROM (${branches.join(' UNION ALL ')}) AS entries
+             ${ORDER} LIMIT ? OFFSET ?`,
+            [...params, pageSize, offset]
         );
 
+        let total = 0;
+        if (wantAccepted) {
+            const [count] = await this.pool.query(
+                `SELECT COUNT(*) AS count FROM \`${VIEW_LOG_TABLE}\`${clause(accepted)}`, accepted.params);
+            total += Number(count[0]?.count || 0);
+        }
+        if (wantRejected) {
+            const [count] = await this.pool.query(
+                `SELECT COUNT(*) AS count FROM \`${TRACKING_REJECTIONS_TABLE}\`${clause(rejected)}`, rejected.params);
+            total += Number(count[0]?.count || 0);
+        }
+
         return {
             entries: rows.map((row) => ({
-                id: row.id,
-                createdAt: row.created_at,
-                appId: row.app_id,
+                id: row.entry_id,
+                at: row.at,
+                appId: row.app_id ?? null,
                 source: row.source,
-                viewId: row.view_id,
-                eventType: row.event_type,
-                isUnique: row.is_unique === 1 || row.is_unique === true,
+                outcome: row.outcome,
+                reason: row.reason ?? null,
+                detail: row.detail ?? null,
+                hostname: row.hostname ?? null,
+                viewId: row.view_id ?? null,
+                eventType: row.event_type ?? null,
+                requests: Number(row.requests),
             })),
-            total: Number(count[0]?.count || 0),
+            total,
+        };
+    }
+
+    /**
+     * How the last `hours` of tracking requests turned out: requests per
+     * outcome, and per reason for the ones not stored.
+     *
+     * @param {{ hours: number, appId?: string }} query
+     * @returns {Promise<{ hours: number, outcomes: Record<string, number>, reasons: Record<string, number> }>}
+     */
+    async trackingSummary({ hours, appId }) {
+        const appClause = appId ? ' AND app_id = ?' : '';
+        const appParams = appId ? [appId] : [];
+        const [acceptedRows] = await this.pool.query(
+            `SELECT COALESCE(SUM(is_unique = 1), 0) AS recorded, COALESCE(SUM(is_unique = 0), 0) AS repeats
+             FROM \`${VIEW_LOG_TABLE}\` WHERE created_at >= DATE_SUB(NOW(3), INTERVAL ? HOUR)${appClause}`,
+            [hours, ...appParams]
+        );
+        const [reasonRows] = await this.pool.query(
+            `SELECT reason, SUM(requests) AS requests FROM \`${TRACKING_REJECTIONS_TABLE}\`
+             WHERE minute >= DATE_SUB(NOW(), INTERVAL ? HOUR)${appClause}
+             GROUP BY reason ORDER BY reason`,
+            [hours, ...appParams]
+        );
+
+        const reasons = Object.fromEntries(reasonRows.map((row) => [row.reason, Number(row.requests)]));
+        const bots = reasons[REJECTION_REASON.BOT] || 0;
+        const rejectedTotal = Object.values(reasons).reduce((sum, value) => sum + value, 0) - bots;
+        return {
+            hours,
+            outcomes: {
+                [TRACKING_OUTCOME.RECORDED]: Number(acceptedRows[0]?.recorded || 0),
+                [TRACKING_OUTCOME.REPEAT]: Number(acceptedRows[0]?.repeats || 0),
+                [TRACKING_OUTCOME.BOT]: bots,
+                [TRACKING_OUTCOME.REJECTED]: rejectedTotal,
+            },
+            reasons,
         };
     }
 }

package/db/adminSchema.js

@@ -10,8 +10,10 @@ const crypto = require('crypto');
 
 const {
     ADMIN_LOG_TABLE,
+    ADMIN_SESSIONS_TABLE,
     DATABASE,
     FIELD_MAX_LENGTH,
+    TRACKING_REJECTIONS_TABLE,
     VIEW_LOG_TABLE,
 } = require('../constants');
 const { getError, logWarning, ErrorType, WarningType } = require('../utils/errorUtils');
@@ -36,11 +38,35 @@ const ADMIN_COLUMNS = [
     { name: 'deleted_at', ddl: 'DATETIME DEFAULT NULL' },
 ];
 
+/**
+ * Columns 3.2 added for richer, still identifier-free analysis: which of the
+ * app's sites and which language, the campaign tags of the landing URL, an
+ * optional region and city (only with a city database configured), and how
+ * long the page was visible and how far it was scrolled. 3.3 added
+ * `last_seen_at`: when the page last reported its engagement, which is how
+ * "right now" still counts a visitor who has been reading one page for a while.
+ */
+const TRACKING_COLUMNS = [
+    { name: 'hostname', ddl: `VARCHAR(${FIELD_MAX_LENGTH.HOSTNAME}) DEFAULT NULL` },
+    { name: 'language', ddl: `VARCHAR(${FIELD_MAX_LENGTH.LANGUAGE}) DEFAULT NULL` },
+    { name: 'utm_source', ddl: `VARCHAR(${FIELD_MAX_LENGTH.UTM}) DEFAULT NULL` },
+    { name: 'utm_medium', ddl: `VARCHAR(${FIELD_MAX_LENGTH.UTM}) DEFAULT NULL` },
+    { name: 'utm_campaign', ddl: `VARCHAR(${FIELD_MAX_LENGTH.UTM}) DEFAULT NULL` },
+    { name: 'utm_term', ddl: `VARCHAR(${FIELD_MAX_LENGTH.UTM}) DEFAULT NULL` },
+    { name: 'utm_content', ddl: `VARCHAR(${FIELD_MAX_LENGTH.UTM}) DEFAULT NULL` },
+    { name: 'region', ddl: `VARCHAR(${FIELD_MAX_LENGTH.REGION}) DEFAULT NULL` },
+    { name: 'city', ddl: `VARCHAR(${FIELD_MAX_LENGTH.CITY}) DEFAULT NULL` },
+    { name: 'engaged_ms', ddl: 'INT UNSIGNED DEFAULT NULL' },
+    { name: 'scroll_depth', ddl: 'TINYINT UNSIGNED DEFAULT NULL' },
+    { name: 'last_seen_at', ddl: 'DATETIME DEFAULT NULL' },
+];
+
 /** Indexes the admin columns need, keyed by index name. */
 const ADMIN_INDEXES = {
     uq_public_id: 'UNIQUE INDEX `uq_public_id` (`public_id`)',
     idx_deleted_at: 'INDEX `idx_deleted_at` (`deleted_at`)',
     idx_admin_modified_at: 'INDEX `idx_admin_modified_at` (`admin_modified_at`)',
+    idx_last_seen_at: 'INDEX `idx_last_seen_at` (`last_seen_at`)',
 };
 
 /**
@@ -83,11 +109,60 @@ const VIEW_LOG_DDL = `
         \`view_id\` CHAR(${FIELD_MAX_LENGTH.UUID}) NOT NULL,
         \`event_type\` VARCHAR(${FIELD_MAX_LENGTH.EVENT_TYPE}) DEFAULT NULL,
         \`is_unique\` TINYINT(1) NOT NULL,
+        \`hostname\` VARCHAR(${FIELD_MAX_LENGTH.HOSTNAME}) DEFAULT NULL,
         INDEX \`idx_created_at\` (\`created_at\`),
         INDEX \`idx_app_created\` (\`app_id\`, \`created_at\`)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
 `;
 
+/**
+ * Admin sign-ins, so a restart or deploy does not sign anyone out.
+ *
+ * Holds only the SHA-256 of each session's token (the browser has the token
+ * itself, in an HttpOnly cookie), so reading this table yields no usable
+ * session. The CSRF token is derived from the session token, not stored.
+ * `password_at` is when the password was last entered, for actions that ask
+ * for it again.
+ */
+const ADMIN_SESSIONS_DDL = `
+    CREATE TABLE IF NOT EXISTS \`${ADMIN_SESSIONS_TABLE}\` (
+        \`token_hash\` CHAR(64) PRIMARY KEY,
+        \`id\` CHAR(${FIELD_MAX_LENGTH.UUID}) NOT NULL,
+        \`created_at\` DATETIME(3) NOT NULL,
+        \`last_seen_at\` DATETIME(3) NOT NULL,
+        \`password_at\` DATETIME(3) NOT NULL,
+        INDEX \`idx_last_seen_at\` (\`last_seen_at\`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+`;
+
+/** Columns the view log gained after it first shipped, for tables created by 3.1. */
+const VIEW_LOG_ADDED_COLUMNS = [
+    { name: 'hostname', ddl: `VARCHAR(${FIELD_MAX_LENGTH.HOSTNAME}) DEFAULT NULL` },
+];
+
+/**
+ * Tracking requests that were not stored: bots and rejections, with why.
+ *
+ * Counted per minute rather than kept one by one, so a flood of bad requests
+ * adds to a counter instead of a row each, and nothing about the requester
+ * (no IP, hash, or User-Agent) is kept. The key columns use '' rather than
+ * NULL for "none", because NULL never matches itself in a unique key and every
+ * NULL would start a new row.
+ */
+const TRACKING_REJECTIONS_DDL = `
+    CREATE TABLE IF NOT EXISTS \`${TRACKING_REJECTIONS_TABLE}\` (
+        \`minute\` DATETIME NOT NULL,
+        \`source\` VARCHAR(16) NOT NULL,
+        \`reason\` VARCHAR(32) NOT NULL,
+        \`app_id\` VARCHAR(64) NOT NULL DEFAULT '',
+        \`detail\` VARCHAR(64) NOT NULL DEFAULT '',
+        \`hostname\` VARCHAR(${FIELD_MAX_LENGTH.HOSTNAME}) NOT NULL DEFAULT '',
+        \`requests\` INT UNSIGNED NOT NULL,
+        PRIMARY KEY (\`minute\`, \`source\`, \`reason\`, \`app_id\`, \`detail\`, \`hostname\`),
+        INDEX \`idx_app_minute\` (\`app_id\`, \`minute\`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+`;
+
 /**
  * Column and index definitions for a brand-new app table, spliced into
  * `appTableDDL` so a fresh table is born in the migrated shape.
@@ -97,6 +172,7 @@ const NEW_TABLE_ADMIN_COLUMNS = [
     `\`note\` VARCHAR(${FIELD_MAX_LENGTH.NOTE}) DEFAULT NULL`,
     '`admin_modified_at` DATETIME DEFAULT NULL',
     '`deleted_at` DATETIME DEFAULT NULL',
+    ...TRACKING_COLUMNS.map((column) => `\`${column.name}\` ${column.ddl}`),
 ];
 
 const NEW_TABLE_ADMIN_INDEXES = Object.values(ADMIN_INDEXES);
@@ -192,7 +268,7 @@ async function migrateAppTable(pool, appId) {
             return { migrated: false, backfilled: 0 };
         }
 
-        for (const column of ADMIN_COLUMNS) {
+        for (const column of [...ADMIN_COLUMNS, ...TRACKING_COLUMNS]) {
             if (!columns.has(column.name)) {
                 await pool.query(`ALTER TABLE \`${appId}\` ADD COLUMN \`${column.name}\` ${column.ddl}`);
             }
@@ -221,21 +297,34 @@ async function migrateAppTable(pool, appId) {
 }
 
 /**
- * Create the log tables. Safe in `connect` mode for the same reason the app
- * registry is: they are the service's own bookkeeping, not the operator's
- * schema.
+ * Create the service's own tables: the logs and the admin sessions. Safe in
+ * `connect` mode for the same reason the app registry is: they are the
+ * service's own bookkeeping, not the operator's schema.
  * @param {object} pool
  */
 async function ensureLogTables(pool) {
     await pool.query(ADMIN_LOG_DDL);
     await pool.query(VIEW_LOG_DDL);
+    await pool.query(TRACKING_REJECTIONS_DDL);
+    await pool.query(ADMIN_SESSIONS_DDL);
+
+    const viewLogColumns = await readColumns(pool, VIEW_LOG_TABLE);
+    for (const column of VIEW_LOG_ADDED_COLUMNS) {
+        if (!viewLogColumns.has(column.name)) {
+            await pool.query(`ALTER TABLE \`${VIEW_LOG_TABLE}\` ADD COLUMN \`${column.name}\` ${column.ddl}`);
+        }
+    }
 }
 
 module.exports = {
     ADMIN_COLUMNS,
+    TRACKING_COLUMNS,
     ADMIN_INDEXES,
     ADMIN_LOG_DDL,
     VIEW_LOG_DDL,
+    VIEW_LOG_ADDED_COLUMNS,
+    TRACKING_REJECTIONS_DDL,
+    ADMIN_SESSIONS_DDL,
     NEW_TABLE_ADMIN_COLUMNS,
     NEW_TABLE_ADMIN_INDEXES,
     backfillPublicIds,

package/db/adminSessionStore.js

@@ -0,0 +1,104 @@
+/**
+ * Admin sessions in the database, so a restart or deploy signs nobody out.
+ *
+ * Same interface as the in-memory store in middleware/adminAuth.js. The table
+ * holds each token's SHA-256, never the token. Ages are computed by the
+ * database's clock, the same clock that wrote the timestamps, and a session's
+ * last-seen time is written at most once a minute however busy the admin is.
+ */
+
+const crypto = require('crypto');
+
+const { ADMIN, ADMIN_SESSIONS_TABLE } = require('../constants');
+const { hashToken, newToken } = require('../middleware/adminAuth');
+
+const TABLE = `\`${ADMIN_SESSIONS_TABLE}\``;
+const seconds = (ms) => Math.floor(ms / 1000);
+
+/**
+ * @param {{ pool: object }} db a DatabaseManager; its pool exists once initialized
+ * @param {{ idleMs?: number, absoluteMs?: number, maxSessions?: number }} [options]
+ */
+function createDbSessionStore(db, {
+    idleMs = ADMIN.SESSION_IDLE_TIMEOUT_MS,
+    absoluteMs = ADMIN.SESSION_ABSOLUTE_TIMEOUT_MS,
+    maxSessions = ADMIN.MAX_SESSIONS,
+} = {}) {
+    const query = (sql, params) => db.pool.query(sql, params);
+
+    return {
+        idleMs,
+        absoluteMs,
+
+        async create() {
+            // Expired sessions go first, so the table never outgrows the
+            // sessions that could still be used.
+            await query(
+                `DELETE FROM ${TABLE}
+                 WHERE last_seen_at < DATE_SUB(NOW(3), INTERVAL ? SECOND)
+                    OR created_at < DATE_SUB(NOW(3), INTERVAL ? SECOND)`,
+                [seconds(idleMs), seconds(absoluteMs)]
+            );
+            const token = newToken();
+            const id = crypto.randomUUID();
+            await query(
+                `INSERT INTO ${TABLE} (token_hash, id, created_at, last_seen_at, password_at)
+                 VALUES (?, ?, NOW(3), NOW(3), NOW(3))`,
+                [hashToken(token), id]
+            );
+            // Bounded like the in-memory store: beyond the limit, the least
+            // recently used sessions end. The derived table lets MySQL read
+            // the table it is deleting from.
+            await query(
+                `DELETE FROM ${TABLE} WHERE token_hash NOT IN (
+                    SELECT token_hash FROM (
+                        SELECT token_hash FROM ${TABLE} ORDER BY last_seen_at DESC, created_at DESC LIMIT ?
+                    ) AS newest
+                 )`,
+                [maxSessions]
+            );
+            return { token, session: { id, passwordAgeMs: 0 } };
+        },
+
+        async get(token) {
+            if (typeof token !== 'string' || token.length === 0) return null;
+            const key = hashToken(token);
+            const [rows] = await query(
+                `SELECT id,
+                    TIMESTAMPDIFF(SECOND, created_at, NOW(3)) AS age_s,
+                    TIMESTAMPDIFF(SECOND, last_seen_at, NOW(3)) AS idle_s,
+                    TIMESTAMPDIFF(SECOND, password_at, NOW(3)) AS password_age_s
+                 FROM ${TABLE} WHERE token_hash = ?`,
+                [key]
+            );
+            const row = rows[0];
+            if (!row) return null;
+
+            const idle = Number(row.idle_s) * 1000;
+            if (idle > idleMs || Number(row.age_s) * 1000 > absoluteMs) {
+                await query(`DELETE FROM ${TABLE} WHERE token_hash = ?`, [key]);
+                return null;
+            }
+            if (idle >= ADMIN.SESSION_TOUCH_INTERVAL_MS) {
+                await query(`UPDATE ${TABLE} SET last_seen_at = NOW(3) WHERE token_hash = ?`, [key]);
+            }
+            return { id: row.id, passwordAgeMs: Number(row.password_age_s) * 1000 };
+        },
+
+        async destroy(token) {
+            if (typeof token !== 'string' || token.length === 0) return false;
+            const [result] = await query(`DELETE FROM ${TABLE} WHERE token_hash = ?`, [hashToken(token)]);
+            return result.affectedRows > 0;
+        },
+
+        async confirmPassword(token) {
+            if (typeof token !== 'string' || token.length === 0) return;
+            await query(
+                `UPDATE ${TABLE} SET password_at = NOW(3), last_seen_at = NOW(3) WHERE token_hash = ?`,
+                [hashToken(token)]
+            );
+        },
+    };
+}
+
+module.exports = { createDbSessionStore };