@harshankur/viewcounter 3.0.0 → 3.1.0

package/db/DatabaseManager.js

@@ -10,12 +10,28 @@ const {
     SERVER,
     TOP_N_RESULTS,
     TREND_PERIOD,
+    VIEW_LOG_SOURCE,
 } = require('../constants');
+
+/**
+ * Every analytics read and the duplicate check see only live rows. A view an
+ * admin has moved to the trash no longer counts anywhere, and one that is
+ * restored counts again, without any stored aggregate needing to be rebuilt.
+ */
+const LIVE_ROW = 'deleted_at IS NULL';
 const PrivacyUtils = require('../utils/privacyUtils');
 const logger = require('../utils/logger');
 const { getError, logWarning, ErrorType, WarningType } = require('../utils/errorUtils');
 const { truncate } = require('../utils/stringUtils');
 const { isValidAppId } = require('../utils/appIdUtils');
+const {
+    NEW_TABLE_ADMIN_COLUMNS,
+    NEW_TABLE_ADMIN_INDEXES,
+    ensureLogTables,
+    migrateAppTable,
+} = require('./adminSchema');
+const LogRepository = require('./LogRepository');
+const AdminRepository = require('./AdminRepository');
 
 /**
  * Columns returned for a session lookup.
@@ -99,6 +115,7 @@ function appTableDDL(appId) {
             \`event_type\` VARCHAR(${FIELD_MAX_LENGTH.EVENT_TYPE}) DEFAULT '${EVENT_TYPE.PAGEVIEW}',
             \`event_data\` JSON DEFAULT NULL,
             \`is_unique\` TINYINT(1) DEFAULT 1,
+            ${NEW_TABLE_ADMIN_COLUMNS.join(',\n            ')},
             INDEX \`idx_timestamp\` (\`timestamp\`),
             INDEX \`idx_visitor_timestamp\` (\`visitor_hash\`, \`timestamp\`),
             INDEX \`idx_masked_ip\` (\`masked_ip\`),
@@ -112,7 +129,8 @@ function appTableDDL(appId) {
             INDEX \`idx_device_type\` (\`device_type\`),
             INDEX \`idx_session_id\` (\`session_id\`),
             INDEX \`idx_event_type\` (\`event_type\`),
-            INDEX \`idx_is_unique\` (\`is_unique\`)
+            INDEX \`idx_is_unique\` (\`is_unique\`),
+            ${NEW_TABLE_ADMIN_INDEXES.join(',\n            ')}
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
     `;
 }
@@ -161,6 +179,29 @@ class DatabaseManager {
         this.config = config;
         this.pool = null;
         this.mode = config.mode || 'connect';
+        this.logs = new LogRepository(this);
+        this.admin = new AdminRepository(this);
+    }
+
+    /**
+     * Bring the service's own tables and every app table up to the current
+     * schema. Idempotent, and run on every start, so an upgrade needs no manual
+     * migration step. Runs in `connect` mode too: the admin columns and log
+     * tables are the service's bookkeeping, like the app registry.
+     *
+     * @param {string[]} appIds
+     * @throws {Error} ErrorType.MIGRATION_FAILED
+     */
+    async migrate(appIds = []) {
+        this.assertReady();
+        await ensureLogTables(this.pool);
+
+        for (const appId of appIds) {
+            const result = await migrateAppTable(this.pool, appId);
+            if (result.backfilled > 0) {
+                logger.info(`Assigned public IDs to ${result.backfilled} existing row(s) in '${appId}'`);
+            }
+        }
     }
 
     /** @throws {Error} when a query is attempted before initialize() */
@@ -171,7 +212,14 @@ class DatabaseManager {
     }
 
     /**
-     * Initialize database connection and optionally create schema
+     * Connect, create the schema in create mode, and bring the given apps'
+     * tables up to the current schema in either mode.
+     *
+     * The migration belongs here, not only at server startup: every query this
+     * manager runs assumes the current schema (public_id, deleted_at, the log
+     * tables), so an application that mounts the routers and calls only
+     * initialize() must still get tables those queries can run against.
+     * Idempotent: a table already in shape is read and left alone.
      */
     async initialize(allowedAppIds = []) {
         try {
@@ -206,8 +254,6 @@ class DatabaseManager {
 
             await this.pool.query('SELECT 1');
             logger.info(`Database connected (mode: ${this.mode})`);
-
-            return true;
         } catch (cause) {
             throw getError(ErrorType.DATABASE_CONNECTION_FAILED, {
                 host: this.config.host,
@@ -216,6 +262,11 @@ class DatabaseManager {
                 cause: cause.message,
             });
         }
+
+        // Outside the try: a failed migration is reported as MIGRATION_FAILED
+        // naming the table, not as a connection failure.
+        await this.migrate(allowedAppIds);
+        return true;
     }
 
     /**
@@ -320,8 +371,10 @@ class DatabaseManager {
         );
 
         // The table is (re)created regardless, so an app registered before its
-        // table existed still converges to a working state.
+        // table existed still converges to a working state. A table that
+        // already existed in an older shape is migrated in place.
         await this.pool.query(appTableDDL(appId));
+        await migrateAppTable(this.pool, appId);
 
         if (existing.length > 0) {
             logWarning(WarningType.APP_ALREADY_REGISTERED, { appId });
@@ -389,6 +442,7 @@ class DatabaseManager {
             uniqueWindowHours = SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS,
             userAgent = '',
             visitorSecret,
+            source = VIEW_LOG_SOURCE.REGISTER_VIEW,
         } = data;
 
         // Privacy boundary. Neither the raw IP nor the raw User-Agent is bound
@@ -407,6 +461,7 @@ class DatabaseManager {
             const [existing] = await this.pool.query(
                 `SELECT id FROM \`${appId}\`
                  WHERE visitor_hash = ? AND event_type = ? AND timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)
+                   AND ${LIVE_ROW}
                  LIMIT 1`,
                 [hashedVisitor, EVENT_TYPE.PAGEVIEW, uniqueWindowHours]
             );
@@ -416,15 +471,19 @@ class DatabaseManager {
             }
         }
 
+        const publicId = crypto.randomUUID();
+        const storedEventType = truncate(eventType, FIELD_MAX_LENGTH.EVENT_TYPE);
+
         const [result] = await this.pool.query(
             `INSERT INTO \`${appId}\` (
-                masked_ip, visitor_hash, country, timestamp, devicesize,
+                public_id, masked_ip, visitor_hash, country, timestamp, devicesize,
                 page_path, page_title,
                 referrer, referrer_domain, source_type,
                 browser, browser_version, os, os_version, device_type,
                 session_id, event_type, event_data, is_unique
-            ) VALUES (?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+            ) VALUES (?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
             [
+                publicId,
                 truncate(maskedIp, FIELD_MAX_LENGTH.MASKED_IP),
                 hashedVisitor,
                 truncate(country, FIELD_MAX_LENGTH.COUNTRY),
@@ -440,15 +499,26 @@ class DatabaseManager {
                 truncate(osVersion, FIELD_MAX_LENGTH.OS_VERSION),
                 truncate(deviceType, FIELD_MAX_LENGTH.DEVICE_TYPE),
                 truncate(sessionId, FIELD_MAX_LENGTH.SESSION_ID),
-                truncate(eventType, FIELD_MAX_LENGTH.EVENT_TYPE),
+                storedEventType,
                 eventData ? JSON.stringify(eventData) : null,
                 isUnique,
             ]
         );
 
+        // The view register log. Written after the row, and never able to fail
+        // the request: the view is already stored.
+        await this.logs.writeViewLog({
+            appId,
+            source,
+            viewId: publicId,
+            eventType: storedEventType,
+            isUnique: isUnique === 1,
+        });
+
         return {
             duplicate: isUnique === 0,
             insertId: result.insertId,
+            publicId,
             isUnique: isUnique === 1,
         };
     }
@@ -477,14 +547,15 @@ class DatabaseManager {
                 COUNT(*) as total_views,
                 SUM(CASE WHEN is_unique = 1 THEN 1 ELSE 0 END) as unique_views,
                 COUNT(DISTINCT visitor_hash) as unique_visitors
-             FROM \`${appId}\``
+             FROM \`${appId}\`
+             WHERE ${LIVE_ROW}`
         );
 
         const stats = totalStats[0];
 
         const [byCountry] = await this.pool.query(
             `SELECT country, COUNT(*) as count FROM \`${appId}\`
-             WHERE country IS NOT NULL
+             WHERE country IS NOT NULL AND ${LIVE_ROW}
              GROUP BY country
              ORDER BY count DESC
              LIMIT ?`,
@@ -493,13 +564,14 @@ class DatabaseManager {
 
         const [byDevice] = await this.pool.query(
             `SELECT devicesize, COUNT(*) as count FROM \`${appId}\`
+             WHERE ${LIVE_ROW}
              GROUP BY devicesize
              ORDER BY count DESC`
         );
 
         const [recent] = await this.pool.query(
             `SELECT COUNT(*) as count FROM \`${appId}\`
-             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)`,
+             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR) AND ${LIVE_ROW}`,
             [SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS]
         );
 
@@ -522,13 +594,14 @@ class DatabaseManager {
         const [views] = await this.pool.query(
             `SELECT masked_ip, country, timestamp, devicesize
              FROM \`${appId}\`
+             WHERE ${LIVE_ROW}
              ORDER BY timestamp DESC
              LIMIT ? OFFSET ?`,
             [limit, offset]
         );
 
         const [total] = await this.pool.query(
-            `SELECT COUNT(*) as count FROM \`${appId}\``
+            `SELECT COUNT(*) as count FROM \`${appId}\` WHERE ${LIVE_ROW}`
         );
 
         return {
@@ -560,7 +633,7 @@ class DatabaseManager {
         const [trends] = await this.pool.query(
             `SELECT ${groupBy} as period, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? DAY)
+             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? DAY) AND ${LIVE_ROW}
              GROUP BY period
              ORDER BY period ASC`,
             [days]
@@ -578,7 +651,7 @@ class DatabaseManager {
         const [bySource] = await this.pool.query(
             `SELECT source_type, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE source_type IS NOT NULL
+             WHERE source_type IS NOT NULL AND ${LIVE_ROW}
              GROUP BY source_type
              ORDER BY count DESC`
         );
@@ -586,7 +659,7 @@ class DatabaseManager {
         const [byDomain] = await this.pool.query(
             `SELECT referrer_domain, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE referrer_domain IS NOT NULL
+             WHERE referrer_domain IS NOT NULL AND ${LIVE_ROW}
              GROUP BY referrer_domain
              ORDER BY count DESC
              LIMIT ?`,
@@ -605,7 +678,7 @@ class DatabaseManager {
         const [byBrowser] = await this.pool.query(
             `SELECT browser, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE browser IS NOT NULL
+             WHERE browser IS NOT NULL AND ${LIVE_ROW}
              GROUP BY browser
              ORDER BY count DESC
              LIMIT ?`,
@@ -615,7 +688,7 @@ class DatabaseManager {
         const [byOS] = await this.pool.query(
             `SELECT os, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE os IS NOT NULL
+             WHERE os IS NOT NULL AND ${LIVE_ROW}
              GROUP BY os
              ORDER BY count DESC
              LIMIT ?`,
@@ -625,7 +698,7 @@ class DatabaseManager {
         const [byDeviceType] = await this.pool.query(
             `SELECT device_type, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE device_type IS NOT NULL
+             WHERE device_type IS NOT NULL AND ${LIVE_ROW}
              GROUP BY device_type
              ORDER BY count DESC`
         );
@@ -642,7 +715,7 @@ class DatabaseManager {
         const [pages] = await this.pool.query(
             `SELECT page_path, page_title, COUNT(*) as views
              FROM \`${appId}\`
-             WHERE page_path IS NOT NULL
+             WHERE page_path IS NOT NULL AND ${LIVE_ROW}
              GROUP BY page_path, page_title
              ORDER BY views DESC
              LIMIT ?`,
@@ -662,7 +735,7 @@ class DatabaseManager {
         const [events] = await this.pool.query(
             `SELECT ${SESSION_COLUMNS}
              FROM \`${appId}\`
-             WHERE session_id = ?
+             WHERE session_id = ? AND ${LIVE_ROW}
              ORDER BY timestamp ASC`,
             [sessionId]
         );
@@ -702,3 +775,4 @@ module.exports = DatabaseManager;
 module.exports.SESSION_COLUMNS = SESSION_COLUMNS;
 module.exports.appTableDDL = appTableDDL;
 module.exports.APP_REGISTRY_DDL = APP_REGISTRY_DDL;
+module.exports.LIVE_ROW = LIVE_ROW;

package/db/LogRepository.js

@@ -0,0 +1,217 @@
+/**
+ * The two service logs: the admin operation log and the view register log.
+ *
+ * Both are append-only from the application's point of view. Nothing here
+ * updates or deletes an entry.
+ */
+
+const crypto = require('crypto');
+
+const { ADMIN, ADMIN_LOG_TABLE, VIEW_LOG_TABLE } = require('../constants');
+const { logWarning, WarningType } = require('../utils/errorUtils');
+
+/**
+ * Columns returned by an admin-log listing. `session_id` identifies which
+ * admin session acted without revealing its token.
+ */
+const ADMIN_LOG_COLUMNS = [
+    'id', 'created_at', 'action', 'session_id', 'masked_ip',
+    'app_id', 'target_count', 'target_ids', 'fields',
+].join(', ');
+
+const VIEW_LOG_COLUMNS = [
+    'id', 'created_at', 'app_id', 'source', 'view_id', 'event_type', 'is_unique',
+].join(', ');
+
+/** mysql2 returns JSON columns parsed; tolerate a string from other drivers. */
+function parseJson(value) {
+    if (value === null || value === undefined) return null;
+    if (typeof value !== 'string') return value;
+    try {
+        return JSON.parse(value);
+    } catch {
+        return null;
+    }
+}
+
+class LogRepository {
+    /**
+     * @param {{ pool: object, assertReady: () => void }} db a DatabaseManager
+     */
+    constructor(db) {
+        this.db = db;
+    }
+
+    get pool() {
+        this.db.assertReady();
+        return this.db.pool;
+    }
+
+    /**
+     * Record an admin operation. Never throws: an operation that succeeded must
+     * not be reported as failed because its log line could not be written
+     * (LOGGING.md §4). The failure is surfaced as a warning instead.
+     *
+     * `targetCount` defaults to the number of IDs; the automatic trash purge
+     * passes a count with no IDs, since it erases by age rather than by ID.
+     *
+     * @param {{ action: string, sessionId?: string|null, maskedIp?: string|null,
+     *   appId?: string|null, targetIds?: string[], targetCount?: number,
+     *   fields?: string[] }} entry
+     * @returns {Promise<boolean>} whether the entry was written
+     */
+    async writeAdminLog({
+        action,
+        sessionId = null,
+        maskedIp = null,
+        appId = null,
+        targetIds = [],
+        targetCount = targetIds.length,
+        fields = [],
+    }) {
+        try {
+            await this.pool.query(
+                `INSERT INTO \`${ADMIN_LOG_TABLE}\`
+                    (id, created_at, action, session_id, masked_ip, app_id, target_count, target_ids, fields)
+                 VALUES (?, NOW(3), ?, ?, ?, ?, ?, ?, ?)`,
+                [
+                    crypto.randomUUID(),
+                    action,
+                    sessionId,
+                    maskedIp,
+                    appId,
+                    targetCount,
+                    targetIds.length ? JSON.stringify(targetIds) : null,
+                    fields.length ? JSON.stringify(fields) : null,
+                ]
+            );
+            return true;
+        } catch (cause) {
+            logWarning(WarningType.ADMIN_LOG_WRITE_FAILED, { action, cause: cause.message });
+            return false;
+        }
+    }
+
+    /**
+     * Record one accepted view or event. Never throws, for the same reason as
+     * writeAdminLog: the view itself was stored, so the visitor's request
+     * succeeded.
+     *
+     * @param {{ appId: string, source: string, viewId: string, eventType: string,
+     *   isUnique: boolean }} entry
+     * @returns {Promise<boolean>}
+     */
+    async writeViewLog({ appId, source, viewId, eventType, isUnique }) {
+        try {
+            await this.pool.query(
+                `INSERT INTO \`${VIEW_LOG_TABLE}\`
+                    (id, created_at, app_id, source, view_id, event_type, is_unique)
+                 VALUES (?, NOW(3), ?, ?, ?, ?, ?)`,
+                [crypto.randomUUID(), appId, source, viewId, eventType, isUnique ? 1 : 0]
+            );
+            return true;
+        } catch (cause) {
+            logWarning(WarningType.VIEW_LOG_WRITE_FAILED, { appId, cause: cause.message });
+            return false;
+        }
+    }
+
+    /**
+     * Remove view-log entries older than `days`, a batch at a time so no single
+     * statement holds its locks for long. Ordered by the indexed `created_at`,
+     * so each batch reads only what it deletes.
+     *
+     * @param {number} days
+     * @returns {Promise<number>} entries removed
+     */
+    async pruneViewLog(days) {
+        let total = 0;
+        for (;;) {
+            const [result] = await this.pool.query(
+                `DELETE FROM \`${VIEW_LOG_TABLE}\` WHERE created_at < DATE_SUB(NOW(3), INTERVAL ? DAY)
+                 ORDER BY created_at LIMIT ?`,
+                [days, ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE]
+            );
+            const removed = Number(result?.affectedRows || 0);
+            total += removed;
+            if (removed < ADMIN.VIEW_LOG_PRUNE_BATCH_SIZE) return total;
+        }
+    }
+
+    /**
+     * @param {{ page: number, pageSize: number, action?: string, appId?: string }} query
+     * @returns {Promise<{ entries: object[], total: number }>}
+     */
+    async listAdminLog({ page, pageSize, action, appId }) {
+        const where = [];
+        const params = [];
+        if (action) { where.push('action = ?'); params.push(action); }
+        if (appId) { where.push('app_id = ?'); params.push(appId); }
+        const clause = where.length ? `WHERE ${where.join(' AND ')}` : '';
+
+        const [rows] = await this.pool.query(
+            `SELECT ${ADMIN_LOG_COLUMNS} FROM \`${ADMIN_LOG_TABLE}\` ${clause}
+             ORDER BY created_at DESC, id ASC LIMIT ? OFFSET ?`,
+            [...params, pageSize, (page - 1) * pageSize]
+        );
+        const [count] = await this.pool.query(
+            `SELECT COUNT(*) AS count FROM \`${ADMIN_LOG_TABLE}\` ${clause}`,
+            params
+        );
+
+        return {
+            entries: rows.map((row) => ({
+                id: row.id,
+                createdAt: row.created_at,
+                action: row.action,
+                sessionId: row.session_id,
+                maskedIp: row.masked_ip,
+                appId: row.app_id,
+                targetCount: row.target_count,
+                targetIds: parseJson(row.target_ids) || [],
+                fields: parseJson(row.fields) || [],
+            })),
+            total: Number(count[0]?.count || 0),
+        };
+    }
+
+    /**
+     * @param {{ page: number, pageSize: number, appId?: string, source?: string }} query
+     * @returns {Promise<{ entries: object[], total: number }>}
+     */
+    async listViewLog({ page, pageSize, appId, source }) {
+        const where = [];
+        const params = [];
+        if (appId) { where.push('app_id = ?'); params.push(appId); }
+        if (source) { where.push('source = ?'); params.push(source); }
+        const clause = where.length ? `WHERE ${where.join(' AND ')}` : '';
+
+        const [rows] = await this.pool.query(
+            `SELECT ${VIEW_LOG_COLUMNS} FROM \`${VIEW_LOG_TABLE}\` ${clause}
+             ORDER BY created_at DESC, id ASC LIMIT ? OFFSET ?`,
+            [...params, pageSize, (page - 1) * pageSize]
+        );
+        const [count] = await this.pool.query(
+            `SELECT COUNT(*) AS count FROM \`${VIEW_LOG_TABLE}\` ${clause}`,
+            params
+        );
+
+        return {
+            entries: rows.map((row) => ({
+                id: row.id,
+                createdAt: row.created_at,
+                appId: row.app_id,
+                source: row.source,
+                viewId: row.view_id,
+                eventType: row.event_type,
+                isUnique: row.is_unique === 1 || row.is_unique === true,
+            })),
+            total: Number(count[0]?.count || 0),
+        };
+    }
+}
+
+module.exports = LogRepository;
+module.exports.ADMIN_LOG_COLUMNS = ADMIN_LOG_COLUMNS;
+module.exports.VIEW_LOG_COLUMNS = VIEW_LOG_COLUMNS;
+module.exports.parseJson = parseJson;