@harshankur/viewcounter 3.0.1 → 3.2.0

package/db/DatabaseManager.js

@@ -9,19 +9,36 @@ const {
     QUERY_LIMITS,
     SERVER,
     TOP_N_RESULTS,
+    TRACKING,
     TREND_PERIOD,
+    VIEW_LOG_SOURCE,
 } = require('../constants');
+
+/**
+ * Every analytics read and the duplicate check see only live rows. A view an
+ * admin has moved to the trash no longer counts anywhere, and one that is
+ * restored counts again, without any stored aggregate needing to be rebuilt.
+ */
+const LIVE_ROW = 'deleted_at IS NULL';
 const PrivacyUtils = require('../utils/privacyUtils');
 const logger = require('../utils/logger');
 const { getError, logWarning, ErrorType, WarningType } = require('../utils/errorUtils');
 const { truncate } = require('../utils/stringUtils');
 const { isValidAppId } = require('../utils/appIdUtils');
+const {
+    NEW_TABLE_ADMIN_COLUMNS,
+    NEW_TABLE_ADMIN_INDEXES,
+    ensureLogTables,
+    migrateAppTable,
+} = require('./adminSchema');
+const LogRepository = require('./LogRepository');
+const AdminRepository = require('./AdminRepository');
 
 /**
  * Columns returned for a session lookup.
  *
  * Deliberately explicit rather than `SELECT *`. The previous wildcard returned
- * `visitor_hash` — the pseudonymous visitor identifier itself — to any caller
+ * `visitor_hash` (the pseudonymous visitor identifier itself) to any caller
  * of the sessions endpoint.
  */
 const SESSION_COLUMNS = [
@@ -49,7 +66,7 @@ const SESSION_COLUMNS = [
  *
  * `id` is a generated UUID and is the row's identity; `app_id` is a uniqueness
  * *constraint*, not an identity (CODE_STANDARDS.md §8). The distinction matters
- * the first time an app is renamed, or deleted and a later one reuses the name —
+ * the first time an app is renamed, or deleted and a later one reuses the name:
  * with the natural key as the primary key, anything referencing the old row
  * would silently re-point at the new one.
  */
@@ -66,7 +83,7 @@ const APP_REGISTRY_DDL = `
  * DDL for one app's event table.
  *
  * `appId` is interpolated because MySQL cannot bind an identifier as a
- * parameter. Every caller must have passed it through `isValidAppId` first —
+ * parameter. Every caller must have passed it through `isValidAppId` first;
  * the assertion below is the backstop, not the primary gate.
  *
  * @param {string} appId
@@ -99,6 +116,7 @@ function appTableDDL(appId) {
             \`event_type\` VARCHAR(${FIELD_MAX_LENGTH.EVENT_TYPE}) DEFAULT '${EVENT_TYPE.PAGEVIEW}',
             \`event_data\` JSON DEFAULT NULL,
             \`is_unique\` TINYINT(1) DEFAULT 1,
+            ${NEW_TABLE_ADMIN_COLUMNS.join(',\n            ')},
             INDEX \`idx_timestamp\` (\`timestamp\`),
             INDEX \`idx_visitor_timestamp\` (\`visitor_hash\`, \`timestamp\`),
             INDEX \`idx_masked_ip\` (\`masked_ip\`),
@@ -112,7 +130,8 @@ function appTableDDL(appId) {
             INDEX \`idx_device_type\` (\`device_type\`),
             INDEX \`idx_session_id\` (\`session_id\`),
             INDEX \`idx_event_type\` (\`event_type\`),
-            INDEX \`idx_is_unique\` (\`is_unique\`)
+            INDEX \`idx_is_unique\` (\`is_unique\`),
+            ${NEW_TABLE_ADMIN_INDEXES.join(',\n            ')}
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
     `;
 }
@@ -123,7 +142,7 @@ function appTableDDL(appId) {
  * `mysql2/promise`'s pool emits the RAW callback-style connection on its
  * `connection` event, not the promise-wrapped one. Its `query()` returns a
  * `Query`, and mysql2 deliberately makes `.then()`/`.catch()` on a `Query`
- * throw — so treating it as a promise crashes the process on the very first
+ * throw, so treating it as a promise crashes the process on the very first
  * database connection. The callback form is the correct API for that object.
  *
  * Failure is swallowed on purpose: MariaDB and MySQL < 5.7.8 have no
@@ -161,6 +180,29 @@ class DatabaseManager {
         this.config = config;
         this.pool = null;
         this.mode = config.mode || 'connect';
+        this.logs = new LogRepository(this);
+        this.admin = new AdminRepository(this);
+    }
+
+    /**
+     * Bring the service's own tables and every app table up to the current
+     * schema. Idempotent, and run on every start, so an upgrade needs no manual
+     * migration step. Runs in `connect` mode too: the admin columns and log
+     * tables are the service's bookkeeping, like the app registry.
+     *
+     * @param {string[]} appIds
+     * @throws {Error} ErrorType.MIGRATION_FAILED
+     */
+    async migrate(appIds = []) {
+        this.assertReady();
+        await ensureLogTables(this.pool);
+
+        for (const appId of appIds) {
+            const result = await migrateAppTable(this.pool, appId);
+            if (result.backfilled > 0) {
+                logger.info(`Assigned public IDs to ${result.backfilled} existing row(s) in '${appId}'`);
+            }
+        }
     }
 
     /** @throws {Error} when a query is attempted before initialize() */
@@ -171,7 +213,14 @@ class DatabaseManager {
     }
 
     /**
-     * Initialize database connection and optionally create schema
+     * Connect, create the schema in create mode, and bring the given apps'
+     * tables up to the current schema in either mode.
+     *
+     * The migration belongs here, not only at server startup: every query this
+     * manager runs assumes the current schema (public_id, deleted_at, the log
+     * tables), so an application that mounts the routers and calls only
+     * initialize() must still get tables those queries can run against.
+     * Idempotent: a table already in shape is read and left alone.
      */
     async initialize(allowedAppIds = []) {
         try {
@@ -206,8 +255,6 @@ class DatabaseManager {
 
             await this.pool.query('SELECT 1');
             logger.info(`Database connected (mode: ${this.mode})`);
-
-            return true;
         } catch (cause) {
             throw getError(ErrorType.DATABASE_CONNECTION_FAILED, {
                 host: this.config.host,
@@ -216,6 +263,11 @@ class DatabaseManager {
                 cause: cause.message,
             });
         }
+
+        // Outside the try: a failed migration is reported as MIGRATION_FAILED
+        // naming the table, not as a connection failure.
+        await this.migrate(allowedAppIds);
+        return true;
     }
 
     /**
@@ -296,7 +348,7 @@ class DatabaseManager {
     /**
      * Provision a new app: validate, create its table, record it.
      *
-     * Idempotent — re-registering an existing app is a no-op rather than an
+     * Idempotent: re-registering an existing app is a no-op rather than an
      * error, so a retried provisioning call cannot fail halfway.
      *
      * @param {string} appId
@@ -320,8 +372,10 @@ class DatabaseManager {
         );
 
         // The table is (re)created regardless, so an app registered before its
-        // table existed still converges to a working state.
+        // table existed still converges to a working state. A table that
+        // already existed in an older shape is migrated in place.
         await this.pool.query(appTableDDL(appId));
+        await migrateAppTable(this.pool, appId);
 
         if (existing.length > 0) {
             logWarning(WarningType.APP_ALREADY_REGISTERED, { appId });
@@ -386,9 +440,19 @@ class DatabaseManager {
             sessionId,
             eventType = EVENT_TYPE.PAGEVIEW,
             eventData,
+            hostname = null,
+            language = null,
+            utmSource = null,
+            utmMedium = null,
+            utmCampaign = null,
+            utmTerm = null,
+            utmContent = null,
+            region = null,
+            city = null,
             uniqueWindowHours = SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS,
             userAgent = '',
             visitorSecret,
+            source = VIEW_LOG_SOURCE.REGISTER_VIEW,
         } = data;
 
         // Privacy boundary. Neither the raw IP nor the raw User-Agent is bound
@@ -407,6 +471,7 @@ class DatabaseManager {
             const [existing] = await this.pool.query(
                 `SELECT id FROM \`${appId}\`
                  WHERE visitor_hash = ? AND event_type = ? AND timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)
+                   AND ${LIVE_ROW}
                  LIMIT 1`,
                 [hashedVisitor, EVENT_TYPE.PAGEVIEW, uniqueWindowHours]
             );
@@ -416,15 +481,21 @@ class DatabaseManager {
             }
         }
 
+        const publicId = crypto.randomUUID();
+        const storedEventType = truncate(eventType, FIELD_MAX_LENGTH.EVENT_TYPE);
+
         const [result] = await this.pool.query(
             `INSERT INTO \`${appId}\` (
-                masked_ip, visitor_hash, country, timestamp, devicesize,
+                public_id, masked_ip, visitor_hash, country, timestamp, devicesize,
                 page_path, page_title,
                 referrer, referrer_domain, source_type,
                 browser, browser_version, os, os_version, device_type,
-                session_id, event_type, event_data, is_unique
-            ) VALUES (?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+                session_id, event_type, event_data, is_unique,
+                hostname, language, utm_source, utm_medium, utm_campaign, utm_term, utm_content,
+                region, city
+            ) VALUES (?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
             [
+                publicId,
                 truncate(maskedIp, FIELD_MAX_LENGTH.MASKED_IP),
                 hashedVisitor,
                 truncate(country, FIELD_MAX_LENGTH.COUNTRY),
@@ -440,19 +511,66 @@ class DatabaseManager {
                 truncate(osVersion, FIELD_MAX_LENGTH.OS_VERSION),
                 truncate(deviceType, FIELD_MAX_LENGTH.DEVICE_TYPE),
                 truncate(sessionId, FIELD_MAX_LENGTH.SESSION_ID),
-                truncate(eventType, FIELD_MAX_LENGTH.EVENT_TYPE),
+                storedEventType,
                 eventData ? JSON.stringify(eventData) : null,
                 isUnique,
+                truncate(hostname, FIELD_MAX_LENGTH.HOSTNAME),
+                truncate(language, FIELD_MAX_LENGTH.LANGUAGE),
+                truncate(utmSource, FIELD_MAX_LENGTH.UTM),
+                truncate(utmMedium, FIELD_MAX_LENGTH.UTM),
+                truncate(utmCampaign, FIELD_MAX_LENGTH.UTM),
+                truncate(utmTerm, FIELD_MAX_LENGTH.UTM),
+                truncate(utmContent, FIELD_MAX_LENGTH.UTM),
+                truncate(region, FIELD_MAX_LENGTH.REGION),
+                truncate(city, FIELD_MAX_LENGTH.CITY),
             ]
         );
 
+        // The view register log. Written after the row, and never able to fail
+        // the request: the view is already stored.
+        await this.logs.writeViewLog({
+            appId,
+            source,
+            viewId: publicId,
+            eventType: storedEventType,
+            isUnique: isUnique === 1,
+            hostname,
+        });
+
         return {
             duplicate: isUnique === 0,
             insertId: result.insertId,
+            publicId,
             isUnique: isUnique === 1,
         };
     }
 
+    /**
+     * Record how long a view's page was visible and how far it was scrolled.
+     *
+     * A page reports this when it is hidden or left, possibly more than once
+     * (hidden, shown again, then left), each time with its running total, so
+     * the larger value always wins. Only a live view from the last
+     * TRACKING.ENGAGE_WINDOW_HOURS is updated: an old or trashed view keeps
+     * what it had.
+     *
+     * @param {string} appId already validated
+     * @param {{ viewId: string, engagedMs: number, scrollDepth: number }} engagement
+     * @returns {Promise<boolean>} whether a view was updated
+     */
+    async addEngagement(appId, { viewId, engagedMs, scrollDepth }) {
+        this.assertReady();
+        const [result] = await this.pool.query(
+            `UPDATE \`${appId}\`
+             SET engaged_ms = GREATEST(COALESCE(engaged_ms, 0), ?),
+                 scroll_depth = GREATEST(COALESCE(scroll_depth, 0), ?)
+             WHERE public_id = ? AND ${LIVE_ROW}
+               AND timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)`,
+            [engagedMs, scrollDepth, viewId, TRACKING.ENGAGE_WINDOW_HOURS]
+        );
+        return result.affectedRows > 0;
+    }
+
     /**
      * Register a view (backward compatible wrapper)
      */
@@ -477,14 +595,15 @@ class DatabaseManager {
                 COUNT(*) as total_views,
                 SUM(CASE WHEN is_unique = 1 THEN 1 ELSE 0 END) as unique_views,
                 COUNT(DISTINCT visitor_hash) as unique_visitors
-             FROM \`${appId}\``
+             FROM \`${appId}\`
+             WHERE ${LIVE_ROW}`
         );
 
         const stats = totalStats[0];
 
         const [byCountry] = await this.pool.query(
             `SELECT country, COUNT(*) as count FROM \`${appId}\`
-             WHERE country IS NOT NULL
+             WHERE country IS NOT NULL AND ${LIVE_ROW}
              GROUP BY country
              ORDER BY count DESC
              LIMIT ?`,
@@ -493,13 +612,14 @@ class DatabaseManager {
 
         const [byDevice] = await this.pool.query(
             `SELECT devicesize, COUNT(*) as count FROM \`${appId}\`
+             WHERE ${LIVE_ROW}
              GROUP BY devicesize
              ORDER BY count DESC`
         );
 
         const [recent] = await this.pool.query(
             `SELECT COUNT(*) as count FROM \`${appId}\`
-             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)`,
+             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR) AND ${LIVE_ROW}`,
             [SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS]
         );
 
@@ -522,13 +642,14 @@ class DatabaseManager {
         const [views] = await this.pool.query(
             `SELECT masked_ip, country, timestamp, devicesize
              FROM \`${appId}\`
+             WHERE ${LIVE_ROW}
              ORDER BY timestamp DESC
              LIMIT ? OFFSET ?`,
             [limit, offset]
         );
 
         const [total] = await this.pool.query(
-            `SELECT COUNT(*) as count FROM \`${appId}\``
+            `SELECT COUNT(*) as count FROM \`${appId}\` WHERE ${LIVE_ROW}`
         );
 
         return {
@@ -560,7 +681,7 @@ class DatabaseManager {
         const [trends] = await this.pool.query(
             `SELECT ${groupBy} as period, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? DAY)
+             WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? DAY) AND ${LIVE_ROW}
              GROUP BY period
              ORDER BY period ASC`,
             [days]
@@ -578,7 +699,7 @@ class DatabaseManager {
         const [bySource] = await this.pool.query(
             `SELECT source_type, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE source_type IS NOT NULL
+             WHERE source_type IS NOT NULL AND ${LIVE_ROW}
              GROUP BY source_type
              ORDER BY count DESC`
         );
@@ -586,7 +707,7 @@ class DatabaseManager {
         const [byDomain] = await this.pool.query(
             `SELECT referrer_domain, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE referrer_domain IS NOT NULL
+             WHERE referrer_domain IS NOT NULL AND ${LIVE_ROW}
              GROUP BY referrer_domain
              ORDER BY count DESC
              LIMIT ?`,
@@ -605,7 +726,7 @@ class DatabaseManager {
         const [byBrowser] = await this.pool.query(
             `SELECT browser, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE browser IS NOT NULL
+             WHERE browser IS NOT NULL AND ${LIVE_ROW}
              GROUP BY browser
              ORDER BY count DESC
              LIMIT ?`,
@@ -615,7 +736,7 @@ class DatabaseManager {
         const [byOS] = await this.pool.query(
             `SELECT os, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE os IS NOT NULL
+             WHERE os IS NOT NULL AND ${LIVE_ROW}
              GROUP BY os
              ORDER BY count DESC
              LIMIT ?`,
@@ -625,7 +746,7 @@ class DatabaseManager {
         const [byDeviceType] = await this.pool.query(
             `SELECT device_type, COUNT(*) as count
              FROM \`${appId}\`
-             WHERE device_type IS NOT NULL
+             WHERE device_type IS NOT NULL AND ${LIVE_ROW}
              GROUP BY device_type
              ORDER BY count DESC`
         );
@@ -642,7 +763,7 @@ class DatabaseManager {
         const [pages] = await this.pool.query(
             `SELECT page_path, page_title, COUNT(*) as views
              FROM \`${appId}\`
-             WHERE page_path IS NOT NULL
+             WHERE page_path IS NOT NULL AND ${LIVE_ROW}
              GROUP BY page_path, page_title
              ORDER BY views DESC
              LIMIT ?`,
@@ -662,7 +783,7 @@ class DatabaseManager {
         const [events] = await this.pool.query(
             `SELECT ${SESSION_COLUMNS}
              FROM \`${appId}\`
-             WHERE session_id = ?
+             WHERE session_id = ? AND ${LIVE_ROW}
              ORDER BY timestamp ASC`,
             [sessionId]
         );
@@ -702,3 +823,4 @@ module.exports = DatabaseManager;
 module.exports.SESSION_COLUMNS = SESSION_COLUMNS;
 module.exports.appTableDDL = appTableDDL;
 module.exports.APP_REGISTRY_DDL = APP_REGISTRY_DDL;
+module.exports.LIVE_ROW = LIVE_ROW;