@harshankur/viewcounter 3.0.1 → 3.2.0

package/db/AdminRepository.js

@@ -0,0 +1,488 @@
+/**
+ * Data access for the admin UI: listing, editing, annotating, soft-deleting,
+ * restoring, and permanently erasing views.
+ *
+ * Rows are addressed by `public_id` only. Every identifier interpolated into a
+ * statement is either an app ID that has passed `isValidAppId` or a column name
+ * taken from a fixed allowlist in constants.js, never caller text.
+ */
+
+const {
+    ADMIN_RANGE_DAYS,
+    ADMIN_SORT_COLUMNS,
+    EDITABLE_FIELDS,
+    MODIFIED_FILTER,
+    SORT_ORDER,
+    SOURCE_TYPE,
+    VIEW_STATUS,
+} = require('../constants');
+const analysis = require('./analysis');
+const { getError, ErrorType } = require('../utils/errorUtils');
+const { isValidAppId } = require('../utils/appIdUtils');
+const { parseJson } = require('./LogRepository');
+
+/**
+ * Columns returned to the admin UI. Excludes the internal auto-increment `id`
+ * (enumerable) and `visitor_hash` (the pseudonymous visitor identifier, which
+ * an admin has no need to see).
+ */
+const ADMIN_VIEW_COLUMNS = [
+    'public_id', 'timestamp', 'masked_ip', 'country', 'devicesize',
+    'page_path', 'page_title', 'referrer', 'referrer_domain', 'source_type',
+    'browser', 'browser_version', 'os', 'os_version', 'device_type',
+    'session_id', 'event_type', 'event_data', 'is_unique',
+    'hostname', 'language', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'region', 'city', 'engaged_ms', 'scroll_depth',
+    'note', 'admin_modified_at', 'deleted_at',
+].join(', ');
+
+/**
+ * Columns an edit may write. The editable content fields plus the two
+ * columns re-derived from `referrer`, which an admin cannot set directly.
+ */
+const WRITABLE_COLUMNS = new Set([
+    ...Object.values(EDITABLE_FIELDS),
+    'referrer_domain',
+    'source_type',
+]);
+
+/** The row-state condition each operation requires of its targets. */
+const STATE = {
+    ACTIVE: 'deleted_at IS NULL',
+    DELETED: 'deleted_at IS NOT NULL',
+    ANY: '1 = 1',
+};
+
+/** Row condition for each listing status. */
+const STATUS_CONDITION = {
+    [VIEW_STATUS.ACTIVE]: STATE.ACTIVE,
+    [VIEW_STATUS.DELETED]: STATE.DELETED,
+    [VIEW_STATUS.ALL]: STATE.ANY,
+};
+
+const MODIFIED_CONDITION = {
+    [MODIFIED_FILTER.ANY]: null,
+    [MODIFIED_FILTER.MODIFIED]: 'admin_modified_at IS NOT NULL',
+    [MODIFIED_FILTER.UNMODIFIED]: 'admin_modified_at IS NULL',
+};
+
+/**
+ * Escape LIKE's wildcards so a search for "50%" matches that text rather than
+ * everything starting with "50".
+ * @param {string} text
+ * @returns {string}
+ */
+function escapeLike(text) {
+    return String(text).replace(/[\\%_]/g, (char) => `\\${char}`);
+}
+
+/**
+ * The WHERE clause shared by the listing and the analysis, so the table and
+ * the charts above it always describe the same rows.
+ *
+ * With `window` 'previous', the range moves back by its own length, for
+ * comparing with the period before: last 7 days against the 7 days before.
+ *
+ * `where` narrows to breakdown values, such as one country or one page, as
+ * the analysis groups them; null matches the rows with no value ("Unknown").
+ *
+ * @param {{ status?: string, modified?: string, search?: string, range?: string,
+ *   eventType?: string, where?: Record<string, string|null> }} query
+ * @param {'previous'} [window]
+ * @returns {{ clause: string, params: unknown[] }}
+ */
+function buildFilter({ status, modified, search, range, eventType, where = {} } = {}, window) {
+    const conditions = [STATUS_CONDITION[status] || STATE.ACTIVE];
+    const params = [];
+
+    const modifiedCondition = MODIFIED_CONDITION[modified];
+    if (modifiedCondition) conditions.push(modifiedCondition);
+
+    if (eventType) {
+        conditions.push('event_type = ?');
+        params.push(eventType);
+    }
+
+    const days = ADMIN_RANGE_DAYS[range];
+    if (days && window === 'previous') {
+        conditions.push('timestamp >= DATE_SUB(NOW(), INTERVAL ? DAY) AND timestamp < DATE_SUB(NOW(), INTERVAL ? DAY)');
+        params.push(days * 2, days);
+    } else if (days) {
+        conditions.push('timestamp >= DATE_SUB(NOW(), INTERVAL ? DAY)');
+        params.push(days);
+    }
+
+    for (const [dim, value] of Object.entries(where)) {
+        // The expression is a fixed literal, looked up by a validated name;
+        // <=> is equality that also matches NULL to NULL.
+        if (!Object.hasOwn(analysis.FILTER_COLUMNS, dim)) continue;
+        conditions.push(`(${analysis.FILTER_COLUMNS[dim]}) <=> ?`);
+        params.push(value);
+    }
+
+    if (search) {
+        const pattern = `%${escapeLike(search)}%`;
+        conditions.push(`(public_id = ? OR page_path LIKE ? OR page_title LIKE ? OR referrer_domain LIKE ?
+            OR hostname LIKE ? OR utm_campaign LIKE ? OR note LIKE ? OR event_type LIKE ? OR session_id LIKE ?)`);
+        params.push(search, pattern, pattern, pattern, pattern, pattern, pattern, pattern, pattern);
+    }
+
+    return { clause: conditions.join(' AND '), params };
+}
+
+/** Shape one row for the API: camelCase, public ID, parsed JSON. */
+function toApiRow(row, appId = row.app_id) {
+    return {
+        id: row.public_id,
+        appId,
+        timestamp: row.timestamp,
+        maskedIp: row.masked_ip,
+        country: row.country,
+        deviceSize: row.devicesize,
+        pagePath: row.page_path,
+        pageTitle: row.page_title,
+        referrer: row.referrer,
+        referrerDomain: row.referrer_domain,
+        sourceType: row.source_type,
+        browser: row.browser,
+        browserVersion: row.browser_version,
+        os: row.os,
+        osVersion: row.os_version,
+        deviceType: row.device_type,
+        sessionId: row.session_id,
+        eventType: row.event_type,
+        eventData: parseJson(row.event_data),
+        isUnique: row.is_unique === 1 || row.is_unique === true,
+        hostname: row.hostname ?? null,
+        language: row.language ?? null,
+        utmSource: row.utm_source ?? null,
+        utmMedium: row.utm_medium ?? null,
+        utmCampaign: row.utm_campaign ?? null,
+        utmTerm: row.utm_term ?? null,
+        utmContent: row.utm_content ?? null,
+        region: row.region ?? null,
+        city: row.city ?? null,
+        engagedMs: row.engaged_ms ?? null,
+        scrollDepth: row.scroll_depth ?? null,
+        note: row.note,
+        adminModifiedAt: row.admin_modified_at,
+        deletedAt: row.deleted_at,
+    };
+}
+
+class AdminRepository {
+    /**
+     * @param {{ pool: object, assertReady: () => void }} db a DatabaseManager
+     */
+    constructor(db) {
+        this.db = db;
+    }
+
+    get pool() {
+        this.db.assertReady();
+        return this.db.pool;
+    }
+
+    /** Backstop for the route-level allowlist check. */
+    table(appId) {
+        if (!isValidAppId(appId)) {
+            throw getError(ErrorType.INVALID_APP_ID, { appId });
+        }
+        return `\`${appId}\``;
+    }
+
+    /**
+     * Row counts per app, for the app picker.
+     * @param {string[]} appIds
+     * @returns {Promise<Array<{appId: string, active: number, deleted: number, modified: number, available: boolean}>>}
+     */
+    async summarizeApps(appIds) {
+        const summaries = [];
+        for (const appId of appIds) {
+            try {
+                const [rows] = await this.pool.query(
+                    `SELECT
+                        SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) AS active,
+                        SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) AS deleted,
+                        SUM(CASE WHEN deleted_at IS NULL AND admin_modified_at IS NOT NULL THEN 1 ELSE 0 END) AS modified
+                     FROM ${this.table(appId)}`
+                );
+                summaries.push({
+                    appId,
+                    active: Number(rows[0]?.active || 0),
+                    deleted: Number(rows[0]?.deleted || 0),
+                    modified: Number(rows[0]?.modified || 0),
+                    available: true,
+                });
+            } catch {
+                // A configured app whose table is missing still appears, so the
+                // operator can see it is broken rather than silently absent.
+                summaries.push({ appId, active: 0, deleted: 0, modified: 0, available: false });
+            }
+        }
+        return summaries;
+    }
+
+    /**
+     * Which of these apps have a table. Listing and analysing all apps at
+     * once skips an app whose table is missing rather than failing outright.
+     * @param {string[]} appIds
+     * @returns {Promise<string[]>} in the order given
+     */
+    async existingTables(appIds) {
+        if (appIds.length === 0) return [];
+        const [rows] = await this.pool.query(
+            `SELECT TABLE_NAME AS name FROM information_schema.TABLES
+             WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)`,
+            [appIds]
+        );
+        const present = new Set(rows.map((row) => row.name));
+        return appIds.filter((appId) => present.has(appId));
+    }
+
+    /**
+     * One page of views from one app or several.
+     *
+     * Several apps are read with UNION ALL. Each branch is sorted and cut to
+     * the rows the requested page could possibly need, so a deep page on a
+     * large table never unions whole tables.
+     *
+     * @param {string|string[]} apps one app ID or several
+     * @param {{ status: string, modified: string, search?: string, range?: string,
+     *   sort: string, order: string, page: number, pageSize: number }} query already validated
+     * @returns {Promise<{ views: object[], total: number }>}
+     */
+    async listViews(apps, query) {
+        const appIds = Array.isArray(apps) ? apps : [apps];
+        if (appIds.length === 0) return { views: [], total: 0 };
+
+        const { page, pageSize, sort, order } = query;
+        const { clause, params } = buildFilter(query);
+        const column = ADMIN_SORT_COLUMNS[sort] || ADMIN_SORT_COLUMNS.timestamp;
+        const direction = order === SORT_ORDER.ASC ? 'ASC' : 'DESC';
+        const offset = (page - 1) * pageSize;
+
+        let rows;
+        if (appIds.length === 1) {
+            [rows] = await this.pool.query(
+                `SELECT ? AS app_id, ${ADMIN_VIEW_COLUMNS} FROM ${this.table(appIds[0])}
+                 WHERE ${clause}
+                 ORDER BY ${column} ${direction}, id DESC
+                 LIMIT ? OFFSET ?`,
+                [appIds[0], ...params, pageSize, offset]
+            );
+        } else {
+            const branches = appIds.map((appId) =>
+                `(SELECT ? AS app_id, ${ADMIN_VIEW_COLUMNS} FROM ${this.table(appId)}
+                  WHERE ${clause}
+                  ORDER BY ${column} ${direction}, public_id ${direction}
+                  LIMIT ?)`);
+            [rows] = await this.pool.query(
+                `${branches.join(' UNION ALL ')}
+                 ORDER BY ${column} ${direction}, public_id ${direction}
+                 LIMIT ? OFFSET ?`,
+                [...appIds.flatMap((appId) => [appId, ...params, offset + pageSize]), pageSize, offset]
+            );
+        }
+
+        const counts = appIds.map((appId) => `(SELECT COUNT(*) FROM ${this.table(appId)} WHERE ${clause})`);
+        const [count] = await this.pool.query(
+            `SELECT ${counts.join(' + ')} AS count`,
+            appIds.flatMap(() => params)
+        );
+
+        return { views: rows.map((row) => toApiRow(row)), total: Number(count[0]?.count || 0) };
+    }
+
+    /**
+     * Aggregates over the same rows a listing with this query would show: see
+     * db/analysis.js. A bounded range is compared with the period before it.
+     *
+     * @param {string[]} appIds
+     * @param {{ status: string, modified: string, search?: string, range?: string, eventType?: string }} query
+     * @returns {Promise<object>}
+     */
+    async analyze(appIds, query) {
+        if (appIds.length === 0) {
+            return { totals: null, previous: null, window: null, ...analysis.emptySections(), eventTypes: [] };
+        }
+        const result = await analysis.runAnalysis(this.pool, (appId) => this.table(appId), appIds,
+            (window) => buildFilter(query, window),
+            { hasPrevious: Boolean(ADMIN_RANGE_DAYS[query.range]), spanDays: ADMIN_RANGE_DAYS[query.range] });
+        // After the analysis: when the filters match nothing is exactly when
+        // the admin needs every type on offer to pick another.
+        return { ...result, eventTypes: await this.eventTypes(appIds, query.status) };
+    }
+
+    /**
+     * Who is on the sites right now, and views per minute over the last half
+     * hour. Live views of these apps only.
+     * @param {string[]} appIds
+     */
+    async realtime(appIds) {
+        if (appIds.length === 0) return { visitors: 0, nowMinute: null, minutes: [], pages: [] };
+        return analysis.runRealtime(this.pool, (appId) => this.table(appId), appIds);
+    }
+
+    /**
+     * Every event type the apps hold in this status, whatever the other
+     * filters say. The event-type filter offers all of them, so narrowing the
+     * range or choosing one type never hides the rest, and a rare type is
+     * never cut off the way the top-N breakdown cuts it.
+     *
+     * @param {string[]} appIds
+     * @param {string} status
+     * @returns {Promise<string[]>}
+     */
+    async eventTypes(appIds, status) {
+        const { clause, params } = buildFilter({ status });
+        const branches = appIds.map((appId) =>
+            `SELECT event_type FROM ${this.table(appId)} WHERE ${clause} AND event_type IS NOT NULL`);
+        const [rows] = await this.pool.query(
+            `SELECT DISTINCT event_type FROM (${branches.join(' UNION ')}) AS e ORDER BY event_type`,
+            appIds.flatMap(() => params)
+        );
+        return rows.map((row) => row.event_type);
+    }
+
+    /**
+     * Which of `ids` exist in `appId` and are in the required state.
+     * @returns {Promise<string[]>}
+     */
+    async matchIds(appId, ids, state) {
+        if (ids.length === 0) return [];
+        const [rows] = await this.pool.query(
+            `SELECT public_id FROM ${this.table(appId)} WHERE public_id IN (?) AND ${state}`,
+            [ids]
+        );
+        return rows.map((row) => row.public_id);
+    }
+
+    /**
+     * Edit content fields and mark the rows as admin-modified. Trashed rows are
+     * not editable; restore them first.
+     *
+     * @param {string} appId
+     * @param {string[]} ids
+     * @param {Record<string, unknown>} columnValues column -> new value
+     * @returns {Promise<string[]>} IDs actually changed
+     */
+    async updateContent(appId, ids, columnValues) {
+        const columns = Object.keys(columnValues);
+        for (const column of columns) {
+            if (!WRITABLE_COLUMNS.has(column)) {
+                throw getError(ErrorType.FIELD_NOT_WRITABLE, { column });
+            }
+        }
+
+        const matched = await this.matchIds(appId, ids, STATE.ACTIVE);
+        if (matched.length === 0 || columns.length === 0) return [];
+
+        const assignments = [];
+        const params = [];
+        for (const column of columns) {
+            if (column === 'source_type' && columnValues.referrer_domain) {
+                // A referrer on the row's own site is internal, as on the write
+                // path; each row is judged by its own site, in the same statement.
+                const site = String(columnValues.referrer_domain).toLowerCase().replace(/^www\./, '');
+                assignments.push('`source_type` = CASE WHEN LOWER(hostname) IN (?, ?) THEN ? ELSE ? END');
+                params.push(site, `www.${site}`, SOURCE_TYPE.INTERNAL, columnValues.source_type);
+            } else {
+                assignments.push(`\`${column}\` = ?`);
+                params.push(columnValues[column]);
+            }
+        }
+        await this.pool.query(
+            `UPDATE ${this.table(appId)} SET ${assignments.join(', ')}, admin_modified_at = NOW()
+             WHERE public_id IN (?) AND ${STATE.ACTIVE}`,
+            [...params, matched]
+        );
+        return matched;
+    }
+
+    /**
+     * Set or clear the note. A note is an annotation, not a change to what was
+     * observed, so it does not mark the row admin-modified.
+     *
+     * @param {string} appId
+     * @param {string[]} ids
+     * @param {string|null} note null clears it
+     * @returns {Promise<string[]>}
+     */
+    async setNote(appId, ids, note) {
+        const matched = await this.matchIds(appId, ids, STATE.ANY);
+        if (matched.length === 0) return [];
+
+        await this.pool.query(
+            `UPDATE ${this.table(appId)} SET note = ? WHERE public_id IN (?)`,
+            [note, matched]
+        );
+        return matched;
+    }
+
+    /** Move active rows to the trash. @returns {Promise<string[]>} */
+    async softDelete(appId, ids) {
+        const matched = await this.matchIds(appId, ids, STATE.ACTIVE);
+        if (matched.length === 0) return [];
+
+        await this.pool.query(
+            `UPDATE ${this.table(appId)} SET deleted_at = NOW() WHERE public_id IN (?) AND ${STATE.ACTIVE}`,
+            [matched]
+        );
+        return matched;
+    }
+
+    /** Bring trashed rows back. @returns {Promise<string[]>} */
+    async restore(appId, ids) {
+        const matched = await this.matchIds(appId, ids, STATE.DELETED);
+        if (matched.length === 0) return [];
+
+        await this.pool.query(
+            `UPDATE ${this.table(appId)} SET deleted_at = NULL WHERE public_id IN (?) AND ${STATE.DELETED}`,
+            [matched]
+        );
+        return matched;
+    }
+
+    /**
+     * Permanently erase rows. Only rows already in the trash can be erased, so
+     * erasure is always a deliberate second step after a soft delete.
+     * @returns {Promise<string[]>}
+     */
+    async purge(appId, ids) {
+        const matched = await this.matchIds(appId, ids, STATE.DELETED);
+        if (matched.length === 0) return [];
+
+        await this.pool.query(
+            `DELETE FROM ${this.table(appId)} WHERE public_id IN (?) AND ${STATE.DELETED}`,
+            [matched]
+        );
+        return matched;
+    }
+
+    /**
+     * Erase trashed rows older than the retention period.
+     * @param {string} appId
+     * @param {number} days
+     * @returns {Promise<number>} rows erased
+     */
+    async purgeExpired(appId, days) {
+        const [result] = await this.pool.query(
+            `DELETE FROM ${this.table(appId)}
+             WHERE ${STATE.DELETED} AND deleted_at < DATE_SUB(NOW(), INTERVAL ? DAY)`,
+            [days]
+        );
+        return Number(result?.affectedRows || 0);
+    }
+}
+
+module.exports = AdminRepository;
+module.exports.ADMIN_VIEW_COLUMNS = ADMIN_VIEW_COLUMNS;
+module.exports.WRITABLE_COLUMNS = WRITABLE_COLUMNS;
+module.exports.escapeLike = escapeLike;
+module.exports.toApiRow = toApiRow;
+module.exports.buildFilter = buildFilter;
+module.exports.chooseBucket = analysis.chooseBucket;
+module.exports.BREAKDOWN_COLUMNS = analysis.BREAKDOWN_COLUMNS;
+module.exports.FILTER_COLUMNS = analysis.FILTER_COLUMNS;
+module.exports.BUCKET_EXPRESSION = analysis.BUCKET_EXPRESSION;