@harshankur/viewcounter 3.1.0 → 3.3.0

package/constants.js

@@ -52,6 +52,14 @@ const FIELD_MAX_LENGTH = {
     NOTE: 1000,
     /** CHAR(36): the canonical textual form of a UUID. */
     UUID: 36,
+    /** The longest a DNS name can be. */
+    HOSTNAME: 253,
+    /** A primary language subtag: two or three letters, rarely up to eight. */
+    LANGUAGE: 8,
+    /** Each of utm_source, utm_medium, utm_campaign, utm_term, utm_content. */
+    UTM: 100,
+    REGION: 100,
+    CITY: 100,
 };
 
 /** Bounds for user-supplied pagination and range parameters. */
@@ -89,7 +97,7 @@ const DATABASE = {
     QUERY_TIMEOUT_MS: 5_000,
     CONNECT_TIMEOUT_MS: 10_000,
     DEFAULT_PORT: 3306,
-    SCHEMA_VERSION: 'admin_schema_v4',
+    SCHEMA_VERSION: 'schema_v6',
     /** Rows given a public_id per statement when backfilling an old table. */
     BACKFILL_BATCH_SIZE: 500,
 };
@@ -145,10 +153,21 @@ const ADMIN = {
     MAX_PASSWORD_INPUT_LENGTH: 1024,
     SESSION_TOKEN_BYTES: 32,
     CSRF_TOKEN_BYTES: 32,
-    /** Signed out after this long without a request. */
-    SESSION_IDLE_TIMEOUT_MS: 30 * 60 * 1000,
-    /** Signed out after this long regardless of activity. */
-    SESSION_ABSOLUTE_TIMEOUT_MS: 12 * 60 * 60 * 1000,
+    /** Default for ADMIN_SESSION_IDLE_TIMEOUT: signed out after this long unused. */
+    SESSION_IDLE_TIMEOUT_MS: 7 * 24 * 60 * 60 * 1000,
+    /** Default for ADMIN_SESSION_MAX_AGE: signed out after this long regardless. */
+    SESSION_ABSOLUTE_TIMEOUT_MS: 30 * 24 * 60 * 60 * 1000,
+    SESSION_IDLE_MIN_MS: 5 * 60 * 1000,
+    SESSION_IDLE_MAX_MS: 90 * 24 * 60 * 60 * 1000,
+    SESSION_MAX_AGE_MIN_MS: 60 * 60 * 1000,
+    SESSION_MAX_AGE_MAX_MS: 365 * 24 * 60 * 60 * 1000,
+    /**
+     * A session's last-seen time is written at most this often, so an active
+     * admin costs one database write a minute rather than one per request.
+     */
+    SESSION_TOUCH_INTERVAL_MS: 60 * 1000,
+    /** Permanent erasure needs the password to have been entered this recently. */
+    REAUTH_WINDOW_MS: 15 * 60 * 1000,
     /** Oldest sessions are evicted beyond this, bounding memory. */
     MAX_SESSIONS: 50,
     LOGIN_RATE_LIMIT_WINDOW_MS: 15 * 60 * 1000,
@@ -176,6 +195,8 @@ const ADMIN = {
     VIEW_LOG_PRUNE_BATCH_SIZE: 5000,
     /** How often expired trash and old view-log entries are checked for. */
     RETENTION_INTERVAL_MS: 60 * 60 * 1000,
+    /** The tracking log's summary covers this many recent hours. */
+    TRACKING_SUMMARY_HOURS: 24,
 };
 
 /**
@@ -183,6 +204,7 @@ const ADMIN = {
  * a number of days. `all` has no lower bound. Only these values reach SQL.
  */
 const ADMIN_RANGE = {
+    DAY: '24h',
     WEEK: '7d',
     MONTH: '30d',
     QUARTER: '90d',
@@ -191,6 +213,7 @@ const ADMIN_RANGE = {
 };
 
 const ADMIN_RANGE_DAYS = {
+    [ADMIN_RANGE.DAY]: 1,
     [ADMIN_RANGE.WEEK]: 7,
     [ADMIN_RANGE.MONTH]: 30,
     [ADMIN_RANGE.QUARTER]: 90,
@@ -203,6 +226,7 @@ const ADMIN_RANGE_DAYS = {
  * actually in the filtered set, so a chart never has thousands of points.
  */
 const TREND_BUCKET = {
+    HOUR: 'hour',
     DAY: 'day',
     WEEK: 'week',
     MONTH: 'month',
@@ -210,12 +234,34 @@ const TREND_BUCKET = {
 
 /** Largest span, in days, charted per day and per week. */
 const TREND_BUCKET_MAX_DAYS = {
+    [TREND_BUCKET.HOUR]: 2,
     [TREND_BUCKET.DAY]: 92,
     [TREND_BUCKET.WEEK]: 731,
 };
 
 /** Rows per breakdown returned by the admin analysis; the rest is "other". */
-const ANALYSIS_TOP_N = 8;
+const ANALYSIS_TOP_N = 10;
+
+/** Bounds of the admin analysis beyond the breakdowns. */
+const ANALYSIS = {
+    /**
+     * A visit ends after this long without a page view, the common 30-minute
+     * convention. Visits are read from the rotating visitor hash, so they
+     * never link a visitor across days.
+     */
+    VISIT_GAP_SECONDS: 30 * 60,
+    /** Page-to-page steps listed in the page flow. */
+    TRANSITIONS_TOP_N: 15,
+    /** The time-of-day heatmap reads at most the last year of the range. */
+    HEATMAP_MAX_DAYS: 365,
+    /** Most recent custom events whose properties are tallied. */
+    EVENT_PROPERTIES_SAMPLE: 5000,
+    /** Property values listed across all custom events. */
+    EVENT_PROPERTIES_TOP_N: 30,
+    /** "Right now" means the last few minutes; the live chart covers half an hour. */
+    REALTIME_VISITOR_MINUTES: 5,
+    REALTIME_CHART_MINUTES: 30,
+};
 
 /** Which rows an admin listing returns. */
 const VIEW_STATUS = {
@@ -248,6 +294,13 @@ const ADMIN_SORT_COLUMNS = {
     eventType: 'event_type',
     source: 'source_type',
     browser: 'browser',
+    os: 'os',
+    title: 'page_title',
+    hostname: 'hostname',
+    referrer: 'referrer_domain',
+    campaign: 'utm_campaign',
+    language: 'language',
+    engagedMs: 'engaged_ms',
     modifiedAt: 'admin_modified_at',
     deletedAt: 'deleted_at',
 };
@@ -283,6 +336,8 @@ const ADMIN_ACTION = {
     VIEWS_PURGED: 'views_purged',
     TRASH_AUTO_PURGED: 'trash_auto_purged',
     VIEW_LOG_PRUNED: 'view_log_pruned',
+    /** The password entered again to allow a permanent erasure. */
+    PASSWORD_CONFIRMED: 'password_confirmed',
 };
 
 /**
@@ -295,20 +350,75 @@ const ADMIN_ERROR_CODE = {
     TOO_MANY_ATTEMPTS: 'TOO_MANY_ATTEMPTS',
     RATE_LIMITED: 'RATE_LIMITED',
     CSRF_REJECTED: 'CSRF_REJECTED',
+    /** The action needs the password entered again (POST /reauth), then a retry. */
+    REAUTH_REQUIRED: 'REAUTH_REQUIRED',
     VALIDATION_FAILED: 'VALIDATION_FAILED',
     NOT_FOUND: 'NOT_FOUND',
     SERVER_ERROR: 'SERVER_ERROR',
 };
 
-/** Which write endpoint a view-register-log entry came through. */
+/** Which tracking endpoint a request came through. */
 const VIEW_LOG_SOURCE = {
     REGISTER_VIEW: 'registerView',
     EVENT: 'event',
+    /** Time on page and scroll depth for a view already recorded. */
+    ENGAGE: 'engage',
 };
 
 /** Service-owned tables. All carry the reserved `_` prefix. */
 const ADMIN_LOG_TABLE = '_admin_log';
+const ADMIN_SESSIONS_TABLE = '_admin_sessions';
 const VIEW_LOG_TABLE = '_view_log';
+/** Tracking requests that were not stored, counted per minute. */
+const TRACKING_REJECTIONS_TABLE = '_tracking_rejections';
+
+/** The tracking pipeline's own bounds. */
+const TRACKING = {
+    /** Longest visible time one page view may report. */
+    MAX_ENGAGED_MS: 6 * 60 * 60 * 1000,
+    /** Engagement is only accepted for a view recorded this recently. */
+    ENGAGE_WINDOW_HOURS: 24,
+    /** A beacon is a few dozen bytes; this bounds what is even parsed. */
+    ENGAGE_BODY_BYTES: 1024,
+    /** How often counted rejections are written to the database. */
+    REJECTION_FLUSH_MS: 15_000,
+    /**
+     * Distinct rejection keys kept with their app, detail, and hostname, per
+     * minute and per hour, however often the counts are written. Beyond these,
+     * a request is still counted, by minute, endpoint, and reason alone, so a
+     * flood of made-up values cannot grow memory or the table: at most a few
+     * thousand rows an hour, whatever arrives.
+     */
+    REJECTION_MAX_KEYS_PER_MINUTE: 100,
+    REJECTION_MAX_KEYS_PER_HOUR: 1000,
+    /** Rejections are only ever stored as per-minute counts. */
+    REJECTION_BUCKET_MS: 60 * 1000,
+};
+
+/** What happened to one tracking request, as the tracking log reports it. */
+const TRACKING_OUTCOME = {
+    /** Stored, and the first view of the page by this visitor in the window. */
+    RECORDED: 'recorded',
+    /** Stored, but the same visitor viewed it again within the window. */
+    REPEAT: 'repeat',
+    /** A crawler, preview fetcher, or headless browser; counted, not stored. */
+    BOT: 'bot',
+    /** Refused; the reason says why. */
+    REJECTED: 'rejected',
+};
+
+/** Why a tracking request was not stored. */
+const REJECTION_REASON = {
+    BOT: 'bot',
+    UNKNOWN_APP: 'unknown_app',
+    ORIGIN_NOT_ALLOWED: 'origin_not_allowed',
+    INVALID_REQUEST: 'invalid_request',
+    INVALID_IP: 'invalid_ip',
+    RATE_LIMITED: 'rate_limited',
+    /** Engagement for a view that does not exist, is trashed, or is too old. */
+    UNKNOWN_VIEW: 'unknown_view',
+    SERVER_ERROR: 'server_error',
+};
 
 /** Canonical UUID text form, any version. Admin row IDs are validated with it. */
 const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -335,6 +445,8 @@ const SOURCE_TYPE = {
     EMAIL: 'email',
     CAMPAIGN: 'campaign',
     REFERRAL: 'referral',
+    /** From another page of the same site: a click within it, not a way in. */
+    INTERNAL: 'internal',
     UNKNOWN: 'unknown',
 };
 
@@ -367,7 +479,7 @@ const SCOPE_ALL = '*';
  * identifiers cannot be bound as parameters. Until now app IDs only ever came
  * from trusted local config; they can now arrive over HTTP from the admin API,
  * so the character set is restricted to what is unambiguously safe as an
- * identifier. This is the gate — not a nicety.
+ * identifier. This is the gate, not a nicety.
  *
  * Letters, digits, underscore, and hyphen only. A backtick is the sole
  * character that can terminate a quoted identifier, and none of these can;
@@ -416,6 +528,7 @@ module.exports = {
     TREND_BUCKET,
     TREND_BUCKET_MAX_DAYS,
     ANALYSIS_TOP_N,
+    ANALYSIS,
     MODIFIED_FILTER,
     SORT_ORDER,
     ADMIN_SORT_COLUMNS,
@@ -424,7 +537,12 @@ module.exports = {
     ADMIN_ERROR_CODE,
     VIEW_LOG_SOURCE,
     ADMIN_LOG_TABLE,
+    ADMIN_SESSIONS_TABLE,
     VIEW_LOG_TABLE,
+    TRACKING_REJECTIONS_TABLE,
+    TRACKING,
+    TRACKING_OUTCOME,
+    REJECTION_REASON,
     UUID_PATTERN,
     EVENT_TYPE,
     TREND_PERIOD,

package/db/AdminRepository.js

@@ -10,14 +10,13 @@
 const {
     ADMIN_RANGE_DAYS,
     ADMIN_SORT_COLUMNS,
-    ANALYSIS_TOP_N,
     EDITABLE_FIELDS,
     MODIFIED_FILTER,
     SORT_ORDER,
-    TREND_BUCKET,
-    TREND_BUCKET_MAX_DAYS,
+    SOURCE_TYPE,
     VIEW_STATUS,
 } = require('../constants');
+const analysis = require('./analysis');
 const { getError, ErrorType } = require('../utils/errorUtils');
 const { isValidAppId } = require('../utils/appIdUtils');
 const { parseJson } = require('./LogRepository');
@@ -32,6 +31,8 @@ const ADMIN_VIEW_COLUMNS = [
     'page_path', 'page_title', 'referrer', 'referrer_domain', 'source_type',
     'browser', 'browser_version', 'os', 'os_version', 'device_type',
     'session_id', 'event_type', 'event_data', 'is_unique',
+    'hostname', 'language', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'region', 'city', 'engaged_ms', 'scroll_depth',
     'note', 'admin_modified_at', 'deleted_at',
 ].join(', ');
 
@@ -75,81 +76,58 @@ function escapeLike(text) {
     return String(text).replace(/[\\%_]/g, (char) => `\\${char}`);
 }
 
-/**
- * Columns the analysis reads. `visitor_hash` is here only so the database can
- * count distinct visitors; it is never selected into a result set.
- */
-const ANALYSIS_COLUMNS = [
-    'country', 'event_type', 'source_type', 'devicesize', 'browser', 'os',
-    'visitor_hash', 'is_unique', 'timestamp', 'admin_modified_at',
-].join(', ');
-
-/**
- * Time-series bucket expressions, keyed by TREND_BUCKET. Fixed literals: the
- * bucket is chosen in code from the data's span, never from caller input.
- * Every bucket is labelled by the date it starts on (YYYY-MM-DD).
- */
-const BUCKET_EXPRESSION = {
-    [TREND_BUCKET.DAY]: "DATE_FORMAT(timestamp, '%Y-%m-%d')",
-    [TREND_BUCKET.WEEK]: "DATE_FORMAT(DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(timestamp) DAY), '%Y-%m-%d')",
-    [TREND_BUCKET.MONTH]: "DATE_FORMAT(timestamp, '%Y-%m-01')",
-};
-
-/** Breakdown dimensions of the analysis: API name -> column. Fixed literals. */
-const BREAKDOWN_COLUMNS = {
-    source: 'source_type',
-    deviceSize: 'devicesize',
-    browser: 'browser',
-    os: 'os',
-    eventType: 'event_type',
-    app: 'app_id',
-};
-
-const DAY_MS = 24 * 60 * 60 * 1000;
-
-/** The coarsest bucket that keeps a chart of this span readable. */
-function chooseBucket(firstAt, lastAt) {
-    if (!firstAt || !lastAt) return TREND_BUCKET.DAY;
-    const days = (new Date(lastAt).getTime() - new Date(firstAt).getTime()) / DAY_MS;
-    if (days <= TREND_BUCKET_MAX_DAYS[TREND_BUCKET.DAY]) return TREND_BUCKET.DAY;
-    if (days <= TREND_BUCKET_MAX_DAYS[TREND_BUCKET.WEEK]) return TREND_BUCKET.WEEK;
-    return TREND_BUCKET.MONTH;
-}
-
 /**
  * The WHERE clause shared by the listing and the analysis, so the table and
  * the charts above it always describe the same rows.
  *
+ * With `window` 'previous', the range moves back by its own length, for
+ * comparing with the period before: last 7 days against the 7 days before.
+ *
+ * `where` narrows to breakdown values, such as one country or one page, as
+ * the analysis groups them; null matches the rows with no value ("Unknown").
+ *
  * @param {{ status?: string, modified?: string, search?: string, range?: string,
- *   eventType?: string }} query
+ *   eventType?: string, where?: Record<string, string|null> }} query
+ * @param {'previous'} [window]
  * @returns {{ clause: string, params: unknown[] }}
  */
-function buildFilter({ status, modified, search, range, eventType } = {}) {
-    const where = [STATUS_CONDITION[status] || STATE.ACTIVE];
+function buildFilter({ status, modified, search, range, eventType, where = {} } = {}, window) {
+    const conditions = [STATUS_CONDITION[status] || STATE.ACTIVE];
     const params = [];
 
     const modifiedCondition = MODIFIED_CONDITION[modified];
-    if (modifiedCondition) where.push(modifiedCondition);
+    if (modifiedCondition) conditions.push(modifiedCondition);
 
     if (eventType) {
-        where.push('event_type = ?');
+        conditions.push('event_type = ?');
         params.push(eventType);
     }
 
     const days = ADMIN_RANGE_DAYS[range];
-    if (days) {
-        where.push('timestamp >= DATE_SUB(NOW(), INTERVAL ? DAY)');
+    if (days && window === 'previous') {
+        conditions.push('timestamp >= DATE_SUB(NOW(), INTERVAL ? DAY) AND timestamp < DATE_SUB(NOW(), INTERVAL ? DAY)');
+        params.push(days * 2, days);
+    } else if (days) {
+        conditions.push('timestamp >= DATE_SUB(NOW(), INTERVAL ? DAY)');
         params.push(days);
     }
 
+    for (const [dim, value] of Object.entries(where)) {
+        // The expression is a fixed literal, looked up by a validated name;
+        // <=> is equality that also matches NULL to NULL.
+        if (!Object.hasOwn(analysis.FILTER_COLUMNS, dim)) continue;
+        conditions.push(`(${analysis.FILTER_COLUMNS[dim]}) <=> ?`);
+        params.push(value);
+    }
+
     if (search) {
         const pattern = `%${escapeLike(search)}%`;
-        where.push(`(public_id = ? OR page_path LIKE ? OR page_title LIKE ? OR referrer_domain LIKE ?
-            OR note LIKE ? OR event_type LIKE ? OR session_id LIKE ?)`);
-        params.push(search, pattern, pattern, pattern, pattern, pattern, pattern);
+        conditions.push(`(public_id = ? OR page_path LIKE ? OR page_title LIKE ? OR referrer_domain LIKE ?
+            OR hostname LIKE ? OR utm_campaign LIKE ? OR note LIKE ? OR event_type LIKE ? OR session_id LIKE ?)`);
+        params.push(search, pattern, pattern, pattern, pattern, pattern, pattern, pattern, pattern);
     }
 
-    return { clause: where.join(' AND '), params };
+    return { clause: conditions.join(' AND '), params };
 }
 
 /** Shape one row for the API: camelCase, public ID, parsed JSON. */
@@ -175,6 +153,17 @@ function toApiRow(row, appId = row.app_id) {
         eventType: row.event_type,
         eventData: parseJson(row.event_data),
         isUnique: row.is_unique === 1 || row.is_unique === true,
+        hostname: row.hostname ?? null,
+        language: row.language ?? null,
+        utmSource: row.utm_source ?? null,
+        utmMedium: row.utm_medium ?? null,
+        utmCampaign: row.utm_campaign ?? null,
+        utmTerm: row.utm_term ?? null,
+        utmContent: row.utm_content ?? null,
+        region: row.region ?? null,
+        city: row.city ?? null,
+        engagedMs: row.engaged_ms ?? null,
+        scrollDepth: row.scroll_depth ?? null,
         note: row.note,
         adminModifiedAt: row.admin_modified_at,
         deletedAt: row.deleted_at,
@@ -306,110 +295,33 @@ class AdminRepository {
     }
 
     /**
-     * Aggregates over the same rows a listing with this query would show.
+     * Aggregates over the same rows a listing with this query would show: see
+     * db/analysis.js. A bounded range is compared with the period before it.
      *
      * @param {string[]} appIds
-     * @param {{ status: string, modified: string, search?: string, range?: string }} query
-     * @returns {Promise<object>} totals, trend, breakdowns, and per-country counts
+     * @param {{ status: string, modified: string, search?: string, range?: string, eventType?: string }} query
+     * @returns {Promise<object>}
      */
     async analyze(appIds, query) {
-        const empty = {
-            totals: { views: 0, uniqueViews: 0, visitors: 0, countries: 0, modified: 0, firstAt: null, lastAt: null },
-            bucket: TREND_BUCKET.DAY,
-            trend: [],
-            breakdowns: Object.fromEntries(Object.keys(BREAKDOWN_COLUMNS).map((dim) => [dim, []])),
-            countries: [],
-            eventTypes: [],
-        };
-        if (appIds.length === 0) return empty;
-
-        const { clause, params } = buildFilter(query);
-        const branches = appIds.map((appId) =>
-            `SELECT ? AS app_id, ${ANALYSIS_COLUMNS} FROM ${this.table(appId)} WHERE ${clause}`);
-        const cte = `WITH v AS (${branches.join(' UNION ALL ')})`;
-        const cteParams = appIds.flatMap((appId) => [appId, ...params]);
-
-        const [totalRows] = await this.pool.query(
-            `${cte} SELECT
-                COUNT(*) AS views,
-                COALESCE(SUM(is_unique), 0) AS unique_views,
-                COUNT(DISTINCT visitor_hash) AS visitors,
-                COUNT(DISTINCT country) AS countries,
-                COALESCE(SUM(admin_modified_at IS NOT NULL), 0) AS modified,
-                MIN(timestamp) AS first_at,
-                MAX(timestamp) AS last_at
-             FROM v`,
-            cteParams
-        );
-        const totalsRow = totalRows[0] || {};
-        const totals = {
-            views: Number(totalsRow.views || 0),
-            uniqueViews: Number(totalsRow.unique_views || 0),
-            visitors: Number(totalsRow.visitors || 0),
-            countries: Number(totalsRow.countries || 0),
-            modified: Number(totalsRow.modified || 0),
-            firstAt: totalsRow.first_at ?? null,
-            lastAt: totalsRow.last_at ?? null,
-        };
-        // Before the early return: when the filters match nothing is exactly
-        // when the admin needs every type on offer to pick another.
-        const eventTypes = await this.eventTypes(appIds, query.status);
-        if (totals.views === 0) return { ...empty, totals, eventTypes };
-
-        const bucket = chooseBucket(totals.firstAt, totals.lastAt);
-        const [trendRows] = await this.pool.query(
-            `${cte} SELECT ${BUCKET_EXPRESSION[bucket]} AS period, COUNT(*) AS views,
-                COALESCE(SUM(is_unique), 0) AS unique_views
-             FROM v GROUP BY period ORDER BY period`,
-            cteParams
-        );
-
-        const groups = Object.entries(BREAKDOWN_COLUMNS).map(([dim, column]) =>
-            `SELECT '${dim}' AS dim, ${column} AS value, COUNT(*) AS views FROM v GROUP BY ${column}`);
-        const [breakdownRows] = await this.pool.query(
-            `${cte} SELECT dim, value, views FROM (
-                SELECT dim, value, views,
-                    ROW_NUMBER() OVER (PARTITION BY dim ORDER BY views DESC, value) AS rank_in_dim
-                FROM (${groups.join(' UNION ALL ')}) AS g
-             ) AS ranked
-             WHERE rank_in_dim <= ?
-             ORDER BY dim, views DESC, value`,
-            [...cteParams, ANALYSIS_TOP_N]
-        );
-        const breakdowns = Object.fromEntries(Object.keys(BREAKDOWN_COLUMNS).map((dim) => [dim, []]));
-        for (const row of breakdownRows) {
-            breakdowns[row.dim]?.push({ value: row.value ?? null, views: Number(row.views) });
+        if (appIds.length === 0) {
+            return { totals: null, previous: null, window: null, ...analysis.emptySections(), eventTypes: [] };
         }
+        const result = await analysis.runAnalysis(this.pool, (appId) => this.table(appId), appIds,
+            (window) => buildFilter(query, window),
+            { hasPrevious: Boolean(ADMIN_RANGE_DAYS[query.range]), spanDays: ADMIN_RANGE_DAYS[query.range] });
+        // After the analysis: when the filters match nothing is exactly when
+        // the admin needs every type on offer to pick another.
+        return { ...result, eventTypes: await this.eventTypes(appIds, query.status) };
+    }
 
-        // Per-country counts, split by the leading event types so the map can
-        // show where each type comes from; the rest are grouped as null.
-        const types = breakdowns.eventType.map((entry) => entry.value).filter((value) => value !== null);
-        const [countryRows] = await this.pool.query(
-            `${cte} SELECT country,
-                CASE WHEN event_type IN (?) THEN event_type ELSE NULL END AS event_type,
-                COUNT(*) AS views
-             FROM v WHERE country IS NOT NULL
-             GROUP BY country, CASE WHEN event_type IN (?) THEN event_type ELSE NULL END
-             ORDER BY country`,
-            [...cteParams, types.length ? types : [''], types.length ? types : ['']]
-        );
-
-        return {
-            totals,
-            bucket,
-            trend: trendRows.map((row) => ({
-                period: String(row.period),
-                views: Number(row.views),
-                uniqueViews: Number(row.unique_views),
-            })),
-            breakdowns,
-            countries: countryRows.map((row) => ({
-                country: row.country,
-                eventType: row.event_type ?? null,
-                views: Number(row.views),
-            })),
-            eventTypes,
-        };
+    /**
+     * Who is on the sites right now, and views per minute over the last half
+     * hour. Live views of these apps only.
+     * @param {string[]} appIds
+     */
+    async realtime(appIds) {
+        if (appIds.length === 0) return { visitors: 0, nowMinute: null, minutes: [], pages: [] };
+        return analysis.runRealtime(this.pool, (appId) => this.table(appId), appIds);
     }
 
     /**
@@ -466,11 +378,24 @@ class AdminRepository {
         const matched = await this.matchIds(appId, ids, STATE.ACTIVE);
         if (matched.length === 0 || columns.length === 0) return [];
 
-        const assignments = columns.map((column) => `\`${column}\` = ?`).join(', ');
+        const assignments = [];
+        const params = [];
+        for (const column of columns) {
+            if (column === 'source_type' && columnValues.referrer_domain) {
+                // A referrer on the row's own site is internal, as on the write
+                // path; each row is judged by its own site, in the same statement.
+                const site = String(columnValues.referrer_domain).toLowerCase().replace(/^www\./, '');
+                assignments.push('`source_type` = CASE WHEN LOWER(hostname) IN (?, ?) THEN ? ELSE ? END');
+                params.push(site, `www.${site}`, SOURCE_TYPE.INTERNAL, columnValues.source_type);
+            } else {
+                assignments.push(`\`${column}\` = ?`);
+                params.push(columnValues[column]);
+            }
+        }
         await this.pool.query(
-            `UPDATE ${this.table(appId)} SET ${assignments}, admin_modified_at = NOW()
+            `UPDATE ${this.table(appId)} SET ${assignments.join(', ')}, admin_modified_at = NOW()
              WHERE public_id IN (?) AND ${STATE.ACTIVE}`,
-            [...columns.map((column) => columnValues[column]), matched]
+            [...params, matched]
         );
         return matched;
     }
@@ -557,6 +482,7 @@ module.exports.WRITABLE_COLUMNS = WRITABLE_COLUMNS;
 module.exports.escapeLike = escapeLike;
 module.exports.toApiRow = toApiRow;
 module.exports.buildFilter = buildFilter;
-module.exports.chooseBucket = chooseBucket;
-module.exports.BREAKDOWN_COLUMNS = BREAKDOWN_COLUMNS;
-module.exports.BUCKET_EXPRESSION = BUCKET_EXPRESSION;
+module.exports.chooseBucket = analysis.chooseBucket;
+module.exports.BREAKDOWN_COLUMNS = analysis.BREAKDOWN_COLUMNS;
+module.exports.FILTER_COLUMNS = analysis.FILTER_COLUMNS;
+module.exports.BUCKET_EXPRESSION = analysis.BUCKET_EXPRESSION;