@harshankur/viewcounter 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +115 -0
- package/LICENSE +21 -0
- package/README.md +797 -0
- package/allowed.sample.json +24 -0
- package/config/index.js +362 -0
- package/constants.js +229 -0
- package/db/DatabaseManager.js +704 -0
- package/db/schema.sql +71 -0
- package/dbInfo.sample.json +8 -0
- package/index.js +184 -0
- package/middleware/auth.js +194 -0
- package/middleware/security.js +109 -0
- package/middleware/validation.js +212 -0
- package/package.json +81 -0
- package/routes/analytics.js +456 -0
- package/scripts/setup.js +191 -0
- package/utils/appIdUtils.js +38 -0
- package/utils/errorUtils.js +157 -0
- package/utils/ipUtils.js +63 -0
- package/utils/logger.js +138 -0
- package/utils/privacyUtils.js +107 -0
- package/utils/referrerParser.js +137 -0
- package/utils/secretStore.js +57 -0
- package/utils/stringUtils.js +36 -0
- package/utils/userAgentParser.js +68 -0
|
@@ -0,0 +1,704 @@
|
|
|
1
|
+
const crypto = require('crypto');
|
|
2
|
+
const mysql = require('mysql2/promise');
|
|
3
|
+
|
|
4
|
+
const {
|
|
5
|
+
APP_REGISTRY_TABLE,
|
|
6
|
+
DATABASE,
|
|
7
|
+
EVENT_TYPE,
|
|
8
|
+
FIELD_MAX_LENGTH,
|
|
9
|
+
QUERY_LIMITS,
|
|
10
|
+
SERVER,
|
|
11
|
+
TOP_N_RESULTS,
|
|
12
|
+
TREND_PERIOD,
|
|
13
|
+
} = require('../constants');
|
|
14
|
+
const PrivacyUtils = require('../utils/privacyUtils');
|
|
15
|
+
const logger = require('../utils/logger');
|
|
16
|
+
const { getError, logWarning, ErrorType, WarningType } = require('../utils/errorUtils');
|
|
17
|
+
const { truncate } = require('../utils/stringUtils');
|
|
18
|
+
const { isValidAppId } = require('../utils/appIdUtils');
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Columns returned for a session lookup.
|
|
22
|
+
*
|
|
23
|
+
* Deliberately explicit rather than `SELECT *`. The previous wildcard returned
|
|
24
|
+
* `visitor_hash` — the pseudonymous visitor identifier itself — to any caller
|
|
25
|
+
* of the sessions endpoint.
|
|
26
|
+
*/
|
|
27
|
+
const SESSION_COLUMNS = [
|
|
28
|
+
'id',
|
|
29
|
+
'country',
|
|
30
|
+
'timestamp',
|
|
31
|
+
'devicesize',
|
|
32
|
+
'page_path',
|
|
33
|
+
'page_title',
|
|
34
|
+
'referrer_domain',
|
|
35
|
+
'source_type',
|
|
36
|
+
'browser',
|
|
37
|
+
'os',
|
|
38
|
+
'device_type',
|
|
39
|
+
'event_type',
|
|
40
|
+
'event_data',
|
|
41
|
+
].join(', ');
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Registry of dynamically provisioned apps.
|
|
45
|
+
*
|
|
46
|
+
* Without this, the set of tenants was whatever `allowed.json` said at boot, so
|
|
47
|
+
* adding one meant editing config and restarting the process. The registry
|
|
48
|
+
* makes tenants data rather than configuration.
|
|
49
|
+
*
|
|
50
|
+
* `id` is a generated UUID and is the row's identity; `app_id` is a uniqueness
|
|
51
|
+
* *constraint*, not an identity (CODE_STANDARDS.md §8). The distinction matters
|
|
52
|
+
* the first time an app is renamed, or deleted and a later one reuses the name —
|
|
53
|
+
* with the natural key as the primary key, anything referencing the old row
|
|
54
|
+
* would silently re-point at the new one.
|
|
55
|
+
*/
|
|
56
|
+
const APP_REGISTRY_DDL = `
|
|
57
|
+
CREATE TABLE IF NOT EXISTS \`${APP_REGISTRY_TABLE}\` (
|
|
58
|
+
\`id\` CHAR(36) PRIMARY KEY,
|
|
59
|
+
\`app_id\` VARCHAR(64) NOT NULL UNIQUE,
|
|
60
|
+
\`origins\` JSON DEFAULT NULL,
|
|
61
|
+
\`created_at\` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
62
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
63
|
+
`;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* DDL for one app's event table.
|
|
67
|
+
*
|
|
68
|
+
* `appId` is interpolated because MySQL cannot bind an identifier as a
|
|
69
|
+
* parameter. Every caller must have passed it through `isValidAppId` first —
|
|
70
|
+
* the assertion below is the backstop, not the primary gate.
|
|
71
|
+
*
|
|
72
|
+
* @param {string} appId
|
|
73
|
+
* @returns {string}
|
|
74
|
+
*/
|
|
75
|
+
function appTableDDL(appId) {
|
|
76
|
+
if (!isValidAppId(appId)) {
|
|
77
|
+
throw getError(ErrorType.INVALID_APP_ID, { appId });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return `
|
|
81
|
+
CREATE TABLE IF NOT EXISTS \`${appId}\` (
|
|
82
|
+
\`id\` BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
83
|
+
\`masked_ip\` VARCHAR(${FIELD_MAX_LENGTH.MASKED_IP}) NOT NULL,
|
|
84
|
+
\`visitor_hash\` VARCHAR(${FIELD_MAX_LENGTH.VISITOR_HASH}) NOT NULL,
|
|
85
|
+
\`country\` VARCHAR(${FIELD_MAX_LENGTH.COUNTRY}) DEFAULT NULL,
|
|
86
|
+
\`timestamp\` DATETIME NOT NULL,
|
|
87
|
+
\`devicesize\` VARCHAR(${FIELD_MAX_LENGTH.DEVICE_SIZE}) NOT NULL,
|
|
88
|
+
\`page_path\` VARCHAR(${FIELD_MAX_LENGTH.PAGE_PATH}) DEFAULT NULL,
|
|
89
|
+
\`page_title\` VARCHAR(${FIELD_MAX_LENGTH.PAGE_TITLE}) DEFAULT NULL,
|
|
90
|
+
\`referrer\` VARCHAR(${FIELD_MAX_LENGTH.REFERRER}) DEFAULT NULL,
|
|
91
|
+
\`referrer_domain\` VARCHAR(${FIELD_MAX_LENGTH.REFERRER_DOMAIN}) DEFAULT NULL,
|
|
92
|
+
\`source_type\` VARCHAR(${FIELD_MAX_LENGTH.SOURCE_TYPE}) DEFAULT NULL,
|
|
93
|
+
\`browser\` VARCHAR(${FIELD_MAX_LENGTH.BROWSER}) DEFAULT NULL,
|
|
94
|
+
\`browser_version\` VARCHAR(${FIELD_MAX_LENGTH.BROWSER_VERSION}) DEFAULT NULL,
|
|
95
|
+
\`os\` VARCHAR(${FIELD_MAX_LENGTH.OS}) DEFAULT NULL,
|
|
96
|
+
\`os_version\` VARCHAR(${FIELD_MAX_LENGTH.OS_VERSION}) DEFAULT NULL,
|
|
97
|
+
\`device_type\` VARCHAR(${FIELD_MAX_LENGTH.DEVICE_TYPE}) DEFAULT NULL,
|
|
98
|
+
\`session_id\` VARCHAR(${FIELD_MAX_LENGTH.SESSION_ID}) DEFAULT NULL,
|
|
99
|
+
\`event_type\` VARCHAR(${FIELD_MAX_LENGTH.EVENT_TYPE}) DEFAULT '${EVENT_TYPE.PAGEVIEW}',
|
|
100
|
+
\`event_data\` JSON DEFAULT NULL,
|
|
101
|
+
\`is_unique\` TINYINT(1) DEFAULT 1,
|
|
102
|
+
INDEX \`idx_timestamp\` (\`timestamp\`),
|
|
103
|
+
INDEX \`idx_visitor_timestamp\` (\`visitor_hash\`, \`timestamp\`),
|
|
104
|
+
INDEX \`idx_masked_ip\` (\`masked_ip\`),
|
|
105
|
+
INDEX \`idx_country\` (\`country\`),
|
|
106
|
+
INDEX \`idx_devicesize\` (\`devicesize\`),
|
|
107
|
+
INDEX \`idx_page_path\` (\`page_path\`(255)),
|
|
108
|
+
INDEX \`idx_referrer_domain\` (\`referrer_domain\`),
|
|
109
|
+
INDEX \`idx_source_type\` (\`source_type\`),
|
|
110
|
+
INDEX \`idx_browser\` (\`browser\`),
|
|
111
|
+
INDEX \`idx_os\` (\`os\`),
|
|
112
|
+
INDEX \`idx_device_type\` (\`device_type\`),
|
|
113
|
+
INDEX \`idx_session_id\` (\`session_id\`),
|
|
114
|
+
INDEX \`idx_event_type\` (\`event_type\`),
|
|
115
|
+
INDEX \`idx_is_unique\` (\`is_unique\`)
|
|
116
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
117
|
+
`;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Apply a server-side statement timeout to a newly opened pool connection.
|
|
122
|
+
*
|
|
123
|
+
* `mysql2/promise`'s pool emits the RAW callback-style connection on its
|
|
124
|
+
* `connection` event, not the promise-wrapped one. Its `query()` returns a
|
|
125
|
+
* `Query`, and mysql2 deliberately makes `.then()`/`.catch()` on a `Query`
|
|
126
|
+
* throw — so treating it as a promise crashes the process on the very first
|
|
127
|
+
* database connection. The callback form is the correct API for that object.
|
|
128
|
+
*
|
|
129
|
+
* Failure is swallowed on purpose: MariaDB and MySQL < 5.7.8 have no
|
|
130
|
+
* MAX_EXECUTION_TIME, and the pool's own limits still bound concurrency there.
|
|
131
|
+
*
|
|
132
|
+
* @param {object} connection raw or promise-wrapped mysql2 connection
|
|
133
|
+
*/
|
|
134
|
+
function setStatementTimeout(connection) {
|
|
135
|
+
const sql = 'SET SESSION MAX_EXECUTION_TIME = ?';
|
|
136
|
+
const params = [DATABASE.QUERY_TIMEOUT_MS];
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
// A raw connection exposes .promise(); a promise-wrapped one does not.
|
|
140
|
+
if (typeof connection.promise === 'function') {
|
|
141
|
+
connection.query(sql, params, () => {
|
|
142
|
+
// Callback form: the error is delivered here, never thrown, and
|
|
143
|
+
// never left as an unhandled 'error' event on the Query.
|
|
144
|
+
});
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const result = connection.query(sql, params);
|
|
149
|
+
if (result && typeof result.catch === 'function') result.catch(() => {});
|
|
150
|
+
} catch {
|
|
151
|
+
// An engine that rejects the statement outright must not stop startup.
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Database Manager
|
|
157
|
+
* Handles both 'connect' mode (use existing DB) and 'create' mode (auto-create DB and tables)
|
|
158
|
+
*/
|
|
159
|
+
class DatabaseManager {
|
|
160
|
+
constructor(config) {
|
|
161
|
+
this.config = config;
|
|
162
|
+
this.pool = null;
|
|
163
|
+
this.mode = config.mode || 'connect';
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** @throws {Error} when a query is attempted before initialize() */
|
|
167
|
+
assertReady() {
|
|
168
|
+
if (!this.pool) {
|
|
169
|
+
throw getError(ErrorType.DATABASE_NOT_INITIALIZED);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Initialize database connection and optionally create schema
|
|
175
|
+
*/
|
|
176
|
+
async initialize(allowedAppIds = []) {
|
|
177
|
+
try {
|
|
178
|
+
if (this.mode === 'create') {
|
|
179
|
+
await this.createDatabaseAndTables(allowedAppIds);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
this.pool = mysql.createPool({
|
|
183
|
+
host: this.config.host,
|
|
184
|
+
port: this.config.port,
|
|
185
|
+
user: this.config.user,
|
|
186
|
+
password: this.config.password,
|
|
187
|
+
database: this.config.database,
|
|
188
|
+
waitForConnections: true,
|
|
189
|
+
connectionLimit: DATABASE.CONNECTION_LIMIT,
|
|
190
|
+
// Finite, so a saturated pool rejects rather than queueing
|
|
191
|
+
// unboundedly. With an unbounded queue a burst of expensive
|
|
192
|
+
// aggregates stalls every later request, including /health.
|
|
193
|
+
queueLimit: DATABASE.QUEUE_LIMIT,
|
|
194
|
+
connectTimeout: DATABASE.CONNECT_TIMEOUT_MS,
|
|
195
|
+
enableKeepAlive: true,
|
|
196
|
+
keepAliveInitialDelay: 0,
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
// Server-side statement timeout. Bounds the cost of any single
|
|
200
|
+
// read so one caller cannot pin a connection indefinitely.
|
|
201
|
+
if (typeof this.pool.on === 'function') {
|
|
202
|
+
this.pool.on('connection', (connection) => {
|
|
203
|
+
setStatementTimeout(connection);
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
await this.pool.query('SELECT 1');
|
|
208
|
+
logger.info(`Database connected (mode: ${this.mode})`);
|
|
209
|
+
|
|
210
|
+
return true;
|
|
211
|
+
} catch (cause) {
|
|
212
|
+
throw getError(ErrorType.DATABASE_CONNECTION_FAILED, {
|
|
213
|
+
host: this.config.host,
|
|
214
|
+
port: this.config.port,
|
|
215
|
+
database: this.config.database,
|
|
216
|
+
cause: cause.message,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Create database and tables (create mode only)
|
|
223
|
+
*/
|
|
224
|
+
async createDatabaseAndTables(allowedAppIds) {
|
|
225
|
+
logger.info('Creating database and tables...');
|
|
226
|
+
|
|
227
|
+
const connection = await mysql.createConnection({
|
|
228
|
+
host: this.config.host,
|
|
229
|
+
port: this.config.port,
|
|
230
|
+
user: this.config.user,
|
|
231
|
+
password: this.config.password,
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
await connection.query(
|
|
236
|
+
`CREATE DATABASE IF NOT EXISTS \`${this.config.database}\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
|
|
237
|
+
);
|
|
238
|
+
logger.info(`Database '${this.config.database}' ready`);
|
|
239
|
+
|
|
240
|
+
await connection.query(`USE \`${this.config.database}\``);
|
|
241
|
+
|
|
242
|
+
await connection.query(`
|
|
243
|
+
CREATE TABLE IF NOT EXISTS \`_migrations\` (
|
|
244
|
+
\`id\` INT AUTO_INCREMENT PRIMARY KEY,
|
|
245
|
+
\`version\` VARCHAR(50) NOT NULL UNIQUE,
|
|
246
|
+
\`applied_at\` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
247
|
+
INDEX \`idx_version\` (\`version\`)
|
|
248
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
249
|
+
`);
|
|
250
|
+
|
|
251
|
+
await connection.query(APP_REGISTRY_DDL);
|
|
252
|
+
|
|
253
|
+
for (const appId of allowedAppIds) {
|
|
254
|
+
await connection.query(appTableDDL(appId));
|
|
255
|
+
logger.info(`Table '${appId}' ready`);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
await connection.query(
|
|
259
|
+
`INSERT IGNORE INTO \`_migrations\` (\`version\`) VALUES (?)`,
|
|
260
|
+
[DATABASE.SCHEMA_VERSION]
|
|
261
|
+
);
|
|
262
|
+
|
|
263
|
+
} finally {
|
|
264
|
+
await connection.end();
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ---- Tenant registry ---------------------------------------------------
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Ensure the registry table exists.
|
|
272
|
+
* Idempotent, and safe in `connect` mode: the registry is the service's own
|
|
273
|
+
* bookkeeping, not part of the operator's pre-existing schema.
|
|
274
|
+
*/
|
|
275
|
+
async ensureRegistry() {
|
|
276
|
+
this.assertReady();
|
|
277
|
+
await this.pool.query(APP_REGISTRY_DDL);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* App IDs registered in the database.
|
|
282
|
+
* @returns {Promise<string[]>}
|
|
283
|
+
*/
|
|
284
|
+
async listRegisteredApps() {
|
|
285
|
+
this.assertReady();
|
|
286
|
+
await this.ensureRegistry();
|
|
287
|
+
|
|
288
|
+
const [rows] = await this.pool.query(
|
|
289
|
+
`SELECT app_id FROM \`${APP_REGISTRY_TABLE}\` ORDER BY app_id ASC`
|
|
290
|
+
);
|
|
291
|
+
// Filtered on the way out as well as in: a row written by an older
|
|
292
|
+
// build, or by hand, must not become a table identifier unchecked.
|
|
293
|
+
return rows.map((row) => row.app_id).filter(isValidAppId);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Provision a new app: validate, create its table, record it.
|
|
298
|
+
*
|
|
299
|
+
* Idempotent — re-registering an existing app is a no-op rather than an
|
|
300
|
+
* error, so a retried provisioning call cannot fail halfway.
|
|
301
|
+
*
|
|
302
|
+
* @param {string} appId
|
|
303
|
+
* @param {string[]} [origins] site origins permitted to write to it
|
|
304
|
+
* @returns {Promise<{appId: string, created: boolean}>}
|
|
305
|
+
* @throws {Error} ErrorType.INVALID_APP_ID for an unsafe identifier
|
|
306
|
+
*/
|
|
307
|
+
async registerApp(appId, origins = []) {
|
|
308
|
+
this.assertReady();
|
|
309
|
+
|
|
310
|
+
// The gate. Everything downstream interpolates this into DDL/DML.
|
|
311
|
+
if (!isValidAppId(appId)) {
|
|
312
|
+
throw getError(ErrorType.INVALID_APP_ID, { appId });
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
await this.ensureRegistry();
|
|
316
|
+
|
|
317
|
+
const [existing] = await this.pool.query(
|
|
318
|
+
`SELECT app_id FROM \`${APP_REGISTRY_TABLE}\` WHERE app_id = ? LIMIT 1`,
|
|
319
|
+
[appId]
|
|
320
|
+
);
|
|
321
|
+
|
|
322
|
+
// The table is (re)created regardless, so an app registered before its
|
|
323
|
+
// table existed still converges to a working state.
|
|
324
|
+
await this.pool.query(appTableDDL(appId));
|
|
325
|
+
|
|
326
|
+
if (existing.length > 0) {
|
|
327
|
+
logWarning(WarningType.APP_ALREADY_REGISTERED, { appId });
|
|
328
|
+
return { appId, created: false };
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
await this.pool.query(
|
|
332
|
+
`INSERT INTO \`${APP_REGISTRY_TABLE}\` (id, app_id, origins) VALUES (?, ?, ?)`,
|
|
333
|
+
[crypto.randomUUID(), appId, origins.length ? JSON.stringify(origins) : null]
|
|
334
|
+
);
|
|
335
|
+
|
|
336
|
+
logger.info(`Registered app '${appId}'`);
|
|
337
|
+
return { appId, created: true };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Per-app origin allowlists recorded in the registry.
|
|
342
|
+
* @returns {Promise<Record<string, string[]>>}
|
|
343
|
+
*/
|
|
344
|
+
async loadRegisteredOrigins() {
|
|
345
|
+
this.assertReady();
|
|
346
|
+
await this.ensureRegistry();
|
|
347
|
+
|
|
348
|
+
const [rows] = await this.pool.query(
|
|
349
|
+
`SELECT app_id, origins FROM \`${APP_REGISTRY_TABLE}\` WHERE origins IS NOT NULL`
|
|
350
|
+
);
|
|
351
|
+
|
|
352
|
+
const map = {};
|
|
353
|
+
for (const row of rows) {
|
|
354
|
+
// mysql2 returns a JSON column already parsed; tolerate a string
|
|
355
|
+
// for drivers or mocks that do not.
|
|
356
|
+
const value = typeof row.origins === 'string' ? JSON.parse(row.origins) : row.origins;
|
|
357
|
+
if (Array.isArray(value) && value.length) map[row.app_id] = value;
|
|
358
|
+
}
|
|
359
|
+
return map;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Register a view/event with all tracking data.
|
|
364
|
+
*
|
|
365
|
+
* Every value is bound as a parameter. `appId` is the sole interpolated
|
|
366
|
+
* identifier and is only ever reached after the caller has checked it
|
|
367
|
+
* against the configured allowlist.
|
|
368
|
+
*/
|
|
369
|
+
async registerEvent(appId, data) {
|
|
370
|
+
this.assertReady();
|
|
371
|
+
|
|
372
|
+
const {
|
|
373
|
+
ip,
|
|
374
|
+
country,
|
|
375
|
+
deviceSize,
|
|
376
|
+
pagePath,
|
|
377
|
+
pageTitle,
|
|
378
|
+
referrer,
|
|
379
|
+
referrerDomain,
|
|
380
|
+
sourceType,
|
|
381
|
+
browser,
|
|
382
|
+
browserVersion,
|
|
383
|
+
os,
|
|
384
|
+
osVersion,
|
|
385
|
+
deviceType,
|
|
386
|
+
sessionId,
|
|
387
|
+
eventType = EVENT_TYPE.PAGEVIEW,
|
|
388
|
+
eventData,
|
|
389
|
+
uniqueWindowHours = SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS,
|
|
390
|
+
userAgent = '',
|
|
391
|
+
visitorSecret,
|
|
392
|
+
} = data;
|
|
393
|
+
|
|
394
|
+
// Privacy boundary. Neither the raw IP nor the raw User-Agent is bound
|
|
395
|
+
// into any statement below; only the masked address and the keyed,
|
|
396
|
+
// rotating hash derived from them.
|
|
397
|
+
const hashedVisitor = PrivacyUtils.generateVisitorHash(
|
|
398
|
+
ip,
|
|
399
|
+
userAgent,
|
|
400
|
+
visitorSecret,
|
|
401
|
+
uniqueWindowHours,
|
|
402
|
+
);
|
|
403
|
+
const maskedIp = PrivacyUtils.maskIP(ip);
|
|
404
|
+
|
|
405
|
+
let isUnique = 1;
|
|
406
|
+
if (uniqueWindowHours > 0 && eventType === EVENT_TYPE.PAGEVIEW) {
|
|
407
|
+
const [existing] = await this.pool.query(
|
|
408
|
+
`SELECT id FROM \`${appId}\`
|
|
409
|
+
WHERE visitor_hash = ? AND event_type = ? AND timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)
|
|
410
|
+
LIMIT 1`,
|
|
411
|
+
[hashedVisitor, EVENT_TYPE.PAGEVIEW, uniqueWindowHours]
|
|
412
|
+
);
|
|
413
|
+
|
|
414
|
+
if (existing.length > 0) {
|
|
415
|
+
isUnique = 0;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const [result] = await this.pool.query(
|
|
420
|
+
`INSERT INTO \`${appId}\` (
|
|
421
|
+
masked_ip, visitor_hash, country, timestamp, devicesize,
|
|
422
|
+
page_path, page_title,
|
|
423
|
+
referrer, referrer_domain, source_type,
|
|
424
|
+
browser, browser_version, os, os_version, device_type,
|
|
425
|
+
session_id, event_type, event_data, is_unique
|
|
426
|
+
) VALUES (?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
427
|
+
[
|
|
428
|
+
truncate(maskedIp, FIELD_MAX_LENGTH.MASKED_IP),
|
|
429
|
+
hashedVisitor,
|
|
430
|
+
truncate(country, FIELD_MAX_LENGTH.COUNTRY),
|
|
431
|
+
truncate(deviceSize, FIELD_MAX_LENGTH.DEVICE_SIZE),
|
|
432
|
+
truncate(pagePath, FIELD_MAX_LENGTH.PAGE_PATH),
|
|
433
|
+
truncate(pageTitle, FIELD_MAX_LENGTH.PAGE_TITLE),
|
|
434
|
+
truncate(referrer, FIELD_MAX_LENGTH.REFERRER),
|
|
435
|
+
truncate(referrerDomain, FIELD_MAX_LENGTH.REFERRER_DOMAIN),
|
|
436
|
+
truncate(sourceType, FIELD_MAX_LENGTH.SOURCE_TYPE),
|
|
437
|
+
truncate(browser, FIELD_MAX_LENGTH.BROWSER),
|
|
438
|
+
truncate(browserVersion, FIELD_MAX_LENGTH.BROWSER_VERSION),
|
|
439
|
+
truncate(os, FIELD_MAX_LENGTH.OS),
|
|
440
|
+
truncate(osVersion, FIELD_MAX_LENGTH.OS_VERSION),
|
|
441
|
+
truncate(deviceType, FIELD_MAX_LENGTH.DEVICE_TYPE),
|
|
442
|
+
truncate(sessionId, FIELD_MAX_LENGTH.SESSION_ID),
|
|
443
|
+
truncate(eventType, FIELD_MAX_LENGTH.EVENT_TYPE),
|
|
444
|
+
eventData ? JSON.stringify(eventData) : null,
|
|
445
|
+
isUnique,
|
|
446
|
+
]
|
|
447
|
+
);
|
|
448
|
+
|
|
449
|
+
return {
|
|
450
|
+
duplicate: isUnique === 0,
|
|
451
|
+
insertId: result.insertId,
|
|
452
|
+
isUnique: isUnique === 1,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Register a view (backward compatible wrapper)
|
|
458
|
+
*/
|
|
459
|
+
async registerView(appId, ip, country, deviceSize, uniqueWindowHours, visitorSecret) {
|
|
460
|
+
return this.registerEvent(appId, {
|
|
461
|
+
ip,
|
|
462
|
+
country,
|
|
463
|
+
deviceSize,
|
|
464
|
+
uniqueWindowHours,
|
|
465
|
+
visitorSecret,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Get statistics for an app
|
|
471
|
+
*/
|
|
472
|
+
async getStats(appId) {
|
|
473
|
+
this.assertReady();
|
|
474
|
+
|
|
475
|
+
const [totalStats] = await this.pool.query(
|
|
476
|
+
`SELECT
|
|
477
|
+
COUNT(*) as total_views,
|
|
478
|
+
SUM(CASE WHEN is_unique = 1 THEN 1 ELSE 0 END) as unique_views,
|
|
479
|
+
COUNT(DISTINCT visitor_hash) as unique_visitors
|
|
480
|
+
FROM \`${appId}\``
|
|
481
|
+
);
|
|
482
|
+
|
|
483
|
+
const stats = totalStats[0];
|
|
484
|
+
|
|
485
|
+
const [byCountry] = await this.pool.query(
|
|
486
|
+
`SELECT country, COUNT(*) as count FROM \`${appId}\`
|
|
487
|
+
WHERE country IS NOT NULL
|
|
488
|
+
GROUP BY country
|
|
489
|
+
ORDER BY count DESC
|
|
490
|
+
LIMIT ?`,
|
|
491
|
+
[TOP_N_RESULTS]
|
|
492
|
+
);
|
|
493
|
+
|
|
494
|
+
const [byDevice] = await this.pool.query(
|
|
495
|
+
`SELECT devicesize, COUNT(*) as count FROM \`${appId}\`
|
|
496
|
+
GROUP BY devicesize
|
|
497
|
+
ORDER BY count DESC`
|
|
498
|
+
);
|
|
499
|
+
|
|
500
|
+
const [recent] = await this.pool.query(
|
|
501
|
+
`SELECT COUNT(*) as count FROM \`${appId}\`
|
|
502
|
+
WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? HOUR)`,
|
|
503
|
+
[SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS]
|
|
504
|
+
);
|
|
505
|
+
|
|
506
|
+
return {
|
|
507
|
+
totalViews: stats.total_views,
|
|
508
|
+
uniqueViews: stats.unique_views,
|
|
509
|
+
uniqueVisitors: stats.unique_visitors,
|
|
510
|
+
last24Hours: recent[0].count,
|
|
511
|
+
byCountry,
|
|
512
|
+
byDevice,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Get recent views with pagination
|
|
518
|
+
*/
|
|
519
|
+
async getViews(appId, limit = QUERY_LIMITS.VIEWS_LIMIT_DEFAULT, offset = QUERY_LIMITS.OFFSET_DEFAULT) {
|
|
520
|
+
this.assertReady();
|
|
521
|
+
|
|
522
|
+
const [views] = await this.pool.query(
|
|
523
|
+
`SELECT masked_ip, country, timestamp, devicesize
|
|
524
|
+
FROM \`${appId}\`
|
|
525
|
+
ORDER BY timestamp DESC
|
|
526
|
+
LIMIT ? OFFSET ?`,
|
|
527
|
+
[limit, offset]
|
|
528
|
+
);
|
|
529
|
+
|
|
530
|
+
const [total] = await this.pool.query(
|
|
531
|
+
`SELECT COUNT(*) as count FROM \`${appId}\``
|
|
532
|
+
);
|
|
533
|
+
|
|
534
|
+
return {
|
|
535
|
+
views,
|
|
536
|
+
total: total[0].count,
|
|
537
|
+
limit,
|
|
538
|
+
offset,
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* Get time-based trends.
|
|
544
|
+
*
|
|
545
|
+
* `groupBy` is chosen from three hard-coded literals, never built from
|
|
546
|
+
* caller input, so the interpolation below cannot carry user data.
|
|
547
|
+
*/
|
|
548
|
+
async getTrends(appId, period = TREND_PERIOD.DAILY, days = QUERY_LIMITS.TREND_DAYS_DEFAULT) {
|
|
549
|
+
this.assertReady();
|
|
550
|
+
|
|
551
|
+
let groupBy;
|
|
552
|
+
if (period === TREND_PERIOD.HOURLY) {
|
|
553
|
+
groupBy = 'DATE_FORMAT(timestamp, "%Y-%m-%d %H:00:00")';
|
|
554
|
+
} else if (period === TREND_PERIOD.WEEKLY) {
|
|
555
|
+
groupBy = 'DATE_FORMAT(timestamp, "%Y-%u")';
|
|
556
|
+
} else {
|
|
557
|
+
groupBy = 'DATE(timestamp)';
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const [trends] = await this.pool.query(
|
|
561
|
+
`SELECT ${groupBy} as period, COUNT(*) as count
|
|
562
|
+
FROM \`${appId}\`
|
|
563
|
+
WHERE timestamp > DATE_SUB(NOW(), INTERVAL ? DAY)
|
|
564
|
+
GROUP BY period
|
|
565
|
+
ORDER BY period ASC`,
|
|
566
|
+
[days]
|
|
567
|
+
);
|
|
568
|
+
|
|
569
|
+
return trends;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Get referrer statistics
|
|
574
|
+
*/
|
|
575
|
+
async getReferrerStats(appId, limit = QUERY_LIMITS.LIST_LIMIT_DEFAULT) {
|
|
576
|
+
this.assertReady();
|
|
577
|
+
|
|
578
|
+
const [bySource] = await this.pool.query(
|
|
579
|
+
`SELECT source_type, COUNT(*) as count
|
|
580
|
+
FROM \`${appId}\`
|
|
581
|
+
WHERE source_type IS NOT NULL
|
|
582
|
+
GROUP BY source_type
|
|
583
|
+
ORDER BY count DESC`
|
|
584
|
+
);
|
|
585
|
+
|
|
586
|
+
const [byDomain] = await this.pool.query(
|
|
587
|
+
`SELECT referrer_domain, COUNT(*) as count
|
|
588
|
+
FROM \`${appId}\`
|
|
589
|
+
WHERE referrer_domain IS NOT NULL
|
|
590
|
+
GROUP BY referrer_domain
|
|
591
|
+
ORDER BY count DESC
|
|
592
|
+
LIMIT ?`,
|
|
593
|
+
[limit]
|
|
594
|
+
);
|
|
595
|
+
|
|
596
|
+
return { bySource, byDomain };
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Get browser/OS statistics
|
|
601
|
+
*/
|
|
602
|
+
async getBrowserStats(appId) {
|
|
603
|
+
this.assertReady();
|
|
604
|
+
|
|
605
|
+
const [byBrowser] = await this.pool.query(
|
|
606
|
+
`SELECT browser, COUNT(*) as count
|
|
607
|
+
FROM \`${appId}\`
|
|
608
|
+
WHERE browser IS NOT NULL
|
|
609
|
+
GROUP BY browser
|
|
610
|
+
ORDER BY count DESC
|
|
611
|
+
LIMIT ?`,
|
|
612
|
+
[TOP_N_RESULTS]
|
|
613
|
+
);
|
|
614
|
+
|
|
615
|
+
const [byOS] = await this.pool.query(
|
|
616
|
+
`SELECT os, COUNT(*) as count
|
|
617
|
+
FROM \`${appId}\`
|
|
618
|
+
WHERE os IS NOT NULL
|
|
619
|
+
GROUP BY os
|
|
620
|
+
ORDER BY count DESC
|
|
621
|
+
LIMIT ?`,
|
|
622
|
+
[TOP_N_RESULTS]
|
|
623
|
+
);
|
|
624
|
+
|
|
625
|
+
const [byDeviceType] = await this.pool.query(
|
|
626
|
+
`SELECT device_type, COUNT(*) as count
|
|
627
|
+
FROM \`${appId}\`
|
|
628
|
+
WHERE device_type IS NOT NULL
|
|
629
|
+
GROUP BY device_type
|
|
630
|
+
ORDER BY count DESC`
|
|
631
|
+
);
|
|
632
|
+
|
|
633
|
+
return { byBrowser, byOS, byDeviceType };
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Get page statistics
|
|
638
|
+
*/
|
|
639
|
+
async getPageStats(appId, limit = QUERY_LIMITS.LIST_LIMIT_DEFAULT) {
|
|
640
|
+
this.assertReady();
|
|
641
|
+
|
|
642
|
+
const [pages] = await this.pool.query(
|
|
643
|
+
`SELECT page_path, page_title, COUNT(*) as views
|
|
644
|
+
FROM \`${appId}\`
|
|
645
|
+
WHERE page_path IS NOT NULL
|
|
646
|
+
GROUP BY page_path, page_title
|
|
647
|
+
ORDER BY views DESC
|
|
648
|
+
LIMIT ?`,
|
|
649
|
+
[limit]
|
|
650
|
+
);
|
|
651
|
+
|
|
652
|
+
return pages;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Get session details.
|
|
657
|
+
* Returns an explicit column list; `visitor_hash` is never exposed.
|
|
658
|
+
*/
|
|
659
|
+
async getSessionDetails(appId, sessionId) {
|
|
660
|
+
this.assertReady();
|
|
661
|
+
|
|
662
|
+
const [events] = await this.pool.query(
|
|
663
|
+
`SELECT ${SESSION_COLUMNS}
|
|
664
|
+
FROM \`${appId}\`
|
|
665
|
+
WHERE session_id = ?
|
|
666
|
+
ORDER BY timestamp ASC`,
|
|
667
|
+
[sessionId]
|
|
668
|
+
);
|
|
669
|
+
|
|
670
|
+
return events;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Health check
|
|
675
|
+
*/
|
|
676
|
+
async healthCheck() {
|
|
677
|
+
if (!this.pool) {
|
|
678
|
+
return { healthy: false, error: 'Pool not initialized' };
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
try {
|
|
682
|
+
await this.pool.query('SELECT 1');
|
|
683
|
+
return { healthy: true };
|
|
684
|
+
} catch (cause) {
|
|
685
|
+
return { healthy: false, error: cause.message };
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Gracefully close all connections
|
|
691
|
+
*/
|
|
692
|
+
async close() {
|
|
693
|
+
if (this.pool) {
|
|
694
|
+
await this.pool.end();
|
|
695
|
+
this.pool = null;
|
|
696
|
+
logger.info('Database connections closed');
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
module.exports = DatabaseManager;
|
|
702
|
+
module.exports.SESSION_COLUMNS = SESSION_COLUMNS;
|
|
703
|
+
module.exports.appTableDDL = appTableDDL;
|
|
704
|
+
module.exports.APP_REGISTRY_DDL = APP_REGISTRY_DDL;
|