@harshankur/viewcounter 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +34 -10
- package/README.md +306 -136
- package/admin/css/admin.css +891 -198
- package/admin/index.html +13 -7
- package/admin/js/api.js +52 -6
- package/admin/js/appTabs.js +100 -0
- package/admin/js/charts.js +529 -189
- package/admin/js/constants.js +98 -9
- package/admin/js/dataTable.js +478 -0
- package/admin/js/format.js +58 -7
- package/admin/js/icons.js +168 -0
- package/admin/js/listbox.js +2 -1
- package/admin/js/logs.js +211 -60
- package/admin/js/main.js +201 -37
- package/admin/js/overview.js +905 -0
- package/admin/js/passwordPrompt.js +75 -0
- package/admin/js/table.js +12 -52
- package/admin/js/viewDialogs.js +30 -14
- package/admin/js/views.js +273 -207
- package/admin/locales/en.json +352 -63
- package/config/index.js +40 -5
- package/constants.js +126 -8
- package/db/AdminRepository.js +85 -159
- package/db/DatabaseManager.js +55 -7
- package/db/LogRepository.js +172 -35
- package/db/adminSchema.js +89 -4
- package/db/adminSessionStore.js +104 -0
- package/db/analysis.js +479 -0
- package/db/rejectionCounter.js +117 -0
- package/index.js +53 -19
- package/middleware/adminAuth.js +83 -43
- package/middleware/adminValidation.js +69 -3
- package/middleware/auth.js +2 -2
- package/middleware/security.js +26 -2
- package/middleware/validation.js +50 -2
- package/package.json +5 -2
- package/routes/admin.js +130 -22
- package/routes/analytics.js +197 -18
- package/tracker/tracker.js +191 -0
- package/utils/appIdUtils.js +1 -1
- package/utils/durationUtils.js +33 -0
- package/utils/errorUtils.js +4 -1
- package/utils/geoCity.js +87 -0
- package/utils/ipUtils.js +1 -1
- package/utils/privacyUtils.js +2 -2
- package/utils/referrerParser.js +23 -5
- package/utils/secretStore.js +1 -1
- package/utils/userAgentParser.js +52 -3
- package/utils/visitorContext.js +70 -0
- package/admin/js/insights.js +0 -192
package/routes/analytics.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
2
4
|
const express = require('express');
|
|
3
5
|
const rateLimit = require('express-rate-limit');
|
|
4
6
|
const geoip = require('geoip-country');
|
|
@@ -7,6 +9,9 @@ const {
|
|
|
7
9
|
EVENT_TYPE,
|
|
8
10
|
HTTP_STATUS,
|
|
9
11
|
QUERY_LIMITS,
|
|
12
|
+
REJECTION_REASON,
|
|
13
|
+
SOURCE_TYPE,
|
|
14
|
+
TRACKING,
|
|
10
15
|
TREND_PERIOD,
|
|
11
16
|
VIEW_LOG_SOURCE,
|
|
12
17
|
} = require('../constants');
|
|
@@ -16,19 +21,66 @@ const PrivacyUtils = require('../utils/privacyUtils');
|
|
|
16
21
|
const logger = require('../utils/logger');
|
|
17
22
|
const { getClientIp, isValidIP, normalizeIp } = require('../utils/ipUtils');
|
|
18
23
|
const { requireReadApiKey, requireAppScope, requireAdminApiKey, appsInScope } = require('../middleware/auth');
|
|
19
|
-
const { requireRegisteredOrigin, noStore } = require('../middleware/security');
|
|
24
|
+
const { requireRegisteredOrigin, requestOrigin, noStore } = require('../middleware/security');
|
|
25
|
+
const { createRejectionCounter } = require('../db/rejectionCounter');
|
|
26
|
+
const { hostnameOf, primaryLanguage, utmTags } = require('../utils/visitorContext');
|
|
20
27
|
const {
|
|
21
28
|
validateAppRegistration,
|
|
22
29
|
validateRegisterView,
|
|
23
30
|
validateEvent,
|
|
31
|
+
validateEngage,
|
|
24
32
|
validateStatsRequest,
|
|
25
33
|
validateTrendsRequest,
|
|
26
34
|
validateListRequest,
|
|
27
35
|
validateViewsRequest,
|
|
28
36
|
validateSessionRequest,
|
|
29
37
|
handleValidationErrors,
|
|
38
|
+
handleTrackingValidation,
|
|
30
39
|
} = require('../middleware/validation');
|
|
31
40
|
|
|
41
|
+
/**
|
|
42
|
+
* The tracker script sites include with <script src=".../tracker.js">. Read
|
|
43
|
+
* once; it is part of the package, not configuration.
|
|
44
|
+
*/
|
|
45
|
+
const TRACKER_SOURCE = fs.readFileSync(path.join(__dirname, '..', 'tracker', 'tracker.js'), 'utf8');
|
|
46
|
+
/** Cached briefly, so a fix reaches every site within the hour. */
|
|
47
|
+
const TRACKER_MAX_AGE_SECONDS = 60 * 60;
|
|
48
|
+
|
|
49
|
+
/** The tracking endpoints, by path, and the source the tracking log files them under. */
|
|
50
|
+
const TRACKING_PATHS = {
|
|
51
|
+
'/registerView': VIEW_LOG_SOURCE.REGISTER_VIEW,
|
|
52
|
+
'/event': VIEW_LOG_SOURCE.EVENT,
|
|
53
|
+
'/engage': VIEW_LOG_SOURCE.ENGAGE,
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {string} path a request path, relative to where the router is mounted
|
|
58
|
+
* @returns {string|null} the tracking source, or null for any other endpoint
|
|
59
|
+
*/
|
|
60
|
+
function trackingSourceFor(path) {
|
|
61
|
+
return Object.hasOwn(TRACKING_PATHS, path) ? TRACKING_PATHS[path] : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* sendBeacon posts text/plain, which needs no CORS preflight; a fetch may
|
|
66
|
+
* send JSON. Either way the handler sees an object, or an empty one for a
|
|
67
|
+
* body that is not JSON, which validation then refuses.
|
|
68
|
+
* @type {import('express').RequestHandler}
|
|
69
|
+
*/
|
|
70
|
+
function parseBeaconBody(req, res, next) {
|
|
71
|
+
if (typeof req.body === 'string') {
|
|
72
|
+
try {
|
|
73
|
+
const parsed = JSON.parse(req.body);
|
|
74
|
+
req.body = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
75
|
+
} catch {
|
|
76
|
+
req.body = {};
|
|
77
|
+
}
|
|
78
|
+
} else if (!req.body || typeof req.body !== 'object') {
|
|
79
|
+
req.body = {};
|
|
80
|
+
}
|
|
81
|
+
next();
|
|
82
|
+
}
|
|
83
|
+
|
|
32
84
|
/**
|
|
33
85
|
* Analytics routes.
|
|
34
86
|
*
|
|
@@ -73,13 +125,13 @@ function intQuery(req, name, fallback) {
|
|
|
73
125
|
* other tenant on the instance depends on, which is the failure mode that
|
|
74
126
|
* matters once the apps belong to different people.
|
|
75
127
|
*
|
|
76
|
-
* Keyed on appId only
|
|
128
|
+
* Keyed on appId only, never on IP, so it is unaffected by how the client's
|
|
77
129
|
* address is derived, and cannot be rotated away by a caller changing address.
|
|
78
130
|
*
|
|
79
131
|
* @param {{ perAppMax: number, windowMs: number }} rateLimitConfig
|
|
80
132
|
* @returns {import('express').RequestHandler}
|
|
81
133
|
*/
|
|
82
|
-
function buildPerAppLimiter(rateLimitConfig) {
|
|
134
|
+
function buildPerAppLimiter(rateLimitConfig, onLimit = () => {}) {
|
|
83
135
|
const { perAppMax, windowMs } = rateLimitConfig || {};
|
|
84
136
|
// Zero disables it, for single-tenant deployments where the per-IP limit
|
|
85
137
|
// is the only bound that means anything.
|
|
@@ -96,6 +148,10 @@ function buildPerAppLimiter(rateLimitConfig) {
|
|
|
96
148
|
// bypass this limiter exists to be immune to.
|
|
97
149
|
keyGenerator: (req) => String(req.query?.appId || req.body?.appId || '__unattributed__'),
|
|
98
150
|
validate: { keyGeneratorIpFallback: false },
|
|
151
|
+
handler: (req, res, next, options) => {
|
|
152
|
+
onLimit(req);
|
|
153
|
+
res.status(options.statusCode).json(options.message);
|
|
154
|
+
},
|
|
99
155
|
});
|
|
100
156
|
}
|
|
101
157
|
|
|
@@ -113,7 +169,7 @@ function withRequestId(req, res, next) {
|
|
|
113
169
|
* Build the correlation context for a log line.
|
|
114
170
|
*
|
|
115
171
|
* Logs the MASKED address, never the raw one. `logRequest` previously wrote
|
|
116
|
-
* the unmasked IP on every view, event, and error
|
|
172
|
+
* the unmasked IP on every view, event, and error, and on any normal
|
|
117
173
|
* deployment stdout is persisted to disk, so the raw addresses the privacy
|
|
118
174
|
* design goes to lengths to keep out of the database were being written beside
|
|
119
175
|
* it anyway.
|
|
@@ -128,7 +184,7 @@ function logContext(req) {
|
|
|
128
184
|
*
|
|
129
185
|
* The client gets a stable message plus the request id; the detail goes to the
|
|
130
186
|
* server log only. Previously the raw database error text was returned to the
|
|
131
|
-
* caller whenever NODE_ENV was not exactly "development"
|
|
187
|
+
* caller whenever NODE_ENV was not exactly "development", which was the
|
|
132
188
|
* default, and which the setup wizard wrote into .env.
|
|
133
189
|
*/
|
|
134
190
|
function handleRouteError(req, res, error, operation) {
|
|
@@ -140,10 +196,12 @@ function handleRouteError(req, res, error, operation) {
|
|
|
140
196
|
}
|
|
141
197
|
|
|
142
198
|
/**
|
|
143
|
-
* @param {{ config: object, dbManager: object, isReady: () => boolean
|
|
199
|
+
* @param {{ config: object, dbManager: object, isReady: () => boolean,
|
|
200
|
+
* geo?: { city: { lookup: (ip: string) => { region: string|null, city: string|null } }|null } }} deps
|
|
201
|
+
* `geo.city` is read on every view, so it can be opened after the router is built
|
|
144
202
|
* @returns {import('express').Router}
|
|
145
203
|
*/
|
|
146
|
-
function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
204
|
+
function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo = { city: null } }) {
|
|
147
205
|
const router = express.Router();
|
|
148
206
|
// Authentication and authorization are separate steps: `requireKey` proves
|
|
149
207
|
// the caller holds a key we issued, `requireScope` proves that key is
|
|
@@ -151,8 +209,38 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
151
209
|
const requireKey = requireReadApiKey(config.auth);
|
|
152
210
|
const requireScope = requireAppScope();
|
|
153
211
|
const requireAdmin = requireAdminApiKey(config.auth);
|
|
154
|
-
|
|
155
|
-
|
|
212
|
+
|
|
213
|
+
// Tracking requests that are not stored are counted for the tracking log,
|
|
214
|
+
// so an operator can see why a site's views are not arriving.
|
|
215
|
+
const rejections = createRejectionCounter({
|
|
216
|
+
write: (rows) => (dbManager.logs?.recordRejections ? dbManager.logs.recordRejections(rows) : Promise.resolve(true)),
|
|
217
|
+
});
|
|
218
|
+
const reject = (req, reason, { appId, detail } = {}) => {
|
|
219
|
+
const source = trackingSourceFor(req.path);
|
|
220
|
+
if (!source) return;
|
|
221
|
+
rejections.count({
|
|
222
|
+
source,
|
|
223
|
+
reason,
|
|
224
|
+
appId: appId ?? req.query?.appId ?? req.body?.appId,
|
|
225
|
+
detail,
|
|
226
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
227
|
+
});
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const requireOrigin = requireRegisteredOrigin(config.allowed, {
|
|
231
|
+
onReject: (req, appId) => reject(req, REJECTION_REASON.ORIGIN_NOT_ALLOWED, { appId }),
|
|
232
|
+
});
|
|
233
|
+
const limitPerApp = buildPerAppLimiter(config.server?.rateLimit,
|
|
234
|
+
(req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'app' }));
|
|
235
|
+
const trackingValidation = handleTrackingValidation(reject);
|
|
236
|
+
|
|
237
|
+
/** Views from these are counted in the tracking log and never stored. */
|
|
238
|
+
const refuseBot = (req) => {
|
|
239
|
+
const userAgent = req.get('user-agent') || '';
|
|
240
|
+
if (!UserAgentParser.isBot(userAgent)) return false;
|
|
241
|
+
reject(req, REJECTION_REASON.BOT, { detail: UserAgentParser.botName(userAgent) });
|
|
242
|
+
return true;
|
|
243
|
+
};
|
|
156
244
|
|
|
157
245
|
router.use(withRequestId);
|
|
158
246
|
|
|
@@ -169,6 +257,16 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
169
257
|
return res.json({ status: 'healthy', uptime: process.uptime() });
|
|
170
258
|
});
|
|
171
259
|
|
|
260
|
+
/**
|
|
261
|
+
* The tracker script. Loaded by other sites, so it opts out of the
|
|
262
|
+
* same-origin resource policy helmet applies to everything else.
|
|
263
|
+
*/
|
|
264
|
+
router.get('/tracker.js', (req, res) => {
|
|
265
|
+
res.set('Cache-Control', `public, max-age=${TRACKER_MAX_AGE_SECONDS}`);
|
|
266
|
+
res.set('Cross-Origin-Resource-Policy', 'cross-origin');
|
|
267
|
+
res.type('application/javascript').send(TRACKER_SOURCE);
|
|
268
|
+
});
|
|
269
|
+
|
|
172
270
|
/**
|
|
173
271
|
* Register a page view.
|
|
174
272
|
*/
|
|
@@ -176,18 +274,23 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
176
274
|
limitPerApp,
|
|
177
275
|
requireOrigin,
|
|
178
276
|
validateRegisterView(config.allowed),
|
|
179
|
-
|
|
277
|
+
trackingValidation,
|
|
180
278
|
async (req, res) => {
|
|
181
279
|
try {
|
|
182
280
|
const { appId, deviceSize, page, title, referrer, sessionId } = req.query;
|
|
183
|
-
|
|
281
|
+
if (refuseBot(req)) {
|
|
282
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false, duplicate: false });
|
|
283
|
+
}
|
|
184
284
|
|
|
285
|
+
const ip = normalizeIp(getClientIp(req));
|
|
185
286
|
if (!isValidIP(ip)) {
|
|
287
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
186
288
|
logger.warn(`Rejected request with unparseable client IP`, logContext(req));
|
|
187
289
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
188
290
|
}
|
|
189
291
|
|
|
190
292
|
const ipInfo = geoip.lookup(ip);
|
|
293
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
191
294
|
const userAgent = req.get('user-agent') || '';
|
|
192
295
|
const uaData = UserAgentParser.parse(userAgent);
|
|
193
296
|
|
|
@@ -198,7 +301,9 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
198
301
|
// page itself, not where the visitor came from, so reading it
|
|
199
302
|
// recorded every direct visit as a referral from the site's own
|
|
200
303
|
// domain. A server relaying views passes the real referrer here.
|
|
201
|
-
const
|
|
304
|
+
const hostname = hostnameOf(requestOrigin(req));
|
|
305
|
+
const referrerData = ReferrerParser.parse(referrer, hostname);
|
|
306
|
+
const utm = utmTags(req.query);
|
|
202
307
|
|
|
203
308
|
const result = await dbManager.registerEvent(appId, {
|
|
204
309
|
ip,
|
|
@@ -208,7 +313,13 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
208
313
|
pageTitle: title,
|
|
209
314
|
referrer: referrerData.referrer,
|
|
210
315
|
referrerDomain: referrerData.referrerDomain,
|
|
211
|
-
|
|
316
|
+
// A tagged link is a campaign, whichever site it was clicked on.
|
|
317
|
+
sourceType: utm.utmSource || utm.utmMedium ? SOURCE_TYPE.CAMPAIGN : referrerData.sourceType,
|
|
318
|
+
hostname,
|
|
319
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
320
|
+
...utm,
|
|
321
|
+
region: place.region,
|
|
322
|
+
city: place.city,
|
|
212
323
|
browser: uaData.browser,
|
|
213
324
|
browserVersion: uaData.browserVersion,
|
|
214
325
|
os: uaData.os,
|
|
@@ -224,15 +335,21 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
224
335
|
|
|
225
336
|
logger.audit('registerView', { ...logContext(req), appId, duplicate: result.duplicate });
|
|
226
337
|
|
|
338
|
+
// `id` lets the page report engagement for this view later.
|
|
227
339
|
if (result.duplicate) {
|
|
228
340
|
return res.status(HTTP_STATUS.OK).json({
|
|
229
341
|
message: 'View already registered recently',
|
|
230
342
|
duplicate: true,
|
|
343
|
+
recorded: true,
|
|
344
|
+
id: result.publicId,
|
|
231
345
|
});
|
|
232
346
|
}
|
|
233
347
|
|
|
234
|
-
return res.status(HTTP_STATUS.OK).json({
|
|
348
|
+
return res.status(HTTP_STATUS.OK).json({
|
|
349
|
+
message: 'Success!', duplicate: false, recorded: true, id: result.publicId,
|
|
350
|
+
});
|
|
235
351
|
} catch (error) {
|
|
352
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
236
353
|
return handleRouteError(req, res, error, 'register view');
|
|
237
354
|
}
|
|
238
355
|
}
|
|
@@ -245,17 +362,22 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
245
362
|
limitPerApp,
|
|
246
363
|
requireOrigin,
|
|
247
364
|
validateEvent(config.allowed),
|
|
248
|
-
|
|
365
|
+
trackingValidation,
|
|
249
366
|
async (req, res) => {
|
|
250
367
|
try {
|
|
251
368
|
const { appId, eventType, eventData, sessionId, page, title } = req.body;
|
|
252
|
-
|
|
369
|
+
if (refuseBot(req)) {
|
|
370
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false });
|
|
371
|
+
}
|
|
253
372
|
|
|
373
|
+
const ip = normalizeIp(getClientIp(req));
|
|
254
374
|
if (!isValidIP(ip)) {
|
|
375
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
255
376
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
256
377
|
}
|
|
257
378
|
|
|
258
379
|
const ipInfo = geoip.lookup(ip);
|
|
380
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
259
381
|
const userAgent = req.get('user-agent') || '';
|
|
260
382
|
const uaData = UserAgentParser.parse(userAgent);
|
|
261
383
|
|
|
@@ -273,6 +395,10 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
273
395
|
sessionId,
|
|
274
396
|
eventType,
|
|
275
397
|
eventData,
|
|
398
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
399
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
400
|
+
region: place.region,
|
|
401
|
+
city: place.city,
|
|
276
402
|
userAgent,
|
|
277
403
|
visitorSecret: config.privacy.visitorSecret,
|
|
278
404
|
// Custom events are never deduplicated.
|
|
@@ -284,14 +410,48 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
284
410
|
|
|
285
411
|
return res.status(HTTP_STATUS.OK).json({
|
|
286
412
|
message: 'Event tracked successfully',
|
|
413
|
+
recorded: true,
|
|
414
|
+
id: result.publicId,
|
|
415
|
+
// Deprecated: the internal row number. Kept for 3.x clients; use `id`.
|
|
287
416
|
insertId: result.insertId,
|
|
288
417
|
});
|
|
289
418
|
} catch (error) {
|
|
419
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
290
420
|
return handleRouteError(req, res, error, 'track event');
|
|
291
421
|
}
|
|
292
422
|
}
|
|
293
423
|
);
|
|
294
424
|
|
|
425
|
+
/**
|
|
426
|
+
* Engagement for a recorded view: how long its page was visible and how
|
|
427
|
+
* far it was scrolled, sent by the tracker when the page is hidden or left.
|
|
428
|
+
*/
|
|
429
|
+
router.post('/engage',
|
|
430
|
+
express.text({ type: () => true, limit: TRACKING.ENGAGE_BODY_BYTES }),
|
|
431
|
+
parseBeaconBody,
|
|
432
|
+
limitPerApp,
|
|
433
|
+
requireOrigin,
|
|
434
|
+
validateEngage(config.allowed),
|
|
435
|
+
trackingValidation,
|
|
436
|
+
async (req, res) => {
|
|
437
|
+
try {
|
|
438
|
+
if (refuseBot(req)) return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
439
|
+
|
|
440
|
+
const { appId, id, ms, scroll } = req.body;
|
|
441
|
+
const updated = await dbManager.addEngagement(appId, {
|
|
442
|
+
viewId: id,
|
|
443
|
+
engagedMs: Number(ms),
|
|
444
|
+
scrollDepth: Number(scroll),
|
|
445
|
+
});
|
|
446
|
+
if (!updated) reject(req, REJECTION_REASON.UNKNOWN_VIEW);
|
|
447
|
+
return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
448
|
+
} catch (error) {
|
|
449
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
450
|
+
return handleRouteError(req, res, error, 'record engagement');
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
);
|
|
454
|
+
|
|
295
455
|
// ---- Read API. Everything below requires a valid key. -------------------
|
|
296
456
|
|
|
297
457
|
router.use(noStore);
|
|
@@ -424,7 +584,7 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
424
584
|
* Provision a new app. Admin tier only.
|
|
425
585
|
*
|
|
426
586
|
* Creates the app's table and records it in the registry, then adds it to
|
|
427
|
-
* the live allowlist so it accepts traffic immediately
|
|
587
|
+
* the live allowlist so it accepts traffic immediately, with no restart. The
|
|
428
588
|
* appId becomes a table identifier, so it is validated against a strict
|
|
429
589
|
* pattern before it reaches any DDL.
|
|
430
590
|
*/
|
|
@@ -459,7 +619,26 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
459
619
|
}
|
|
460
620
|
);
|
|
461
621
|
|
|
622
|
+
/** For the host app's own limiter, which runs before this router, and for shutdown. */
|
|
623
|
+
router.countRejection = reject;
|
|
624
|
+
router.flushRejections = () => rejections.flush();
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* For the app that mounts this router, after its body parser: a body that
|
|
628
|
+
* is malformed JSON or over the size limit never reaches the router, so it
|
|
629
|
+
* is counted here, as an invalid request to the endpoint it was sent to.
|
|
630
|
+
* @type {import('express').ErrorRequestHandler}
|
|
631
|
+
*/
|
|
632
|
+
// eslint-disable-next-line no-unused-vars
|
|
633
|
+
router.bodyErrorHandler = (err, req, res, next) => {
|
|
634
|
+
const status = err.status || err.statusCode || HTTP_STATUS.INTERNAL_SERVER_ERROR;
|
|
635
|
+
logger.warn(`Request rejected: ${err.message}`, { requestId: req.id });
|
|
636
|
+
reject(req, REJECTION_REASON.INVALID_REQUEST, { detail: 'body' });
|
|
637
|
+
res.status(status === HTTP_STATUS.INTERNAL_SERVER_ERROR ? HTTP_STATUS.BAD_REQUEST : status)
|
|
638
|
+
.json({ message: 'Malformed or oversized request' });
|
|
639
|
+
};
|
|
640
|
+
|
|
462
641
|
return router;
|
|
463
642
|
}
|
|
464
643
|
|
|
465
|
-
module.exports = { createAnalyticsRouter, handleRouteError, logContext, withRequestId };
|
|
644
|
+
module.exports = { createAnalyticsRouter, handleRouteError, logContext, withRequestId, trackingSourceFor };
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* viewcounter tracker, https://viewcounter.harshankur.com
|
|
3
|
+
*
|
|
4
|
+
* <script defer src="https://your-server/tracker.js" data-app="blog"></script>
|
|
5
|
+
*
|
|
6
|
+
* Records a view of each page (including page changes in single-page apps),
|
|
7
|
+
* how long it was visible and how far it was scrolled, clicks on links to
|
|
8
|
+
* other sites and on downloads, and the campaign tags of the landing URL.
|
|
9
|
+
* The page before is sent as its origin and path only.
|
|
10
|
+
*
|
|
11
|
+
* It stores nothing on the visitor's device (no cookie, no localStorage, no
|
|
12
|
+
* sessionStorage), so it needs no consent banner, and it sends no identifier:
|
|
13
|
+
* the server tells repeat visits apart with a hash it rotates every day.
|
|
14
|
+
*
|
|
15
|
+
* Options, as attributes on the script tag:
|
|
16
|
+
* data-app="blog" required: the app ID the views belong to
|
|
17
|
+
* data-hosts="example.com,www.example.com"
|
|
18
|
+
* only track on these hostnames (keeps dev
|
|
19
|
+
* servers and previews out of the data)
|
|
20
|
+
* data-spa="false" do not treat history changes as page views
|
|
21
|
+
* data-outbound="false" do not record clicks on links to other sites
|
|
22
|
+
* data-downloads="false" do not record clicks on downloads
|
|
23
|
+
* data-respect-dnt="true" send nothing when Do Not Track is on
|
|
24
|
+
*
|
|
25
|
+
* Custom events: window.viewcounter.track('signup', { plan: 'pro' }).
|
|
26
|
+
*/
|
|
27
|
+
(() => {
|
|
28
|
+
const script = document.currentScript;
|
|
29
|
+
const app = script && script.dataset.app;
|
|
30
|
+
if (!app) return;
|
|
31
|
+
|
|
32
|
+
const option = (name, fallback) => (script.dataset[name] === undefined ? fallback : script.dataset[name] !== 'false');
|
|
33
|
+
const hosts = (script.dataset.hosts || '').split(',').map((host) => host.trim().toLowerCase()).filter(Boolean);
|
|
34
|
+
if (hosts.length && !hosts.includes(location.hostname.toLowerCase())) return;
|
|
35
|
+
if (location.protocol !== 'http:' && location.protocol !== 'https:') return;
|
|
36
|
+
// Automated browsers are not visitors.
|
|
37
|
+
if (navigator.webdriver) return;
|
|
38
|
+
if (option('respectDnt', false) && (navigator.doNotTrack === '1' || window.doNotTrack === '1')) return;
|
|
39
|
+
|
|
40
|
+
const base = script.src.replace(/[^/]*$/, '');
|
|
41
|
+
const UTM = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content'];
|
|
42
|
+
const DOWNLOAD = /\.(pdf|zip|gz|tgz|rar|7z|dmg|exe|msi|pkg|deb|rpm|apk|iso|csv|xlsx?|docx?|pptx?|odt|ods|epub|mp3|mp4|mov|avi|wav)$/i;
|
|
43
|
+
const MAX_ENGAGED_MS = 6 * 60 * 60 * 1000;
|
|
44
|
+
|
|
45
|
+
const deviceSize = () => (innerWidth < 768 ? 'small' : innerWidth < 1200 ? 'medium' : 'large');
|
|
46
|
+
/** How much of the page has been on screen, from 0 to 100. */
|
|
47
|
+
const seen = () => {
|
|
48
|
+
const height = Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0);
|
|
49
|
+
return height <= 0 ? 100 : Math.min(100, Math.round(((scrollY + innerHeight) / height) * 100));
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/** A URL's origin and path: its query and fragment can carry tokens or emails. */
|
|
53
|
+
const originAndPath = (url) => {
|
|
54
|
+
try {
|
|
55
|
+
const parsed = new URL(url);
|
|
56
|
+
return `${parsed.origin}${parsed.pathname}`;
|
|
57
|
+
} catch {
|
|
58
|
+
return '';
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
let view = null;
|
|
63
|
+
let referrer = document.referrer ? originAndPath(document.referrer) : '';
|
|
64
|
+
let path = location.pathname;
|
|
65
|
+
|
|
66
|
+
/** Tell the server how long the current page was visible and how far it was scrolled. */
|
|
67
|
+
function reportEngagement() {
|
|
68
|
+
if (!view || !view.id) return;
|
|
69
|
+
const ms = Math.min(MAX_ENGAGED_MS, Math.round(view.visibleMs + (view.visibleSince === null ? 0 : performance.now() - view.visibleSince)));
|
|
70
|
+
if (ms <= view.sentMs && view.scroll <= view.sentScroll) return;
|
|
71
|
+
view.sentMs = ms;
|
|
72
|
+
view.sentScroll = view.scroll;
|
|
73
|
+
const body = JSON.stringify({ appId: app, id: view.id, ms, scroll: view.scroll });
|
|
74
|
+
// text/plain needs no CORS preflight, so the beacon survives the page closing.
|
|
75
|
+
if (!(navigator.sendBeacon && navigator.sendBeacon(`${base}engage`, new Blob([body], { type: 'text/plain' })))) {
|
|
76
|
+
fetch(`${base}engage`, { method: 'POST', body, keepalive: true, credentials: 'omit', headers: { 'Content-Type': 'text/plain' } })
|
|
77
|
+
.catch(() => {});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function pageview() {
|
|
82
|
+
reportEngagement();
|
|
83
|
+
const params = new URLSearchParams({
|
|
84
|
+
appId: app,
|
|
85
|
+
deviceSize: deviceSize(),
|
|
86
|
+
page: location.pathname.slice(0, 500),
|
|
87
|
+
title: document.title.slice(0, 200),
|
|
88
|
+
referrer: referrer.slice(0, 500),
|
|
89
|
+
});
|
|
90
|
+
// Only the campaign tags: the rest of a query string can carry
|
|
91
|
+
// emails, tokens, or IDs, and never leaves the page.
|
|
92
|
+
const query = new URLSearchParams(location.search);
|
|
93
|
+
for (const key of UTM) {
|
|
94
|
+
const value = query.get(key);
|
|
95
|
+
if (value) params.set(key, value.slice(0, 100));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const current = {
|
|
99
|
+
id: null,
|
|
100
|
+
visibleMs: 0,
|
|
101
|
+
visibleSince: document.visibilityState === 'visible' ? performance.now() : null,
|
|
102
|
+
scroll: seen(),
|
|
103
|
+
sentMs: 0,
|
|
104
|
+
sentScroll: 0,
|
|
105
|
+
};
|
|
106
|
+
view = current;
|
|
107
|
+
fetch(`${base}registerView?${params}`, { keepalive: true, credentials: 'omit', referrerPolicy: 'no-referrer' })
|
|
108
|
+
.then((response) => (response.ok ? response.json() : null))
|
|
109
|
+
.then((result) => { if (result && result.id) current.id = result.id; })
|
|
110
|
+
.catch(() => {});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function track(eventType, eventData) {
|
|
114
|
+
if (typeof eventType !== 'string' || !eventType) return;
|
|
115
|
+
fetch(`${base}event`, {
|
|
116
|
+
method: 'POST',
|
|
117
|
+
keepalive: true,
|
|
118
|
+
credentials: 'omit',
|
|
119
|
+
referrerPolicy: 'no-referrer',
|
|
120
|
+
headers: { 'Content-Type': 'application/json' },
|
|
121
|
+
body: JSON.stringify({
|
|
122
|
+
appId: app,
|
|
123
|
+
eventType: eventType.slice(0, 50),
|
|
124
|
+
eventData: eventData && typeof eventData === 'object' ? eventData : undefined,
|
|
125
|
+
page: location.pathname.slice(0, 500),
|
|
126
|
+
title: document.title.slice(0, 200),
|
|
127
|
+
}),
|
|
128
|
+
}).catch(() => {});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// A page change in a single-page app is a new page view, with the page it
|
|
132
|
+
// came from as its referrer (which the server files as internal).
|
|
133
|
+
function navigated() {
|
|
134
|
+
if (location.pathname === path) return;
|
|
135
|
+
referrer = `${location.origin}${path}`;
|
|
136
|
+
path = location.pathname;
|
|
137
|
+
pageview();
|
|
138
|
+
}
|
|
139
|
+
if (option('spa', true)) {
|
|
140
|
+
for (const method of ['pushState', 'replaceState']) {
|
|
141
|
+
const original = history[method];
|
|
142
|
+
history[method] = function patched(...args) {
|
|
143
|
+
const result = original.apply(this, args);
|
|
144
|
+
navigated();
|
|
145
|
+
return result;
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
addEventListener('popstate', navigated);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let scrollQueued = false;
|
|
152
|
+
addEventListener('scroll', () => {
|
|
153
|
+
if (scrollQueued) return;
|
|
154
|
+
scrollQueued = true;
|
|
155
|
+
requestAnimationFrame(() => {
|
|
156
|
+
scrollQueued = false;
|
|
157
|
+
if (view) view.scroll = Math.max(view.scroll, seen());
|
|
158
|
+
});
|
|
159
|
+
}, { passive: true });
|
|
160
|
+
|
|
161
|
+
document.addEventListener('visibilitychange', () => {
|
|
162
|
+
if (!view) return;
|
|
163
|
+
if (document.visibilityState === 'hidden') {
|
|
164
|
+
if (view.visibleSince !== null) view.visibleMs += performance.now() - view.visibleSince;
|
|
165
|
+
view.visibleSince = null;
|
|
166
|
+
reportEngagement();
|
|
167
|
+
} else if (view.visibleSince === null) {
|
|
168
|
+
view.visibleSince = performance.now();
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
addEventListener('pagehide', reportEngagement);
|
|
172
|
+
|
|
173
|
+
// Links out and downloads. Only the other site's hostname, or the file's
|
|
174
|
+
// name, is recorded: never the whole URL, which can carry personal data.
|
|
175
|
+
document.addEventListener('click', (event) => {
|
|
176
|
+
const link = event.target && event.target.closest ? event.target.closest('a[href]') : null;
|
|
177
|
+
if (!link) return;
|
|
178
|
+
let url;
|
|
179
|
+
try { url = new URL(link.href, location.href); } catch { return; }
|
|
180
|
+
if (option('downloads', true) && DOWNLOAD.test(url.pathname)) {
|
|
181
|
+
let file = url.pathname.split('/').pop();
|
|
182
|
+
try { file = decodeURIComponent(file); } catch { /* keep it encoded */ }
|
|
183
|
+
track('download', { file: file.slice(0, 100) });
|
|
184
|
+
} else if (option('outbound', true) && /^https?:$/.test(url.protocol) && url.hostname !== location.hostname) {
|
|
185
|
+
track('outbound', { host: url.hostname });
|
|
186
|
+
}
|
|
187
|
+
}, { capture: true });
|
|
188
|
+
|
|
189
|
+
window.viewcounter = { track };
|
|
190
|
+
pageview();
|
|
191
|
+
})();
|
package/utils/appIdUtils.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* App ID validation.
|
|
3
3
|
*
|
|
4
4
|
* An app ID becomes a MySQL table name. Identifiers cannot be bound as query
|
|
5
|
-
* parameters, so they are interpolated
|
|
5
|
+
* parameters, so they are interpolated, which is safe only because the value
|
|
6
6
|
* is checked here first. App IDs used to come exclusively from local config;
|
|
7
7
|
* the admin API now accepts them over HTTP, so this is a live injection
|
|
8
8
|
* boundary, not a formatting preference.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Human-readable durations for configuration: "30m", "12h", "7d".
|
|
3
|
+
*
|
|
4
|
+
* Deliberately small: whole numbers of minutes, hours, or days, nothing else,
|
|
5
|
+
* so a value like "30 min" or "1.5h" is refused rather than read as something
|
|
6
|
+
* the operator did not mean. For a session timeout that difference matters.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const UNIT_MS = { m: 60 * 1000, h: 60 * 60 * 1000, d: 24 * 60 * 60 * 1000 };
|
|
10
|
+
const DURATION = /^(\d{1,6})([mhd])$/;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {string} raw e.g. "7d"
|
|
14
|
+
* @returns {number|null} milliseconds, or null when the value is not a duration
|
|
15
|
+
*/
|
|
16
|
+
function parseDuration(raw) {
|
|
17
|
+
const match = DURATION.exec(String(raw ?? '').trim().toLowerCase());
|
|
18
|
+
return match ? Number(match[1]) * UNIT_MS[match[2]] : null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The largest whole unit that expresses `ms` exactly: 604800000 -> "7d".
|
|
23
|
+
* @param {number} ms
|
|
24
|
+
* @returns {string}
|
|
25
|
+
*/
|
|
26
|
+
function formatDuration(ms) {
|
|
27
|
+
for (const unit of ['d', 'h', 'm']) {
|
|
28
|
+
if (ms % UNIT_MS[unit] === 0) return `${ms / UNIT_MS[unit]}${unit}`;
|
|
29
|
+
}
|
|
30
|
+
return `${ms}ms`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
module.exports = { parseDuration, formatDuration };
|
package/utils/errorUtils.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* identified by an enum member, its message text lives in exactly one table
|
|
7
7
|
* here, and callers branch on `err.code` rather than parsing message strings.
|
|
8
8
|
*
|
|
9
|
-
* `getError` builds and reports but deliberately does NOT throw
|
|
9
|
+
* `getError` builds and reports but deliberately does NOT throw: the `throw`
|
|
10
10
|
* stays visible at the call site and under the caller's control.
|
|
11
11
|
*/
|
|
12
12
|
|
|
@@ -45,6 +45,7 @@ const WarningType = {
|
|
|
45
45
|
VIEW_LOG_PRUNE_FAILED: 'VIEW_LOG_PRUNE_FAILED',
|
|
46
46
|
MIGRATION_TABLE_MISSING: 'MIGRATION_TABLE_MISSING',
|
|
47
47
|
VIEW_LOG_WRITE_FAILED: 'VIEW_LOG_WRITE_FAILED',
|
|
48
|
+
TRACKING_LOG_WRITE_FAILED: 'TRACKING_LOG_WRITE_FAILED',
|
|
48
49
|
ADMIN_LOG_WRITE_FAILED: 'ADMIN_LOG_WRITE_FAILED',
|
|
49
50
|
TRASH_PURGE_FAILED: 'TRASH_PURGE_FAILED',
|
|
50
51
|
};
|
|
@@ -113,6 +114,8 @@ const WARNING_MESSAGES = {
|
|
|
113
114
|
'Behind a TLS-terminating proxy, set TRUST_PROXY and have the proxy pass X-Forwarded-Proto and the original Host.',
|
|
114
115
|
[WarningType.MIGRATION_TABLE_MISSING]: (info) =>
|
|
115
116
|
`Table '${info?.table}' does not exist; skipping its schema migration.`,
|
|
117
|
+
[WarningType.TRACKING_LOG_WRITE_FAILED]: (info) =>
|
|
118
|
+
`Could not write counted tracking rejections: ${info?.cause}`,
|
|
116
119
|
[WarningType.VIEW_LOG_WRITE_FAILED]: (info) =>
|
|
117
120
|
`Could not write the view register log entry for '${info?.appId}': ${info?.cause}`,
|
|
118
121
|
[WarningType.ADMIN_LOG_WRITE_FAILED]: (info) =>
|