@harshankur/viewcounter 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +37 -11
- package/README.md +330 -138
- package/admin/css/admin.css +891 -198
- package/admin/index.html +13 -7
- package/admin/js/api.js +52 -6
- package/admin/js/appTabs.js +100 -0
- package/admin/js/charts.js +529 -189
- package/admin/js/constants.js +98 -9
- package/admin/js/dataTable.js +478 -0
- package/admin/js/format.js +58 -7
- package/admin/js/icons.js +168 -0
- package/admin/js/listbox.js +2 -1
- package/admin/js/logs.js +211 -60
- package/admin/js/main.js +201 -37
- package/admin/js/overview.js +905 -0
- package/admin/js/passwordPrompt.js +75 -0
- package/admin/js/table.js +12 -52
- package/admin/js/viewDialogs.js +30 -14
- package/admin/js/views.js +273 -207
- package/admin/locales/en.json +352 -63
- package/config/index.js +40 -5
- package/constants.js +126 -8
- package/db/AdminRepository.js +85 -159
- package/db/DatabaseManager.js +57 -7
- package/db/LogRepository.js +172 -35
- package/db/adminSchema.js +93 -4
- package/db/adminSessionStore.js +104 -0
- package/db/analysis.js +484 -0
- package/db/rejectionCounter.js +117 -0
- package/index.js +49 -26
- package/middleware/adminAuth.js +83 -43
- package/middleware/adminValidation.js +69 -3
- package/middleware/auth.js +2 -2
- package/middleware/security.js +26 -2
- package/middleware/validation.js +50 -2
- package/package.json +5 -2
- package/routes/admin.js +130 -22
- package/routes/analytics.js +236 -18
- package/tracker/tracker.js +240 -0
- package/utils/appIdUtils.js +1 -1
- package/utils/durationUtils.js +33 -0
- package/utils/errorUtils.js +4 -1
- package/utils/geoCity.js +87 -0
- package/utils/ipUtils.js +1 -1
- package/utils/privacyUtils.js +2 -2
- package/utils/referrerParser.js +23 -5
- package/utils/secretStore.js +1 -1
- package/utils/userAgentParser.js +52 -3
- package/utils/visitorContext.js +70 -0
- package/admin/js/insights.js +0 -192
package/routes/analytics.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
2
4
|
const express = require('express');
|
|
3
5
|
const rateLimit = require('express-rate-limit');
|
|
4
6
|
const geoip = require('geoip-country');
|
|
@@ -7,6 +9,9 @@ const {
|
|
|
7
9
|
EVENT_TYPE,
|
|
8
10
|
HTTP_STATUS,
|
|
9
11
|
QUERY_LIMITS,
|
|
12
|
+
REJECTION_REASON,
|
|
13
|
+
SOURCE_TYPE,
|
|
14
|
+
TRACKING,
|
|
10
15
|
TREND_PERIOD,
|
|
11
16
|
VIEW_LOG_SOURCE,
|
|
12
17
|
} = require('../constants');
|
|
@@ -16,19 +21,66 @@ const PrivacyUtils = require('../utils/privacyUtils');
|
|
|
16
21
|
const logger = require('../utils/logger');
|
|
17
22
|
const { getClientIp, isValidIP, normalizeIp } = require('../utils/ipUtils');
|
|
18
23
|
const { requireReadApiKey, requireAppScope, requireAdminApiKey, appsInScope } = require('../middleware/auth');
|
|
19
|
-
const { requireRegisteredOrigin, noStore } = require('../middleware/security');
|
|
24
|
+
const { requireRegisteredOrigin, requestOrigin, noStore } = require('../middleware/security');
|
|
25
|
+
const { createRejectionCounter } = require('../db/rejectionCounter');
|
|
26
|
+
const { hostnameOf, primaryLanguage, utmTags } = require('../utils/visitorContext');
|
|
20
27
|
const {
|
|
21
28
|
validateAppRegistration,
|
|
22
29
|
validateRegisterView,
|
|
23
30
|
validateEvent,
|
|
31
|
+
validateEngage,
|
|
24
32
|
validateStatsRequest,
|
|
25
33
|
validateTrendsRequest,
|
|
26
34
|
validateListRequest,
|
|
27
35
|
validateViewsRequest,
|
|
28
36
|
validateSessionRequest,
|
|
29
37
|
handleValidationErrors,
|
|
38
|
+
handleTrackingValidation,
|
|
30
39
|
} = require('../middleware/validation');
|
|
31
40
|
|
|
41
|
+
/**
|
|
42
|
+
* The tracker script sites include with <script src=".../tracker.js">. Read
|
|
43
|
+
* once; it is part of the package, not configuration.
|
|
44
|
+
*/
|
|
45
|
+
const TRACKER_SOURCE = fs.readFileSync(path.join(__dirname, '..', 'tracker', 'tracker.js'), 'utf8');
|
|
46
|
+
/** Cached briefly, so a fix reaches every site within the hour. */
|
|
47
|
+
const TRACKER_MAX_AGE_SECONDS = 60 * 60;
|
|
48
|
+
|
|
49
|
+
/** The tracking endpoints, by path, and the source the tracking log files them under. */
|
|
50
|
+
const TRACKING_PATHS = {
|
|
51
|
+
'/registerView': VIEW_LOG_SOURCE.REGISTER_VIEW,
|
|
52
|
+
'/event': VIEW_LOG_SOURCE.EVENT,
|
|
53
|
+
'/engage': VIEW_LOG_SOURCE.ENGAGE,
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {string} path a request path, relative to where the router is mounted
|
|
58
|
+
* @returns {string|null} the tracking source, or null for any other endpoint
|
|
59
|
+
*/
|
|
60
|
+
function trackingSourceFor(path) {
|
|
61
|
+
return Object.hasOwn(TRACKING_PATHS, path) ? TRACKING_PATHS[path] : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* sendBeacon posts text/plain, which needs no CORS preflight; a fetch may
|
|
66
|
+
* send JSON. Either way the handler sees an object, or an empty one for a
|
|
67
|
+
* body that is not JSON, which validation then refuses.
|
|
68
|
+
* @type {import('express').RequestHandler}
|
|
69
|
+
*/
|
|
70
|
+
function parseBeaconBody(req, res, next) {
|
|
71
|
+
if (typeof req.body === 'string') {
|
|
72
|
+
try {
|
|
73
|
+
const parsed = JSON.parse(req.body);
|
|
74
|
+
req.body = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
75
|
+
} catch {
|
|
76
|
+
req.body = {};
|
|
77
|
+
}
|
|
78
|
+
} else if (!req.body || typeof req.body !== 'object') {
|
|
79
|
+
req.body = {};
|
|
80
|
+
}
|
|
81
|
+
next();
|
|
82
|
+
}
|
|
83
|
+
|
|
32
84
|
/**
|
|
33
85
|
* Analytics routes.
|
|
34
86
|
*
|
|
@@ -73,13 +125,13 @@ function intQuery(req, name, fallback) {
|
|
|
73
125
|
* other tenant on the instance depends on, which is the failure mode that
|
|
74
126
|
* matters once the apps belong to different people.
|
|
75
127
|
*
|
|
76
|
-
* Keyed on appId only
|
|
128
|
+
* Keyed on appId only, never on IP, so it is unaffected by how the client's
|
|
77
129
|
* address is derived, and cannot be rotated away by a caller changing address.
|
|
78
130
|
*
|
|
79
131
|
* @param {{ perAppMax: number, windowMs: number }} rateLimitConfig
|
|
80
132
|
* @returns {import('express').RequestHandler}
|
|
81
133
|
*/
|
|
82
|
-
function buildPerAppLimiter(rateLimitConfig) {
|
|
134
|
+
function buildPerAppLimiter(rateLimitConfig, onLimit = () => {}) {
|
|
83
135
|
const { perAppMax, windowMs } = rateLimitConfig || {};
|
|
84
136
|
// Zero disables it, for single-tenant deployments where the per-IP limit
|
|
85
137
|
// is the only bound that means anything.
|
|
@@ -96,9 +148,47 @@ function buildPerAppLimiter(rateLimitConfig) {
|
|
|
96
148
|
// bypass this limiter exists to be immune to.
|
|
97
149
|
keyGenerator: (req) => String(req.query?.appId || req.body?.appId || '__unattributed__'),
|
|
98
150
|
validate: { keyGeneratorIpFallback: false },
|
|
151
|
+
handler: (req, res, next, options) => {
|
|
152
|
+
onLimit(req);
|
|
153
|
+
res.status(options.statusCode).json(options.message);
|
|
154
|
+
},
|
|
99
155
|
});
|
|
100
156
|
}
|
|
101
157
|
|
|
158
|
+
/** Whether a request is an engagement report, the one tracking request a page repeats. */
|
|
159
|
+
const isEngagement = (req) => trackingSourceFor(req.path) === VIEW_LOG_SOURCE.ENGAGE;
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The per-IP limiters: one budget for engagement reports, one for everything
|
|
163
|
+
* else, each of `max` requests per window.
|
|
164
|
+
*
|
|
165
|
+
* They are separate because a page being read reports its engagement every
|
|
166
|
+
* half minute. On one shared budget, a few dozen readers behind one address
|
|
167
|
+
* (an office, a campus) would use it up with those reports alone, and the page
|
|
168
|
+
* views of everyone at that address would be refused. Apart, reports can only
|
|
169
|
+
* ever crowd out other reports.
|
|
170
|
+
*
|
|
171
|
+
* @param {{ max: number, windowMs: number }} rateLimitConfig
|
|
172
|
+
* @param {(req: import('express').Request) => void} [onLimit] called for a refused request
|
|
173
|
+
* @returns {import('express').RequestHandler[]}
|
|
174
|
+
*/
|
|
175
|
+
function buildPerIpLimiters(rateLimitConfig, onLimit = () => {}) {
|
|
176
|
+
const { max, windowMs } = rateLimitConfig || {};
|
|
177
|
+
const limiter = (skip) => rateLimit({
|
|
178
|
+
windowMs,
|
|
179
|
+
limit: max,
|
|
180
|
+
message: { message: 'Too many requests, please try again later.' },
|
|
181
|
+
standardHeaders: true,
|
|
182
|
+
legacyHeaders: false,
|
|
183
|
+
skip,
|
|
184
|
+
handler: (req, res, next, options) => {
|
|
185
|
+
onLimit(req);
|
|
186
|
+
res.status(options.statusCode).json(options.message);
|
|
187
|
+
},
|
|
188
|
+
});
|
|
189
|
+
return [limiter(isEngagement), limiter((req) => !isEngagement(req))];
|
|
190
|
+
}
|
|
191
|
+
|
|
102
192
|
/**
|
|
103
193
|
* Attach a request id used for correlating a client-visible error with the
|
|
104
194
|
* server-side log line that has the real detail.
|
|
@@ -113,7 +203,7 @@ function withRequestId(req, res, next) {
|
|
|
113
203
|
* Build the correlation context for a log line.
|
|
114
204
|
*
|
|
115
205
|
* Logs the MASKED address, never the raw one. `logRequest` previously wrote
|
|
116
|
-
* the unmasked IP on every view, event, and error
|
|
206
|
+
* the unmasked IP on every view, event, and error, and on any normal
|
|
117
207
|
* deployment stdout is persisted to disk, so the raw addresses the privacy
|
|
118
208
|
* design goes to lengths to keep out of the database were being written beside
|
|
119
209
|
* it anyway.
|
|
@@ -128,7 +218,7 @@ function logContext(req) {
|
|
|
128
218
|
*
|
|
129
219
|
* The client gets a stable message plus the request id; the detail goes to the
|
|
130
220
|
* server log only. Previously the raw database error text was returned to the
|
|
131
|
-
* caller whenever NODE_ENV was not exactly "development"
|
|
221
|
+
* caller whenever NODE_ENV was not exactly "development", which was the
|
|
132
222
|
* default, and which the setup wizard wrote into .env.
|
|
133
223
|
*/
|
|
134
224
|
function handleRouteError(req, res, error, operation) {
|
|
@@ -140,10 +230,12 @@ function handleRouteError(req, res, error, operation) {
|
|
|
140
230
|
}
|
|
141
231
|
|
|
142
232
|
/**
|
|
143
|
-
* @param {{ config: object, dbManager: object, isReady: () => boolean
|
|
233
|
+
* @param {{ config: object, dbManager: object, isReady: () => boolean,
|
|
234
|
+
* geo?: { city: { lookup: (ip: string) => { region: string|null, city: string|null } }|null } }} deps
|
|
235
|
+
* `geo.city` is read on every view, so it can be opened after the router is built
|
|
144
236
|
* @returns {import('express').Router}
|
|
145
237
|
*/
|
|
146
|
-
function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
238
|
+
function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo = { city: null } }) {
|
|
147
239
|
const router = express.Router();
|
|
148
240
|
// Authentication and authorization are separate steps: `requireKey` proves
|
|
149
241
|
// the caller holds a key we issued, `requireScope` proves that key is
|
|
@@ -151,8 +243,43 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
151
243
|
const requireKey = requireReadApiKey(config.auth);
|
|
152
244
|
const requireScope = requireAppScope();
|
|
153
245
|
const requireAdmin = requireAdminApiKey(config.auth);
|
|
154
|
-
|
|
155
|
-
|
|
246
|
+
|
|
247
|
+
// Tracking requests that are not stored are counted for the tracking log,
|
|
248
|
+
// so an operator can see why a site's views are not arriving.
|
|
249
|
+
const rejections = createRejectionCounter({
|
|
250
|
+
write: (rows) => (dbManager.logs?.recordRejections ? dbManager.logs.recordRejections(rows) : Promise.resolve(true)),
|
|
251
|
+
});
|
|
252
|
+
const reject = (req, reason, { appId, detail } = {}) => {
|
|
253
|
+
const source = trackingSourceFor(req.path);
|
|
254
|
+
if (!source) return;
|
|
255
|
+
rejections.count({
|
|
256
|
+
source,
|
|
257
|
+
reason,
|
|
258
|
+
appId: appId ?? req.query?.appId ?? req.body?.appId,
|
|
259
|
+
detail,
|
|
260
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
261
|
+
});
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
const requireOrigin = requireRegisteredOrigin(config.allowed, {
|
|
265
|
+
onReject: (req, appId) => reject(req, REJECTION_REASON.ORIGIN_NOT_ALLOWED, { appId }),
|
|
266
|
+
});
|
|
267
|
+
const limitPerApp = buildPerAppLimiter(config.server?.rateLimit,
|
|
268
|
+
(req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'app' }));
|
|
269
|
+
// Engagement reports draw on a per-app budget of their own, for the reason
|
|
270
|
+
// buildPerIpLimiters gives: an app with many readers must not have its
|
|
271
|
+
// views refused because of the reports those readers' pages send.
|
|
272
|
+
const limitEngagePerApp = buildPerAppLimiter(config.server?.rateLimit,
|
|
273
|
+
(req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'app' }));
|
|
274
|
+
const trackingValidation = handleTrackingValidation(reject);
|
|
275
|
+
|
|
276
|
+
/** Views from these are counted in the tracking log and never stored. */
|
|
277
|
+
const refuseBot = (req) => {
|
|
278
|
+
const userAgent = req.get('user-agent') || '';
|
|
279
|
+
if (!UserAgentParser.isBot(userAgent)) return false;
|
|
280
|
+
reject(req, REJECTION_REASON.BOT, { detail: UserAgentParser.botName(userAgent) });
|
|
281
|
+
return true;
|
|
282
|
+
};
|
|
156
283
|
|
|
157
284
|
router.use(withRequestId);
|
|
158
285
|
|
|
@@ -169,6 +296,16 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
169
296
|
return res.json({ status: 'healthy', uptime: process.uptime() });
|
|
170
297
|
});
|
|
171
298
|
|
|
299
|
+
/**
|
|
300
|
+
* The tracker script. Loaded by other sites, so it opts out of the
|
|
301
|
+
* same-origin resource policy helmet applies to everything else.
|
|
302
|
+
*/
|
|
303
|
+
router.get('/tracker.js', (req, res) => {
|
|
304
|
+
res.set('Cache-Control', `public, max-age=${TRACKER_MAX_AGE_SECONDS}`);
|
|
305
|
+
res.set('Cross-Origin-Resource-Policy', 'cross-origin');
|
|
306
|
+
res.type('application/javascript').send(TRACKER_SOURCE);
|
|
307
|
+
});
|
|
308
|
+
|
|
172
309
|
/**
|
|
173
310
|
* Register a page view.
|
|
174
311
|
*/
|
|
@@ -176,18 +313,23 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
176
313
|
limitPerApp,
|
|
177
314
|
requireOrigin,
|
|
178
315
|
validateRegisterView(config.allowed),
|
|
179
|
-
|
|
316
|
+
trackingValidation,
|
|
180
317
|
async (req, res) => {
|
|
181
318
|
try {
|
|
182
319
|
const { appId, deviceSize, page, title, referrer, sessionId } = req.query;
|
|
183
|
-
|
|
320
|
+
if (refuseBot(req)) {
|
|
321
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false, duplicate: false });
|
|
322
|
+
}
|
|
184
323
|
|
|
324
|
+
const ip = normalizeIp(getClientIp(req));
|
|
185
325
|
if (!isValidIP(ip)) {
|
|
326
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
186
327
|
logger.warn(`Rejected request with unparseable client IP`, logContext(req));
|
|
187
328
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
188
329
|
}
|
|
189
330
|
|
|
190
331
|
const ipInfo = geoip.lookup(ip);
|
|
332
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
191
333
|
const userAgent = req.get('user-agent') || '';
|
|
192
334
|
const uaData = UserAgentParser.parse(userAgent);
|
|
193
335
|
|
|
@@ -198,7 +340,9 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
198
340
|
// page itself, not where the visitor came from, so reading it
|
|
199
341
|
// recorded every direct visit as a referral from the site's own
|
|
200
342
|
// domain. A server relaying views passes the real referrer here.
|
|
201
|
-
const
|
|
343
|
+
const hostname = hostnameOf(requestOrigin(req));
|
|
344
|
+
const referrerData = ReferrerParser.parse(referrer, hostname);
|
|
345
|
+
const utm = utmTags(req.query);
|
|
202
346
|
|
|
203
347
|
const result = await dbManager.registerEvent(appId, {
|
|
204
348
|
ip,
|
|
@@ -208,7 +352,13 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
208
352
|
pageTitle: title,
|
|
209
353
|
referrer: referrerData.referrer,
|
|
210
354
|
referrerDomain: referrerData.referrerDomain,
|
|
211
|
-
|
|
355
|
+
// A tagged link is a campaign, whichever site it was clicked on.
|
|
356
|
+
sourceType: utm.utmSource || utm.utmMedium ? SOURCE_TYPE.CAMPAIGN : referrerData.sourceType,
|
|
357
|
+
hostname,
|
|
358
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
359
|
+
...utm,
|
|
360
|
+
region: place.region,
|
|
361
|
+
city: place.city,
|
|
212
362
|
browser: uaData.browser,
|
|
213
363
|
browserVersion: uaData.browserVersion,
|
|
214
364
|
os: uaData.os,
|
|
@@ -224,15 +374,21 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
224
374
|
|
|
225
375
|
logger.audit('registerView', { ...logContext(req), appId, duplicate: result.duplicate });
|
|
226
376
|
|
|
377
|
+
// `id` lets the page report engagement for this view later.
|
|
227
378
|
if (result.duplicate) {
|
|
228
379
|
return res.status(HTTP_STATUS.OK).json({
|
|
229
380
|
message: 'View already registered recently',
|
|
230
381
|
duplicate: true,
|
|
382
|
+
recorded: true,
|
|
383
|
+
id: result.publicId,
|
|
231
384
|
});
|
|
232
385
|
}
|
|
233
386
|
|
|
234
|
-
return res.status(HTTP_STATUS.OK).json({
|
|
387
|
+
return res.status(HTTP_STATUS.OK).json({
|
|
388
|
+
message: 'Success!', duplicate: false, recorded: true, id: result.publicId,
|
|
389
|
+
});
|
|
235
390
|
} catch (error) {
|
|
391
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
236
392
|
return handleRouteError(req, res, error, 'register view');
|
|
237
393
|
}
|
|
238
394
|
}
|
|
@@ -245,17 +401,22 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
245
401
|
limitPerApp,
|
|
246
402
|
requireOrigin,
|
|
247
403
|
validateEvent(config.allowed),
|
|
248
|
-
|
|
404
|
+
trackingValidation,
|
|
249
405
|
async (req, res) => {
|
|
250
406
|
try {
|
|
251
407
|
const { appId, eventType, eventData, sessionId, page, title } = req.body;
|
|
252
|
-
|
|
408
|
+
if (refuseBot(req)) {
|
|
409
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false });
|
|
410
|
+
}
|
|
253
411
|
|
|
412
|
+
const ip = normalizeIp(getClientIp(req));
|
|
254
413
|
if (!isValidIP(ip)) {
|
|
414
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
255
415
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
256
416
|
}
|
|
257
417
|
|
|
258
418
|
const ipInfo = geoip.lookup(ip);
|
|
419
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
259
420
|
const userAgent = req.get('user-agent') || '';
|
|
260
421
|
const uaData = UserAgentParser.parse(userAgent);
|
|
261
422
|
|
|
@@ -273,6 +434,10 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
273
434
|
sessionId,
|
|
274
435
|
eventType,
|
|
275
436
|
eventData,
|
|
437
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
438
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
439
|
+
region: place.region,
|
|
440
|
+
city: place.city,
|
|
276
441
|
userAgent,
|
|
277
442
|
visitorSecret: config.privacy.visitorSecret,
|
|
278
443
|
// Custom events are never deduplicated.
|
|
@@ -284,14 +449,48 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
284
449
|
|
|
285
450
|
return res.status(HTTP_STATUS.OK).json({
|
|
286
451
|
message: 'Event tracked successfully',
|
|
452
|
+
recorded: true,
|
|
453
|
+
id: result.publicId,
|
|
454
|
+
// Deprecated: the internal row number. Kept for 3.x clients; use `id`.
|
|
287
455
|
insertId: result.insertId,
|
|
288
456
|
});
|
|
289
457
|
} catch (error) {
|
|
458
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
290
459
|
return handleRouteError(req, res, error, 'track event');
|
|
291
460
|
}
|
|
292
461
|
}
|
|
293
462
|
);
|
|
294
463
|
|
|
464
|
+
/**
|
|
465
|
+
* Engagement for a recorded view: how long its page was visible and how
|
|
466
|
+
* far it was scrolled, sent by the tracker when the page is hidden or left.
|
|
467
|
+
*/
|
|
468
|
+
router.post('/engage',
|
|
469
|
+
express.text({ type: () => true, limit: TRACKING.ENGAGE_BODY_BYTES }),
|
|
470
|
+
parseBeaconBody,
|
|
471
|
+
limitEngagePerApp,
|
|
472
|
+
requireOrigin,
|
|
473
|
+
validateEngage(config.allowed),
|
|
474
|
+
trackingValidation,
|
|
475
|
+
async (req, res) => {
|
|
476
|
+
try {
|
|
477
|
+
if (refuseBot(req)) return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
478
|
+
|
|
479
|
+
const { appId, id, ms, scroll } = req.body;
|
|
480
|
+
const updated = await dbManager.addEngagement(appId, {
|
|
481
|
+
viewId: id,
|
|
482
|
+
engagedMs: Number(ms),
|
|
483
|
+
scrollDepth: Number(scroll),
|
|
484
|
+
});
|
|
485
|
+
if (!updated) reject(req, REJECTION_REASON.UNKNOWN_VIEW);
|
|
486
|
+
return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
487
|
+
} catch (error) {
|
|
488
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
489
|
+
return handleRouteError(req, res, error, 'record engagement');
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
);
|
|
493
|
+
|
|
295
494
|
// ---- Read API. Everything below requires a valid key. -------------------
|
|
296
495
|
|
|
297
496
|
router.use(noStore);
|
|
@@ -424,7 +623,7 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
424
623
|
* Provision a new app. Admin tier only.
|
|
425
624
|
*
|
|
426
625
|
* Creates the app's table and records it in the registry, then adds it to
|
|
427
|
-
* the live allowlist so it accepts traffic immediately
|
|
626
|
+
* the live allowlist so it accepts traffic immediately, with no restart. The
|
|
428
627
|
* appId becomes a table identifier, so it is validated against a strict
|
|
429
628
|
* pattern before it reaches any DDL.
|
|
430
629
|
*/
|
|
@@ -459,7 +658,26 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
459
658
|
}
|
|
460
659
|
);
|
|
461
660
|
|
|
661
|
+
/** For the host app's own limiter, which runs before this router, and for shutdown. */
|
|
662
|
+
router.countRejection = reject;
|
|
663
|
+
router.flushRejections = () => rejections.flush();
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* For the app that mounts this router, after its body parser: a body that
|
|
667
|
+
* is malformed JSON or over the size limit never reaches the router, so it
|
|
668
|
+
* is counted here, as an invalid request to the endpoint it was sent to.
|
|
669
|
+
* @type {import('express').ErrorRequestHandler}
|
|
670
|
+
*/
|
|
671
|
+
// eslint-disable-next-line no-unused-vars
|
|
672
|
+
router.bodyErrorHandler = (err, req, res, next) => {
|
|
673
|
+
const status = err.status || err.statusCode || HTTP_STATUS.INTERNAL_SERVER_ERROR;
|
|
674
|
+
logger.warn(`Request rejected: ${err.message}`, { requestId: req.id });
|
|
675
|
+
reject(req, REJECTION_REASON.INVALID_REQUEST, { detail: 'body' });
|
|
676
|
+
res.status(status === HTTP_STATUS.INTERNAL_SERVER_ERROR ? HTTP_STATUS.BAD_REQUEST : status)
|
|
677
|
+
.json({ message: 'Malformed or oversized request' });
|
|
678
|
+
};
|
|
679
|
+
|
|
462
680
|
return router;
|
|
463
681
|
}
|
|
464
682
|
|
|
465
|
-
module.exports = { createAnalyticsRouter, handleRouteError, logContext, withRequestId };
|
|
683
|
+
module.exports = { createAnalyticsRouter, buildPerIpLimiters, handleRouteError, logContext, withRequestId, trackingSourceFor };
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* viewcounter tracker, https://viewcounter.harshankur.com
|
|
3
|
+
*
|
|
4
|
+
* <script defer src="https://your-server/tracker.js" data-app="blog"></script>
|
|
5
|
+
*
|
|
6
|
+
* Records a view of each page (including page changes in single-page apps),
|
|
7
|
+
* how long it was visible and how far it was scrolled (reported when the page
|
|
8
|
+
* is hidden or left, and every half minute while it is being read), clicks on links to
|
|
9
|
+
* other sites and on downloads, and the campaign tags of the landing URL.
|
|
10
|
+
* The page before is sent as its origin and path only.
|
|
11
|
+
*
|
|
12
|
+
* It stores nothing on the visitor's device (no cookie, no localStorage, no
|
|
13
|
+
* sessionStorage), so it needs no consent banner, and it sends no identifier:
|
|
14
|
+
* the server tells repeat visits apart with a hash it rotates every day.
|
|
15
|
+
*
|
|
16
|
+
* Options, as attributes on the script tag:
|
|
17
|
+
* data-app="blog" required: the app ID the views belong to
|
|
18
|
+
* data-hosts="example.com,www.example.com"
|
|
19
|
+
* only track on these hostnames (keeps dev
|
|
20
|
+
* servers and previews out of the data)
|
|
21
|
+
* data-spa="false" do not treat history changes as page views
|
|
22
|
+
* data-hash="#docs/,#spec/" count a URL fragment that starts with one of
|
|
23
|
+
* these as its own page (hash-routed pages)
|
|
24
|
+
* data-heartbeat="false" report time on page only when the page is
|
|
25
|
+
* hidden or left, not while it is being read
|
|
26
|
+
* data-outbound="false" do not record clicks on links to other sites
|
|
27
|
+
* data-downloads="false" do not record clicks on downloads
|
|
28
|
+
* data-respect-dnt="true" send nothing when Do Not Track is on
|
|
29
|
+
*
|
|
30
|
+
* Custom events: window.viewcounter.track('signup', { plan: 'pro' }).
|
|
31
|
+
*/
|
|
32
|
+
(() => {
|
|
33
|
+
const script = document.currentScript;
|
|
34
|
+
const app = script && script.dataset.app;
|
|
35
|
+
if (!app) return;
|
|
36
|
+
|
|
37
|
+
const option = (name, fallback) => (script.dataset[name] === undefined ? fallback : script.dataset[name] !== 'false');
|
|
38
|
+
const hosts = (script.dataset.hosts || '').split(',').map((host) => host.trim().toLowerCase()).filter(Boolean);
|
|
39
|
+
if (hosts.length && !hosts.includes(location.hostname.toLowerCase())) return;
|
|
40
|
+
const hashRoutes = (script.dataset.hash || '').split(',').map((prefix) => prefix.trim()).filter(Boolean);
|
|
41
|
+
if (location.protocol !== 'http:' && location.protocol !== 'https:') return;
|
|
42
|
+
// Automated browsers are not visitors.
|
|
43
|
+
if (navigator.webdriver) return;
|
|
44
|
+
if (option('respectDnt', false) && (navigator.doNotTrack === '1' || window.doNotTrack === '1')) return;
|
|
45
|
+
|
|
46
|
+
const base = script.src.replace(/[^/]*$/, '');
|
|
47
|
+
const UTM = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content'];
|
|
48
|
+
const DOWNLOAD = /\.(pdf|zip|gz|tgz|rar|7z|dmg|exe|msi|pkg|deb|rpm|apk|iso|csv|xlsx?|docx?|pptx?|odt|ods|epub|mp3|mp4|mov|avi|wav)$/i;
|
|
49
|
+
const MAX_ENGAGED_MS = 6 * 60 * 60 * 1000;
|
|
50
|
+
const ENGAGE_WINDOW_MS = 24 * 60 * 60 * 1000;
|
|
51
|
+
const HEARTBEAT_MS = 30 * 1000;
|
|
52
|
+
// A tab left open with nobody at it stops reporting after this long without input.
|
|
53
|
+
const IDLE_MS = 30 * 60 * 1000;
|
|
54
|
+
|
|
55
|
+
const deviceSize = () => (innerWidth < 768 ? 'small' : innerWidth < 1200 ? 'medium' : 'large');
|
|
56
|
+
/** How much of the page has been on screen, from 0 to 100. */
|
|
57
|
+
const seen = () => {
|
|
58
|
+
const height = Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0);
|
|
59
|
+
return height <= 0 ? 100 : Math.min(100, Math.round(((scrollY + innerHeight) / height) * 100));
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** A URL's origin and path: its query and fragment can carry tokens or emails. */
|
|
63
|
+
const originAndPath = (url) => {
|
|
64
|
+
try {
|
|
65
|
+
const parsed = new URL(url);
|
|
66
|
+
return `${parsed.origin}${parsed.pathname}`;
|
|
67
|
+
} catch {
|
|
68
|
+
return '';
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** The page on screen: its path, and its fragment when that is one of the site's own routes. */
|
|
73
|
+
const currentPage = () => location.pathname
|
|
74
|
+
+ (hashRoutes.some((prefix) => location.hash.startsWith(prefix)) ? location.hash : '');
|
|
75
|
+
|
|
76
|
+
let view = null;
|
|
77
|
+
let referrer = document.referrer ? originAndPath(document.referrer) : '';
|
|
78
|
+
let path = currentPage();
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Tell the server how long a page was visible and how far it was scrolled.
|
|
82
|
+
* `alive` sends it even when neither has grown since the last report: that
|
|
83
|
+
* is the heartbeat saying the visitor is still there.
|
|
84
|
+
*/
|
|
85
|
+
function reportEngagement(of = view, alive = false) {
|
|
86
|
+
if (!of || !of.id) return;
|
|
87
|
+
const ms = Math.min(MAX_ENGAGED_MS, Math.round(of.visibleMs + (of.visibleSince === null ? 0 : performance.now() - of.visibleSince)));
|
|
88
|
+
if (!alive && ms <= of.sentMs && of.scroll <= of.sentScroll) return;
|
|
89
|
+
of.sentMs = ms;
|
|
90
|
+
of.sentScroll = of.scroll;
|
|
91
|
+
const body = JSON.stringify({ appId: app, id: of.id, ms, scroll: of.scroll });
|
|
92
|
+
// text/plain needs no CORS preflight, so the beacon survives the page closing.
|
|
93
|
+
if (!(navigator.sendBeacon && navigator.sendBeacon(`${base}engage`, new Blob([body], { type: 'text/plain' })))) {
|
|
94
|
+
fetch(`${base}engage`, { method: 'POST', body, keepalive: true, credentials: 'omit', headers: { 'Content-Type': 'text/plain' } })
|
|
95
|
+
.catch(() => {});
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function pageview() {
|
|
100
|
+
// The page being left stops counting here, and reports what it has.
|
|
101
|
+
if (view) {
|
|
102
|
+
if (view.visibleSince !== null) view.visibleMs += performance.now() - view.visibleSince;
|
|
103
|
+
view.visibleSince = null;
|
|
104
|
+
reportEngagement();
|
|
105
|
+
}
|
|
106
|
+
const params = new URLSearchParams({
|
|
107
|
+
appId: app,
|
|
108
|
+
deviceSize: deviceSize(),
|
|
109
|
+
page: currentPage().slice(0, 500),
|
|
110
|
+
title: document.title.slice(0, 200),
|
|
111
|
+
referrer: referrer.slice(0, 500),
|
|
112
|
+
});
|
|
113
|
+
// Only the campaign tags: the rest of a query string can carry
|
|
114
|
+
// emails, tokens, or IDs, and never leaves the page.
|
|
115
|
+
const query = new URLSearchParams(location.search);
|
|
116
|
+
for (const key of UTM) {
|
|
117
|
+
const value = query.get(key);
|
|
118
|
+
if (value) params.set(key, value.slice(0, 100));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const current = {
|
|
122
|
+
id: null,
|
|
123
|
+
startedAt: Date.now(),
|
|
124
|
+
visibleMs: 0,
|
|
125
|
+
visibleSince: document.visibilityState === 'visible' ? performance.now() : null,
|
|
126
|
+
scroll: seen(),
|
|
127
|
+
sentMs: 0,
|
|
128
|
+
sentScroll: 0,
|
|
129
|
+
};
|
|
130
|
+
view = current;
|
|
131
|
+
fetch(`${base}registerView?${params}`, { keepalive: true, credentials: 'omit', referrerPolicy: 'no-referrer' })
|
|
132
|
+
.then((response) => (response.ok ? response.json() : null))
|
|
133
|
+
.then((result) => {
|
|
134
|
+
if (!result || !result.id) return;
|
|
135
|
+
current.id = result.id;
|
|
136
|
+
// Left before the server answered: its report could not go then, so it goes now.
|
|
137
|
+
if (view !== current) reportEngagement(current);
|
|
138
|
+
})
|
|
139
|
+
.catch(() => {});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function track(eventType, eventData) {
|
|
143
|
+
if (typeof eventType !== 'string' || !eventType) return;
|
|
144
|
+
fetch(`${base}event`, {
|
|
145
|
+
method: 'POST',
|
|
146
|
+
keepalive: true,
|
|
147
|
+
credentials: 'omit',
|
|
148
|
+
referrerPolicy: 'no-referrer',
|
|
149
|
+
headers: { 'Content-Type': 'application/json' },
|
|
150
|
+
body: JSON.stringify({
|
|
151
|
+
appId: app,
|
|
152
|
+
eventType: eventType.slice(0, 50),
|
|
153
|
+
eventData: eventData && typeof eventData === 'object' ? eventData : undefined,
|
|
154
|
+
page: currentPage().slice(0, 500),
|
|
155
|
+
title: document.title.slice(0, 200),
|
|
156
|
+
}),
|
|
157
|
+
}).catch(() => {});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// A page change in a single-page app is a new page view, with the page it
|
|
161
|
+
// came from as its referrer (which the server files as internal).
|
|
162
|
+
function navigated() {
|
|
163
|
+
if (currentPage() === path) return;
|
|
164
|
+
// Like every referrer, without the fragment: the server keeps an origin and a path.
|
|
165
|
+
referrer = `${location.origin}${path.split('#')[0]}`;
|
|
166
|
+
path = currentPage();
|
|
167
|
+
pageview();
|
|
168
|
+
}
|
|
169
|
+
if (option('spa', true)) {
|
|
170
|
+
for (const method of ['pushState', 'replaceState']) {
|
|
171
|
+
const original = history[method];
|
|
172
|
+
history[method] = function patched(...args) {
|
|
173
|
+
const result = original.apply(this, args);
|
|
174
|
+
navigated();
|
|
175
|
+
return result;
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
addEventListener('popstate', navigated);
|
|
179
|
+
}
|
|
180
|
+
// Asked for by name, so it does not wait on data-spa.
|
|
181
|
+
if (hashRoutes.length) addEventListener('hashchange', navigated);
|
|
182
|
+
|
|
183
|
+
let scrollQueued = false;
|
|
184
|
+
addEventListener('scroll', () => {
|
|
185
|
+
if (scrollQueued) return;
|
|
186
|
+
scrollQueued = true;
|
|
187
|
+
requestAnimationFrame(() => {
|
|
188
|
+
scrollQueued = false;
|
|
189
|
+
if (view) view.scroll = Math.max(view.scroll, seen());
|
|
190
|
+
});
|
|
191
|
+
}, { passive: true });
|
|
192
|
+
|
|
193
|
+
document.addEventListener('visibilitychange', () => {
|
|
194
|
+
if (!view) return;
|
|
195
|
+
if (document.visibilityState === 'hidden') {
|
|
196
|
+
if (view.visibleSince !== null) view.visibleMs += performance.now() - view.visibleSince;
|
|
197
|
+
view.visibleSince = null;
|
|
198
|
+
reportEngagement();
|
|
199
|
+
} else if (view.visibleSince === null) {
|
|
200
|
+
view.visibleSince = performance.now();
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
addEventListener('pagehide', () => reportEngagement());
|
|
204
|
+
|
|
205
|
+
// While the page is being read, report as it goes: the server then knows the
|
|
206
|
+
// visitor is still there, and a tab the browser kills without warning (common
|
|
207
|
+
// on phones) loses half a minute of its time at most, not all of it.
|
|
208
|
+
if (option('heartbeat', true)) {
|
|
209
|
+
let lastInput = performance.now();
|
|
210
|
+
const active = () => { lastInput = performance.now(); };
|
|
211
|
+
for (const type of ['pointerdown', 'pointermove', 'keydown', 'scroll', 'touchstart']) {
|
|
212
|
+
addEventListener(type, active, { passive: true, capture: true });
|
|
213
|
+
}
|
|
214
|
+
setInterval(() => {
|
|
215
|
+
if (!view || document.visibilityState !== 'visible' || performance.now() - lastInput > IDLE_MS) return;
|
|
216
|
+
// The server takes reports for a view for a day. A page open longer says no more.
|
|
217
|
+
if (Date.now() - view.startedAt > ENGAGE_WINDOW_MS) return;
|
|
218
|
+
reportEngagement(view, true);
|
|
219
|
+
}, HEARTBEAT_MS);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// Links out and downloads. Only the other site's hostname, or the file's
|
|
223
|
+
// name, is recorded: never the whole URL, which can carry personal data.
|
|
224
|
+
document.addEventListener('click', (event) => {
|
|
225
|
+
const link = event.target && event.target.closest ? event.target.closest('a[href]') : null;
|
|
226
|
+
if (!link) return;
|
|
227
|
+
let url;
|
|
228
|
+
try { url = new URL(link.href, location.href); } catch { return; }
|
|
229
|
+
if (option('downloads', true) && DOWNLOAD.test(url.pathname)) {
|
|
230
|
+
let file = url.pathname.split('/').pop();
|
|
231
|
+
try { file = decodeURIComponent(file); } catch { /* keep it encoded */ }
|
|
232
|
+
track('download', { file: file.slice(0, 100) });
|
|
233
|
+
} else if (option('outbound', true) && /^https?:$/.test(url.protocol) && url.hostname !== location.hostname) {
|
|
234
|
+
track('outbound', { host: url.hostname });
|
|
235
|
+
}
|
|
236
|
+
}, { capture: true });
|
|
237
|
+
|
|
238
|
+
window.viewcounter = { track };
|
|
239
|
+
pageview();
|
|
240
|
+
})();
|