@harshankur/viewcounter 3.0.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +50 -6
- package/README.md +444 -104
- package/admin/apple-touch-icon.png +0 -0
- package/admin/assets/world-map.json +1 -0
- package/admin/css/admin.css +2568 -0
- package/admin/favicon.ico +0 -0
- package/admin/favicon.svg +9 -0
- package/admin/icon-192.png +0 -0
- package/admin/icon-512.png +0 -0
- package/admin/index.html +95 -0
- package/admin/js/api.js +146 -0
- package/admin/js/appTabs.js +100 -0
- package/admin/js/charts.js +842 -0
- package/admin/js/clamp.js +41 -0
- package/admin/js/constants.js +239 -0
- package/admin/js/dataTable.js +478 -0
- package/admin/js/dom.js +83 -0
- package/admin/js/format.js +130 -0
- package/admin/js/i18n.js +80 -0
- package/admin/js/icons.js +168 -0
- package/admin/js/listbox.js +145 -0
- package/admin/js/logs.js +318 -0
- package/admin/js/main.js +399 -0
- package/admin/js/modal.js +171 -0
- package/admin/js/overview.js +905 -0
- package/admin/js/passwordPrompt.js +75 -0
- package/admin/js/table.js +94 -0
- package/admin/js/theme.js +72 -0
- package/admin/js/toast.js +47 -0
- package/admin/js/viewDialogs.js +224 -0
- package/admin/js/views.js +751 -0
- package/admin/locales/en.json +683 -0
- package/admin/site.webmanifest +20 -0
- package/config/index.js +122 -4
- package/constants.js +334 -3
- package/db/AdminRepository.js +488 -0
- package/db/DatabaseManager.js +148 -26
- package/db/LogRepository.js +354 -0
- package/db/adminSchema.js +329 -0
- package/db/adminSessionStore.js +104 -0
- package/db/analysis.js +479 -0
- package/db/rejectionCounter.js +117 -0
- package/db/retention.js +97 -0
- package/index.js +91 -24
- package/middleware/adminAuth.js +244 -0
- package/middleware/adminValidation.js +319 -0
- package/middleware/auth.js +2 -2
- package/middleware/security.js +26 -2
- package/middleware/validation.js +50 -2
- package/package.json +20 -10
- package/routes/admin.js +546 -0
- package/routes/analytics.js +207 -19
- package/tracker/tracker.js +191 -0
- package/utils/appIdUtils.js +1 -1
- package/utils/cookieUtils.js +47 -0
- package/utils/durationUtils.js +33 -0
- package/utils/errorUtils.js +39 -1
- package/utils/geoCity.js +87 -0
- package/utils/ipUtils.js +1 -1
- package/utils/privacyUtils.js +2 -2
- package/utils/referrerParser.js +23 -5
- package/utils/secretStore.js +1 -1
- package/utils/userAgentParser.js +52 -3
- package/utils/visitorContext.js +70 -0
package/routes/analytics.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
2
4
|
const express = require('express');
|
|
3
5
|
const rateLimit = require('express-rate-limit');
|
|
4
6
|
const geoip = require('geoip-country');
|
|
@@ -7,7 +9,11 @@ const {
|
|
|
7
9
|
EVENT_TYPE,
|
|
8
10
|
HTTP_STATUS,
|
|
9
11
|
QUERY_LIMITS,
|
|
12
|
+
REJECTION_REASON,
|
|
13
|
+
SOURCE_TYPE,
|
|
14
|
+
TRACKING,
|
|
10
15
|
TREND_PERIOD,
|
|
16
|
+
VIEW_LOG_SOURCE,
|
|
11
17
|
} = require('../constants');
|
|
12
18
|
const UserAgentParser = require('../utils/userAgentParser');
|
|
13
19
|
const ReferrerParser = require('../utils/referrerParser');
|
|
@@ -15,19 +21,66 @@ const PrivacyUtils = require('../utils/privacyUtils');
|
|
|
15
21
|
const logger = require('../utils/logger');
|
|
16
22
|
const { getClientIp, isValidIP, normalizeIp } = require('../utils/ipUtils');
|
|
17
23
|
const { requireReadApiKey, requireAppScope, requireAdminApiKey, appsInScope } = require('../middleware/auth');
|
|
18
|
-
const { requireRegisteredOrigin, noStore } = require('../middleware/security');
|
|
24
|
+
const { requireRegisteredOrigin, requestOrigin, noStore } = require('../middleware/security');
|
|
25
|
+
const { createRejectionCounter } = require('../db/rejectionCounter');
|
|
26
|
+
const { hostnameOf, primaryLanguage, utmTags } = require('../utils/visitorContext');
|
|
19
27
|
const {
|
|
20
28
|
validateAppRegistration,
|
|
21
29
|
validateRegisterView,
|
|
22
30
|
validateEvent,
|
|
31
|
+
validateEngage,
|
|
23
32
|
validateStatsRequest,
|
|
24
33
|
validateTrendsRequest,
|
|
25
34
|
validateListRequest,
|
|
26
35
|
validateViewsRequest,
|
|
27
36
|
validateSessionRequest,
|
|
28
37
|
handleValidationErrors,
|
|
38
|
+
handleTrackingValidation,
|
|
29
39
|
} = require('../middleware/validation');
|
|
30
40
|
|
|
41
|
+
/**
|
|
42
|
+
* The tracker script sites include with <script src=".../tracker.js">. Read
|
|
43
|
+
* once; it is part of the package, not configuration.
|
|
44
|
+
*/
|
|
45
|
+
const TRACKER_SOURCE = fs.readFileSync(path.join(__dirname, '..', 'tracker', 'tracker.js'), 'utf8');
|
|
46
|
+
/** Cached briefly, so a fix reaches every site within the hour. */
|
|
47
|
+
const TRACKER_MAX_AGE_SECONDS = 60 * 60;
|
|
48
|
+
|
|
49
|
+
/** The tracking endpoints, by path, and the source the tracking log files them under. */
|
|
50
|
+
const TRACKING_PATHS = {
|
|
51
|
+
'/registerView': VIEW_LOG_SOURCE.REGISTER_VIEW,
|
|
52
|
+
'/event': VIEW_LOG_SOURCE.EVENT,
|
|
53
|
+
'/engage': VIEW_LOG_SOURCE.ENGAGE,
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {string} path a request path, relative to where the router is mounted
|
|
58
|
+
* @returns {string|null} the tracking source, or null for any other endpoint
|
|
59
|
+
*/
|
|
60
|
+
function trackingSourceFor(path) {
|
|
61
|
+
return Object.hasOwn(TRACKING_PATHS, path) ? TRACKING_PATHS[path] : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* sendBeacon posts text/plain, which needs no CORS preflight; a fetch may
|
|
66
|
+
* send JSON. Either way the handler sees an object, or an empty one for a
|
|
67
|
+
* body that is not JSON, which validation then refuses.
|
|
68
|
+
* @type {import('express').RequestHandler}
|
|
69
|
+
*/
|
|
70
|
+
function parseBeaconBody(req, res, next) {
|
|
71
|
+
if (typeof req.body === 'string') {
|
|
72
|
+
try {
|
|
73
|
+
const parsed = JSON.parse(req.body);
|
|
74
|
+
req.body = parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
75
|
+
} catch {
|
|
76
|
+
req.body = {};
|
|
77
|
+
}
|
|
78
|
+
} else if (!req.body || typeof req.body !== 'object') {
|
|
79
|
+
req.body = {};
|
|
80
|
+
}
|
|
81
|
+
next();
|
|
82
|
+
}
|
|
83
|
+
|
|
31
84
|
/**
|
|
32
85
|
* Analytics routes.
|
|
33
86
|
*
|
|
@@ -72,13 +125,13 @@ function intQuery(req, name, fallback) {
|
|
|
72
125
|
* other tenant on the instance depends on, which is the failure mode that
|
|
73
126
|
* matters once the apps belong to different people.
|
|
74
127
|
*
|
|
75
|
-
* Keyed on appId only
|
|
128
|
+
* Keyed on appId only, never on IP, so it is unaffected by how the client's
|
|
76
129
|
* address is derived, and cannot be rotated away by a caller changing address.
|
|
77
130
|
*
|
|
78
131
|
* @param {{ perAppMax: number, windowMs: number }} rateLimitConfig
|
|
79
132
|
* @returns {import('express').RequestHandler}
|
|
80
133
|
*/
|
|
81
|
-
function buildPerAppLimiter(rateLimitConfig) {
|
|
134
|
+
function buildPerAppLimiter(rateLimitConfig, onLimit = () => {}) {
|
|
82
135
|
const { perAppMax, windowMs } = rateLimitConfig || {};
|
|
83
136
|
// Zero disables it, for single-tenant deployments where the per-IP limit
|
|
84
137
|
// is the only bound that means anything.
|
|
@@ -95,6 +148,10 @@ function buildPerAppLimiter(rateLimitConfig) {
|
|
|
95
148
|
// bypass this limiter exists to be immune to.
|
|
96
149
|
keyGenerator: (req) => String(req.query?.appId || req.body?.appId || '__unattributed__'),
|
|
97
150
|
validate: { keyGeneratorIpFallback: false },
|
|
151
|
+
handler: (req, res, next, options) => {
|
|
152
|
+
onLimit(req);
|
|
153
|
+
res.status(options.statusCode).json(options.message);
|
|
154
|
+
},
|
|
98
155
|
});
|
|
99
156
|
}
|
|
100
157
|
|
|
@@ -112,7 +169,7 @@ function withRequestId(req, res, next) {
|
|
|
112
169
|
* Build the correlation context for a log line.
|
|
113
170
|
*
|
|
114
171
|
* Logs the MASKED address, never the raw one. `logRequest` previously wrote
|
|
115
|
-
* the unmasked IP on every view, event, and error
|
|
172
|
+
* the unmasked IP on every view, event, and error, and on any normal
|
|
116
173
|
* deployment stdout is persisted to disk, so the raw addresses the privacy
|
|
117
174
|
* design goes to lengths to keep out of the database were being written beside
|
|
118
175
|
* it anyway.
|
|
@@ -127,7 +184,7 @@ function logContext(req) {
|
|
|
127
184
|
*
|
|
128
185
|
* The client gets a stable message plus the request id; the detail goes to the
|
|
129
186
|
* server log only. Previously the raw database error text was returned to the
|
|
130
|
-
* caller whenever NODE_ENV was not exactly "development"
|
|
187
|
+
* caller whenever NODE_ENV was not exactly "development", which was the
|
|
131
188
|
* default, and which the setup wizard wrote into .env.
|
|
132
189
|
*/
|
|
133
190
|
function handleRouteError(req, res, error, operation) {
|
|
@@ -139,10 +196,12 @@ function handleRouteError(req, res, error, operation) {
|
|
|
139
196
|
}
|
|
140
197
|
|
|
141
198
|
/**
|
|
142
|
-
* @param {{ config: object, dbManager: object, isReady: () => boolean
|
|
199
|
+
* @param {{ config: object, dbManager: object, isReady: () => boolean,
|
|
200
|
+
* geo?: { city: { lookup: (ip: string) => { region: string|null, city: string|null } }|null } }} deps
|
|
201
|
+
* `geo.city` is read on every view, so it can be opened after the router is built
|
|
143
202
|
* @returns {import('express').Router}
|
|
144
203
|
*/
|
|
145
|
-
function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
204
|
+
function createAnalyticsRouter({ config, dbManager, isReady = () => true, geo = { city: null } }) {
|
|
146
205
|
const router = express.Router();
|
|
147
206
|
// Authentication and authorization are separate steps: `requireKey` proves
|
|
148
207
|
// the caller holds a key we issued, `requireScope` proves that key is
|
|
@@ -150,8 +209,38 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
150
209
|
const requireKey = requireReadApiKey(config.auth);
|
|
151
210
|
const requireScope = requireAppScope();
|
|
152
211
|
const requireAdmin = requireAdminApiKey(config.auth);
|
|
153
|
-
|
|
154
|
-
|
|
212
|
+
|
|
213
|
+
// Tracking requests that are not stored are counted for the tracking log,
|
|
214
|
+
// so an operator can see why a site's views are not arriving.
|
|
215
|
+
const rejections = createRejectionCounter({
|
|
216
|
+
write: (rows) => (dbManager.logs?.recordRejections ? dbManager.logs.recordRejections(rows) : Promise.resolve(true)),
|
|
217
|
+
});
|
|
218
|
+
const reject = (req, reason, { appId, detail } = {}) => {
|
|
219
|
+
const source = trackingSourceFor(req.path);
|
|
220
|
+
if (!source) return;
|
|
221
|
+
rejections.count({
|
|
222
|
+
source,
|
|
223
|
+
reason,
|
|
224
|
+
appId: appId ?? req.query?.appId ?? req.body?.appId,
|
|
225
|
+
detail,
|
|
226
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
227
|
+
});
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
const requireOrigin = requireRegisteredOrigin(config.allowed, {
|
|
231
|
+
onReject: (req, appId) => reject(req, REJECTION_REASON.ORIGIN_NOT_ALLOWED, { appId }),
|
|
232
|
+
});
|
|
233
|
+
const limitPerApp = buildPerAppLimiter(config.server?.rateLimit,
|
|
234
|
+
(req) => reject(req, REJECTION_REASON.RATE_LIMITED, { detail: 'app' }));
|
|
235
|
+
const trackingValidation = handleTrackingValidation(reject);
|
|
236
|
+
|
|
237
|
+
/** Views from these are counted in the tracking log and never stored. */
|
|
238
|
+
const refuseBot = (req) => {
|
|
239
|
+
const userAgent = req.get('user-agent') || '';
|
|
240
|
+
if (!UserAgentParser.isBot(userAgent)) return false;
|
|
241
|
+
reject(req, REJECTION_REASON.BOT, { detail: UserAgentParser.botName(userAgent) });
|
|
242
|
+
return true;
|
|
243
|
+
};
|
|
155
244
|
|
|
156
245
|
router.use(withRequestId);
|
|
157
246
|
|
|
@@ -168,6 +257,16 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
168
257
|
return res.json({ status: 'healthy', uptime: process.uptime() });
|
|
169
258
|
});
|
|
170
259
|
|
|
260
|
+
/**
|
|
261
|
+
* The tracker script. Loaded by other sites, so it opts out of the
|
|
262
|
+
* same-origin resource policy helmet applies to everything else.
|
|
263
|
+
*/
|
|
264
|
+
router.get('/tracker.js', (req, res) => {
|
|
265
|
+
res.set('Cache-Control', `public, max-age=${TRACKER_MAX_AGE_SECONDS}`);
|
|
266
|
+
res.set('Cross-Origin-Resource-Policy', 'cross-origin');
|
|
267
|
+
res.type('application/javascript').send(TRACKER_SOURCE);
|
|
268
|
+
});
|
|
269
|
+
|
|
171
270
|
/**
|
|
172
271
|
* Register a page view.
|
|
173
272
|
*/
|
|
@@ -175,23 +274,36 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
175
274
|
limitPerApp,
|
|
176
275
|
requireOrigin,
|
|
177
276
|
validateRegisterView(config.allowed),
|
|
178
|
-
|
|
277
|
+
trackingValidation,
|
|
179
278
|
async (req, res) => {
|
|
180
279
|
try {
|
|
181
280
|
const { appId, deviceSize, page, title, referrer, sessionId } = req.query;
|
|
182
|
-
|
|
281
|
+
if (refuseBot(req)) {
|
|
282
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false, duplicate: false });
|
|
283
|
+
}
|
|
183
284
|
|
|
285
|
+
const ip = normalizeIp(getClientIp(req));
|
|
184
286
|
if (!isValidIP(ip)) {
|
|
287
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
185
288
|
logger.warn(`Rejected request with unparseable client IP`, logContext(req));
|
|
186
289
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
187
290
|
}
|
|
188
291
|
|
|
189
292
|
const ipInfo = geoip.lookup(ip);
|
|
293
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
190
294
|
const userAgent = req.get('user-agent') || '';
|
|
191
295
|
const uaData = UserAgentParser.parse(userAgent);
|
|
192
296
|
|
|
193
|
-
|
|
194
|
-
|
|
297
|
+
// The `referrer` parameter is the only source of the visitor's
|
|
298
|
+
// referrer; absent or empty means a direct visit. The Referer
|
|
299
|
+
// header is deliberately not a fallback: on every browser
|
|
300
|
+
// integration (a fetch or an <img> beacon) it names the tracked
|
|
301
|
+
// page itself, not where the visitor came from, so reading it
|
|
302
|
+
// recorded every direct visit as a referral from the site's own
|
|
303
|
+
// domain. A server relaying views passes the real referrer here.
|
|
304
|
+
const hostname = hostnameOf(requestOrigin(req));
|
|
305
|
+
const referrerData = ReferrerParser.parse(referrer, hostname);
|
|
306
|
+
const utm = utmTags(req.query);
|
|
195
307
|
|
|
196
308
|
const result = await dbManager.registerEvent(appId, {
|
|
197
309
|
ip,
|
|
@@ -201,7 +313,13 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
201
313
|
pageTitle: title,
|
|
202
314
|
referrer: referrerData.referrer,
|
|
203
315
|
referrerDomain: referrerData.referrerDomain,
|
|
204
|
-
|
|
316
|
+
// A tagged link is a campaign, whichever site it was clicked on.
|
|
317
|
+
sourceType: utm.utmSource || utm.utmMedium ? SOURCE_TYPE.CAMPAIGN : referrerData.sourceType,
|
|
318
|
+
hostname,
|
|
319
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
320
|
+
...utm,
|
|
321
|
+
region: place.region,
|
|
322
|
+
city: place.city,
|
|
205
323
|
browser: uaData.browser,
|
|
206
324
|
browserVersion: uaData.browserVersion,
|
|
207
325
|
os: uaData.os,
|
|
@@ -212,19 +330,26 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
212
330
|
userAgent,
|
|
213
331
|
visitorSecret: config.privacy.visitorSecret,
|
|
214
332
|
uniqueWindowHours: config.server.uniqueVisitorWindowHours,
|
|
333
|
+
source: VIEW_LOG_SOURCE.REGISTER_VIEW,
|
|
215
334
|
});
|
|
216
335
|
|
|
217
336
|
logger.audit('registerView', { ...logContext(req), appId, duplicate: result.duplicate });
|
|
218
337
|
|
|
338
|
+
// `id` lets the page report engagement for this view later.
|
|
219
339
|
if (result.duplicate) {
|
|
220
340
|
return res.status(HTTP_STATUS.OK).json({
|
|
221
341
|
message: 'View already registered recently',
|
|
222
342
|
duplicate: true,
|
|
343
|
+
recorded: true,
|
|
344
|
+
id: result.publicId,
|
|
223
345
|
});
|
|
224
346
|
}
|
|
225
347
|
|
|
226
|
-
return res.status(HTTP_STATUS.OK).json({
|
|
348
|
+
return res.status(HTTP_STATUS.OK).json({
|
|
349
|
+
message: 'Success!', duplicate: false, recorded: true, id: result.publicId,
|
|
350
|
+
});
|
|
227
351
|
} catch (error) {
|
|
352
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
228
353
|
return handleRouteError(req, res, error, 'register view');
|
|
229
354
|
}
|
|
230
355
|
}
|
|
@@ -237,17 +362,22 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
237
362
|
limitPerApp,
|
|
238
363
|
requireOrigin,
|
|
239
364
|
validateEvent(config.allowed),
|
|
240
|
-
|
|
365
|
+
trackingValidation,
|
|
241
366
|
async (req, res) => {
|
|
242
367
|
try {
|
|
243
368
|
const { appId, eventType, eventData, sessionId, page, title } = req.body;
|
|
244
|
-
|
|
369
|
+
if (refuseBot(req)) {
|
|
370
|
+
return res.json({ message: 'Automated clients are not counted', recorded: false });
|
|
371
|
+
}
|
|
245
372
|
|
|
373
|
+
const ip = normalizeIp(getClientIp(req));
|
|
246
374
|
if (!isValidIP(ip)) {
|
|
375
|
+
reject(req, REJECTION_REASON.INVALID_IP);
|
|
247
376
|
return res.status(HTTP_STATUS.BAD_REQUEST).json({ message: 'Invalid IP address format' });
|
|
248
377
|
}
|
|
249
378
|
|
|
250
379
|
const ipInfo = geoip.lookup(ip);
|
|
380
|
+
const place = geo.city ? geo.city.lookup(ip) : { region: null, city: null };
|
|
251
381
|
const userAgent = req.get('user-agent') || '';
|
|
252
382
|
const uaData = UserAgentParser.parse(userAgent);
|
|
253
383
|
|
|
@@ -265,24 +395,63 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
265
395
|
sessionId,
|
|
266
396
|
eventType,
|
|
267
397
|
eventData,
|
|
398
|
+
hostname: hostnameOf(requestOrigin(req)),
|
|
399
|
+
language: primaryLanguage(req.get('accept-language')),
|
|
400
|
+
region: place.region,
|
|
401
|
+
city: place.city,
|
|
268
402
|
userAgent,
|
|
269
403
|
visitorSecret: config.privacy.visitorSecret,
|
|
270
404
|
// Custom events are never deduplicated.
|
|
271
405
|
uniqueWindowHours: 0,
|
|
406
|
+
source: VIEW_LOG_SOURCE.EVENT,
|
|
272
407
|
});
|
|
273
408
|
|
|
274
409
|
logger.audit('trackEvent', { ...logContext(req), appId, eventType });
|
|
275
410
|
|
|
276
411
|
return res.status(HTTP_STATUS.OK).json({
|
|
277
412
|
message: 'Event tracked successfully',
|
|
413
|
+
recorded: true,
|
|
414
|
+
id: result.publicId,
|
|
415
|
+
// Deprecated: the internal row number. Kept for 3.x clients; use `id`.
|
|
278
416
|
insertId: result.insertId,
|
|
279
417
|
});
|
|
280
418
|
} catch (error) {
|
|
419
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
281
420
|
return handleRouteError(req, res, error, 'track event');
|
|
282
421
|
}
|
|
283
422
|
}
|
|
284
423
|
);
|
|
285
424
|
|
|
425
|
+
/**
|
|
426
|
+
* Engagement for a recorded view: how long its page was visible and how
|
|
427
|
+
* far it was scrolled, sent by the tracker when the page is hidden or left.
|
|
428
|
+
*/
|
|
429
|
+
router.post('/engage',
|
|
430
|
+
express.text({ type: () => true, limit: TRACKING.ENGAGE_BODY_BYTES }),
|
|
431
|
+
parseBeaconBody,
|
|
432
|
+
limitPerApp,
|
|
433
|
+
requireOrigin,
|
|
434
|
+
validateEngage(config.allowed),
|
|
435
|
+
trackingValidation,
|
|
436
|
+
async (req, res) => {
|
|
437
|
+
try {
|
|
438
|
+
if (refuseBot(req)) return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
439
|
+
|
|
440
|
+
const { appId, id, ms, scroll } = req.body;
|
|
441
|
+
const updated = await dbManager.addEngagement(appId, {
|
|
442
|
+
viewId: id,
|
|
443
|
+
engagedMs: Number(ms),
|
|
444
|
+
scrollDepth: Number(scroll),
|
|
445
|
+
});
|
|
446
|
+
if (!updated) reject(req, REJECTION_REASON.UNKNOWN_VIEW);
|
|
447
|
+
return res.status(HTTP_STATUS.NO_CONTENT).end();
|
|
448
|
+
} catch (error) {
|
|
449
|
+
reject(req, REJECTION_REASON.SERVER_ERROR);
|
|
450
|
+
return handleRouteError(req, res, error, 'record engagement');
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
);
|
|
454
|
+
|
|
286
455
|
// ---- Read API. Everything below requires a valid key. -------------------
|
|
287
456
|
|
|
288
457
|
router.use(noStore);
|
|
@@ -415,7 +584,7 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
415
584
|
* Provision a new app. Admin tier only.
|
|
416
585
|
*
|
|
417
586
|
* Creates the app's table and records it in the registry, then adds it to
|
|
418
|
-
* the live allowlist so it accepts traffic immediately
|
|
587
|
+
* the live allowlist so it accepts traffic immediately, with no restart. The
|
|
419
588
|
* appId becomes a table identifier, so it is validated against a strict
|
|
420
589
|
* pattern before it reaches any DDL.
|
|
421
590
|
*/
|
|
@@ -450,7 +619,26 @@ function createAnalyticsRouter({ config, dbManager, isReady = () => true }) {
|
|
|
450
619
|
}
|
|
451
620
|
);
|
|
452
621
|
|
|
622
|
+
/** For the host app's own limiter, which runs before this router, and for shutdown. */
|
|
623
|
+
router.countRejection = reject;
|
|
624
|
+
router.flushRejections = () => rejections.flush();
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* For the app that mounts this router, after its body parser: a body that
|
|
628
|
+
* is malformed JSON or over the size limit never reaches the router, so it
|
|
629
|
+
* is counted here, as an invalid request to the endpoint it was sent to.
|
|
630
|
+
* @type {import('express').ErrorRequestHandler}
|
|
631
|
+
*/
|
|
632
|
+
// eslint-disable-next-line no-unused-vars
|
|
633
|
+
router.bodyErrorHandler = (err, req, res, next) => {
|
|
634
|
+
const status = err.status || err.statusCode || HTTP_STATUS.INTERNAL_SERVER_ERROR;
|
|
635
|
+
logger.warn(`Request rejected: ${err.message}`, { requestId: req.id });
|
|
636
|
+
reject(req, REJECTION_REASON.INVALID_REQUEST, { detail: 'body' });
|
|
637
|
+
res.status(status === HTTP_STATUS.INTERNAL_SERVER_ERROR ? HTTP_STATUS.BAD_REQUEST : status)
|
|
638
|
+
.json({ message: 'Malformed or oversized request' });
|
|
639
|
+
};
|
|
640
|
+
|
|
453
641
|
return router;
|
|
454
642
|
}
|
|
455
643
|
|
|
456
|
-
module.exports = { createAnalyticsRouter, handleRouteError, logContext, withRequestId };
|
|
644
|
+
module.exports = { createAnalyticsRouter, handleRouteError, logContext, withRequestId, trackingSourceFor };
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* viewcounter tracker, https://viewcounter.harshankur.com
|
|
3
|
+
*
|
|
4
|
+
* <script defer src="https://your-server/tracker.js" data-app="blog"></script>
|
|
5
|
+
*
|
|
6
|
+
* Records a view of each page (including page changes in single-page apps),
|
|
7
|
+
* how long it was visible and how far it was scrolled, clicks on links to
|
|
8
|
+
* other sites and on downloads, and the campaign tags of the landing URL.
|
|
9
|
+
* The page before is sent as its origin and path only.
|
|
10
|
+
*
|
|
11
|
+
* It stores nothing on the visitor's device (no cookie, no localStorage, no
|
|
12
|
+
* sessionStorage), so it needs no consent banner, and it sends no identifier:
|
|
13
|
+
* the server tells repeat visits apart with a hash it rotates every day.
|
|
14
|
+
*
|
|
15
|
+
* Options, as attributes on the script tag:
|
|
16
|
+
* data-app="blog" required: the app ID the views belong to
|
|
17
|
+
* data-hosts="example.com,www.example.com"
|
|
18
|
+
* only track on these hostnames (keeps dev
|
|
19
|
+
* servers and previews out of the data)
|
|
20
|
+
* data-spa="false" do not treat history changes as page views
|
|
21
|
+
* data-outbound="false" do not record clicks on links to other sites
|
|
22
|
+
* data-downloads="false" do not record clicks on downloads
|
|
23
|
+
* data-respect-dnt="true" send nothing when Do Not Track is on
|
|
24
|
+
*
|
|
25
|
+
* Custom events: window.viewcounter.track('signup', { plan: 'pro' }).
|
|
26
|
+
*/
|
|
27
|
+
(() => {
|
|
28
|
+
const script = document.currentScript;
|
|
29
|
+
const app = script && script.dataset.app;
|
|
30
|
+
if (!app) return;
|
|
31
|
+
|
|
32
|
+
const option = (name, fallback) => (script.dataset[name] === undefined ? fallback : script.dataset[name] !== 'false');
|
|
33
|
+
const hosts = (script.dataset.hosts || '').split(',').map((host) => host.trim().toLowerCase()).filter(Boolean);
|
|
34
|
+
if (hosts.length && !hosts.includes(location.hostname.toLowerCase())) return;
|
|
35
|
+
if (location.protocol !== 'http:' && location.protocol !== 'https:') return;
|
|
36
|
+
// Automated browsers are not visitors.
|
|
37
|
+
if (navigator.webdriver) return;
|
|
38
|
+
if (option('respectDnt', false) && (navigator.doNotTrack === '1' || window.doNotTrack === '1')) return;
|
|
39
|
+
|
|
40
|
+
const base = script.src.replace(/[^/]*$/, '');
|
|
41
|
+
const UTM = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content'];
|
|
42
|
+
const DOWNLOAD = /\.(pdf|zip|gz|tgz|rar|7z|dmg|exe|msi|pkg|deb|rpm|apk|iso|csv|xlsx?|docx?|pptx?|odt|ods|epub|mp3|mp4|mov|avi|wav)$/i;
|
|
43
|
+
const MAX_ENGAGED_MS = 6 * 60 * 60 * 1000;
|
|
44
|
+
|
|
45
|
+
const deviceSize = () => (innerWidth < 768 ? 'small' : innerWidth < 1200 ? 'medium' : 'large');
|
|
46
|
+
/** How much of the page has been on screen, from 0 to 100. */
|
|
47
|
+
const seen = () => {
|
|
48
|
+
const height = Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0);
|
|
49
|
+
return height <= 0 ? 100 : Math.min(100, Math.round(((scrollY + innerHeight) / height) * 100));
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/** A URL's origin and path: its query and fragment can carry tokens or emails. */
|
|
53
|
+
const originAndPath = (url) => {
|
|
54
|
+
try {
|
|
55
|
+
const parsed = new URL(url);
|
|
56
|
+
return `${parsed.origin}${parsed.pathname}`;
|
|
57
|
+
} catch {
|
|
58
|
+
return '';
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
let view = null;
|
|
63
|
+
let referrer = document.referrer ? originAndPath(document.referrer) : '';
|
|
64
|
+
let path = location.pathname;
|
|
65
|
+
|
|
66
|
+
/** Tell the server how long the current page was visible and how far it was scrolled. */
|
|
67
|
+
function reportEngagement() {
|
|
68
|
+
if (!view || !view.id) return;
|
|
69
|
+
const ms = Math.min(MAX_ENGAGED_MS, Math.round(view.visibleMs + (view.visibleSince === null ? 0 : performance.now() - view.visibleSince)));
|
|
70
|
+
if (ms <= view.sentMs && view.scroll <= view.sentScroll) return;
|
|
71
|
+
view.sentMs = ms;
|
|
72
|
+
view.sentScroll = view.scroll;
|
|
73
|
+
const body = JSON.stringify({ appId: app, id: view.id, ms, scroll: view.scroll });
|
|
74
|
+
// text/plain needs no CORS preflight, so the beacon survives the page closing.
|
|
75
|
+
if (!(navigator.sendBeacon && navigator.sendBeacon(`${base}engage`, new Blob([body], { type: 'text/plain' })))) {
|
|
76
|
+
fetch(`${base}engage`, { method: 'POST', body, keepalive: true, credentials: 'omit', headers: { 'Content-Type': 'text/plain' } })
|
|
77
|
+
.catch(() => {});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function pageview() {
|
|
82
|
+
reportEngagement();
|
|
83
|
+
const params = new URLSearchParams({
|
|
84
|
+
appId: app,
|
|
85
|
+
deviceSize: deviceSize(),
|
|
86
|
+
page: location.pathname.slice(0, 500),
|
|
87
|
+
title: document.title.slice(0, 200),
|
|
88
|
+
referrer: referrer.slice(0, 500),
|
|
89
|
+
});
|
|
90
|
+
// Only the campaign tags: the rest of a query string can carry
|
|
91
|
+
// emails, tokens, or IDs, and never leaves the page.
|
|
92
|
+
const query = new URLSearchParams(location.search);
|
|
93
|
+
for (const key of UTM) {
|
|
94
|
+
const value = query.get(key);
|
|
95
|
+
if (value) params.set(key, value.slice(0, 100));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const current = {
|
|
99
|
+
id: null,
|
|
100
|
+
visibleMs: 0,
|
|
101
|
+
visibleSince: document.visibilityState === 'visible' ? performance.now() : null,
|
|
102
|
+
scroll: seen(),
|
|
103
|
+
sentMs: 0,
|
|
104
|
+
sentScroll: 0,
|
|
105
|
+
};
|
|
106
|
+
view = current;
|
|
107
|
+
fetch(`${base}registerView?${params}`, { keepalive: true, credentials: 'omit', referrerPolicy: 'no-referrer' })
|
|
108
|
+
.then((response) => (response.ok ? response.json() : null))
|
|
109
|
+
.then((result) => { if (result && result.id) current.id = result.id; })
|
|
110
|
+
.catch(() => {});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function track(eventType, eventData) {
|
|
114
|
+
if (typeof eventType !== 'string' || !eventType) return;
|
|
115
|
+
fetch(`${base}event`, {
|
|
116
|
+
method: 'POST',
|
|
117
|
+
keepalive: true,
|
|
118
|
+
credentials: 'omit',
|
|
119
|
+
referrerPolicy: 'no-referrer',
|
|
120
|
+
headers: { 'Content-Type': 'application/json' },
|
|
121
|
+
body: JSON.stringify({
|
|
122
|
+
appId: app,
|
|
123
|
+
eventType: eventType.slice(0, 50),
|
|
124
|
+
eventData: eventData && typeof eventData === 'object' ? eventData : undefined,
|
|
125
|
+
page: location.pathname.slice(0, 500),
|
|
126
|
+
title: document.title.slice(0, 200),
|
|
127
|
+
}),
|
|
128
|
+
}).catch(() => {});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// A page change in a single-page app is a new page view, with the page it
|
|
132
|
+
// came from as its referrer (which the server files as internal).
|
|
133
|
+
function navigated() {
|
|
134
|
+
if (location.pathname === path) return;
|
|
135
|
+
referrer = `${location.origin}${path}`;
|
|
136
|
+
path = location.pathname;
|
|
137
|
+
pageview();
|
|
138
|
+
}
|
|
139
|
+
if (option('spa', true)) {
|
|
140
|
+
for (const method of ['pushState', 'replaceState']) {
|
|
141
|
+
const original = history[method];
|
|
142
|
+
history[method] = function patched(...args) {
|
|
143
|
+
const result = original.apply(this, args);
|
|
144
|
+
navigated();
|
|
145
|
+
return result;
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
addEventListener('popstate', navigated);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let scrollQueued = false;
|
|
152
|
+
addEventListener('scroll', () => {
|
|
153
|
+
if (scrollQueued) return;
|
|
154
|
+
scrollQueued = true;
|
|
155
|
+
requestAnimationFrame(() => {
|
|
156
|
+
scrollQueued = false;
|
|
157
|
+
if (view) view.scroll = Math.max(view.scroll, seen());
|
|
158
|
+
});
|
|
159
|
+
}, { passive: true });
|
|
160
|
+
|
|
161
|
+
document.addEventListener('visibilitychange', () => {
|
|
162
|
+
if (!view) return;
|
|
163
|
+
if (document.visibilityState === 'hidden') {
|
|
164
|
+
if (view.visibleSince !== null) view.visibleMs += performance.now() - view.visibleSince;
|
|
165
|
+
view.visibleSince = null;
|
|
166
|
+
reportEngagement();
|
|
167
|
+
} else if (view.visibleSince === null) {
|
|
168
|
+
view.visibleSince = performance.now();
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
addEventListener('pagehide', reportEngagement);
|
|
172
|
+
|
|
173
|
+
// Links out and downloads. Only the other site's hostname, or the file's
|
|
174
|
+
// name, is recorded: never the whole URL, which can carry personal data.
|
|
175
|
+
document.addEventListener('click', (event) => {
|
|
176
|
+
const link = event.target && event.target.closest ? event.target.closest('a[href]') : null;
|
|
177
|
+
if (!link) return;
|
|
178
|
+
let url;
|
|
179
|
+
try { url = new URL(link.href, location.href); } catch { return; }
|
|
180
|
+
if (option('downloads', true) && DOWNLOAD.test(url.pathname)) {
|
|
181
|
+
let file = url.pathname.split('/').pop();
|
|
182
|
+
try { file = decodeURIComponent(file); } catch { /* keep it encoded */ }
|
|
183
|
+
track('download', { file: file.slice(0, 100) });
|
|
184
|
+
} else if (option('outbound', true) && /^https?:$/.test(url.protocol) && url.hostname !== location.hostname) {
|
|
185
|
+
track('outbound', { host: url.hostname });
|
|
186
|
+
}
|
|
187
|
+
}, { capture: true });
|
|
188
|
+
|
|
189
|
+
window.viewcounter = { track };
|
|
190
|
+
pageview();
|
|
191
|
+
})();
|
package/utils/appIdUtils.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* App ID validation.
|
|
3
3
|
*
|
|
4
4
|
* An app ID becomes a MySQL table name. Identifiers cannot be bound as query
|
|
5
|
-
* parameters, so they are interpolated
|
|
5
|
+
* parameters, so they are interpolated, which is safe only because the value
|
|
6
6
|
* is checked here first. App IDs used to come exclusively from local config;
|
|
7
7
|
* the admin API now accepts them over HTTP, so this is a live injection
|
|
8
8
|
* boundary, not a formatting preference.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cookie header parsing.
|
|
3
|
+
*
|
|
4
|
+
* Small enough to own rather than pull in a dependency for, and strict about
|
|
5
|
+
* what it accepts: a malformed pair is skipped, never thrown on, because the
|
|
6
|
+
* header is attacker-controlled input.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** Bounds the work one request can cause here. */
|
|
10
|
+
const MAX_COOKIE_PAIRS = 64;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Parse a `Cookie` request header into a name -> value map.
|
|
14
|
+
*
|
|
15
|
+
* The first occurrence of a name wins, matching how browsers order cookies
|
|
16
|
+
* (most specific path first) and denying a later, attacker-planted duplicate.
|
|
17
|
+
* Values that are not valid percent-encoding are kept verbatim.
|
|
18
|
+
*
|
|
19
|
+
* @param {string|undefined} header
|
|
20
|
+
* @returns {Record<string, string>}
|
|
21
|
+
*/
|
|
22
|
+
function parseCookies(header) {
|
|
23
|
+
const cookies = Object.create(null);
|
|
24
|
+
if (typeof header !== 'string' || header.length === 0) return cookies;
|
|
25
|
+
|
|
26
|
+
const pairs = header.split(';').slice(0, MAX_COOKIE_PAIRS);
|
|
27
|
+
for (const pair of pairs) {
|
|
28
|
+
const separator = pair.indexOf('=');
|
|
29
|
+
if (separator <= 0) continue;
|
|
30
|
+
|
|
31
|
+
const name = pair.slice(0, separator).trim();
|
|
32
|
+
if (!name || name in cookies) continue;
|
|
33
|
+
|
|
34
|
+
let value = pair.slice(separator + 1).trim();
|
|
35
|
+
if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
|
|
36
|
+
value = value.slice(1, -1);
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
cookies[name] = decodeURIComponent(value);
|
|
40
|
+
} catch {
|
|
41
|
+
cookies[name] = value;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return cookies;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
module.exports = { parseCookies, MAX_COOKIE_PAIRS };
|