@harshankur/viewcounter 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +115 -0
- package/LICENSE +21 -0
- package/README.md +797 -0
- package/allowed.sample.json +24 -0
- package/config/index.js +362 -0
- package/constants.js +229 -0
- package/db/DatabaseManager.js +704 -0
- package/db/schema.sql +71 -0
- package/dbInfo.sample.json +8 -0
- package/index.js +184 -0
- package/middleware/auth.js +194 -0
- package/middleware/security.js +109 -0
- package/middleware/validation.js +212 -0
- package/package.json +81 -0
- package/routes/analytics.js +456 -0
- package/scripts/setup.js +191 -0
- package/utils/appIdUtils.js +38 -0
- package/utils/errorUtils.js +157 -0
- package/utils/ipUtils.js +63 -0
- package/utils/logger.js +138 -0
- package/utils/privacyUtils.js +107 -0
- package/utils/referrerParser.js +137 -0
- package/utils/secretStore.js +57 -0
- package/utils/stringUtils.js +36 -0
- package/utils/userAgentParser.js +68 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Centralized error and warning handling.
|
|
3
|
+
*
|
|
4
|
+
* Per agent-instructions CODE_STANDARDS.md §3, application code never calls
|
|
5
|
+
* `new Error("...")` or `console.warn/error` directly. Every failure is
|
|
6
|
+
* identified by an enum member, its message text lives in exactly one table
|
|
7
|
+
* here, and callers branch on `err.code` rather than parsing message strings.
|
|
8
|
+
*
|
|
9
|
+
* `getError` builds and reports but deliberately does NOT throw — the `throw`
|
|
10
|
+
* stays visible at the call site and under the caller's control.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const logger = require('./logger');
|
|
14
|
+
|
|
15
|
+
/** Fatal conditions. Every thrown error in this project is one of these. */
|
|
16
|
+
const ErrorType = {
|
|
17
|
+
DATABASE_NOT_INITIALIZED: 'DATABASE_NOT_INITIALIZED',
|
|
18
|
+
DATABASE_CONNECTION_FAILED: 'DATABASE_CONNECTION_FAILED',
|
|
19
|
+
DATABASE_QUERY_FAILED: 'DATABASE_QUERY_FAILED',
|
|
20
|
+
CONFIG_INSECURE_DEFAULT: 'CONFIG_INSECURE_DEFAULT',
|
|
21
|
+
CONFIG_MISSING_REQUIRED: 'CONFIG_MISSING_REQUIRED',
|
|
22
|
+
CONFIG_INVALID_VALUE: 'CONFIG_INVALID_VALUE',
|
|
23
|
+
INVALID_APP_ID: 'INVALID_APP_ID',
|
|
24
|
+
SECRET_PERSIST_FAILED: 'SECRET_PERSIST_FAILED',
|
|
25
|
+
SECRET_UNAVAILABLE: 'SECRET_UNAVAILABLE',
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** Non-fatal conditions worth surfacing but not worth stopping for. */
|
|
29
|
+
const WarningType = {
|
|
30
|
+
CONFIG_FILE_UNREADABLE: 'CONFIG_FILE_UNREADABLE',
|
|
31
|
+
CONFIG_FIELD_MISSING: 'CONFIG_FIELD_MISSING',
|
|
32
|
+
PROXY_TRUST_PERMISSIVE: 'PROXY_TRUST_PERMISSIVE',
|
|
33
|
+
SECRET_GENERATED: 'SECRET_GENERATED',
|
|
34
|
+
READ_API_UNPROTECTED: 'READ_API_UNPROTECTED',
|
|
35
|
+
API_KEY_TOO_SHORT: 'API_KEY_TOO_SHORT',
|
|
36
|
+
API_KEY_EMPTY_SCOPE: 'API_KEY_EMPTY_SCOPE',
|
|
37
|
+
APP_ALREADY_REGISTERED: 'APP_ALREADY_REGISTERED',
|
|
38
|
+
FIELD_TRUNCATED: 'FIELD_TRUNCATED',
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Message text, in exactly one place. A plain string for a fixed message, a
|
|
43
|
+
* function where dynamic detail has to be interpolated.
|
|
44
|
+
* @type {Record<string, string | ((info: any) => string)>}
|
|
45
|
+
*/
|
|
46
|
+
const ERROR_MESSAGES = {
|
|
47
|
+
[ErrorType.DATABASE_NOT_INITIALIZED]: 'Database not initialized',
|
|
48
|
+
[ErrorType.DATABASE_CONNECTION_FAILED]: (info) =>
|
|
49
|
+
`Failed to connect to database '${info?.database}' at ${info?.host}:${info?.port}`,
|
|
50
|
+
[ErrorType.DATABASE_QUERY_FAILED]: (info) => `Database query failed: ${info?.operation}`,
|
|
51
|
+
[ErrorType.CONFIG_INSECURE_DEFAULT]: (info) =>
|
|
52
|
+
`Refusing to start: ${info?.field} is still at its insecure default. ` +
|
|
53
|
+
'Set it explicitly via dbInfo.json, allowed.json, or the environment.',
|
|
54
|
+
[ErrorType.CONFIG_MISSING_REQUIRED]: (info) =>
|
|
55
|
+
`Refusing to start: required configuration '${info?.field}' is missing or empty.`,
|
|
56
|
+
[ErrorType.CONFIG_INVALID_VALUE]: (info) =>
|
|
57
|
+
`Invalid configuration for '${info?.field}': ${info?.reason}`,
|
|
58
|
+
[ErrorType.INVALID_APP_ID]: (info) =>
|
|
59
|
+
`Invalid appId '${info?.appId}'. Must be 1-64 characters of letters, digits, ` +
|
|
60
|
+
'underscore, or hyphen, and must not start with an underscore.',
|
|
61
|
+
[ErrorType.SECRET_PERSIST_FAILED]: (info) =>
|
|
62
|
+
`Could not persist the visitor-hash secret to ${info?.path}. ` +
|
|
63
|
+
'Without a stable secret, visitor hashes are not reversible-resistant across restarts.',
|
|
64
|
+
[ErrorType.SECRET_UNAVAILABLE]: 'Visitor-hash secret has not been initialized',
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/** @type {Record<string, string | ((info: any) => string)>} */
|
|
68
|
+
const WARNING_MESSAGES = {
|
|
69
|
+
[WarningType.CONFIG_FILE_UNREADABLE]: (info) =>
|
|
70
|
+
`Could not parse ${info?.file}; falling back to environment variables and defaults.`,
|
|
71
|
+
[WarningType.CONFIG_FIELD_MISSING]: (info) =>
|
|
72
|
+
`Config field '${info?.field}' absent from ${info?.file}; using ${info?.source}.`,
|
|
73
|
+
[WarningType.PROXY_TRUST_PERMISSIVE]: (info) =>
|
|
74
|
+
`TRUST_PROXY is set to '${info?.value}'. Client-supplied forwarding headers will be ` +
|
|
75
|
+
'trusted, which lets a caller forge their own IP and bypass rate limiting. ' +
|
|
76
|
+
'Set it to the number of proxy hops or an explicit CIDR list.',
|
|
77
|
+
[WarningType.SECRET_GENERATED]: (info) =>
|
|
78
|
+
`Generated a new visitor-hash secret at ${info?.path}. Existing visitor hashes ` +
|
|
79
|
+
'are now unlinkable from new ones, which is the intended privacy behaviour.',
|
|
80
|
+
[WarningType.READ_API_UNPROTECTED]: 'No read API keys configured; analytics read endpoints are disabled.',
|
|
81
|
+
[WarningType.API_KEY_TOO_SHORT]: (info) =>
|
|
82
|
+
`Ignoring an API key of length ${info?.length}; keys must be at least 32 characters.`,
|
|
83
|
+
[WarningType.API_KEY_EMPTY_SCOPE]: (info) =>
|
|
84
|
+
`Ignoring an API key with an unusable scope (${info?.scope}). Use "*" or a non-empty array of app IDs.`,
|
|
85
|
+
[WarningType.APP_ALREADY_REGISTERED]: (info) =>
|
|
86
|
+
`App '${info?.appId}' is already registered; leaving it as-is.`,
|
|
87
|
+
[WarningType.FIELD_TRUNCATED]: (info) =>
|
|
88
|
+
`Field '${info?.field}' exceeded ${info?.max} characters and was truncated before storage.`,
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Resolve a message from its table, whether it is a literal or a builder.
|
|
93
|
+
* @returns {string}
|
|
94
|
+
*/
|
|
95
|
+
function resolveMessage(table, type, info) {
|
|
96
|
+
const entry = table[type];
|
|
97
|
+
if (entry === undefined) return `Unknown issue: ${type}`;
|
|
98
|
+
return typeof entry === 'function' ? entry(info) : entry;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The one place an issue is reported. Routes through the logger so the project
|
|
103
|
+
* has a single console sink rather than two competing ones.
|
|
104
|
+
* @param {{type: 'error'|'warning', code: string, message: string, details?: unknown}} issue
|
|
105
|
+
*/
|
|
106
|
+
function report(issue) {
|
|
107
|
+
const context = issue.details && typeof issue.details === 'object' ? issue.details : {};
|
|
108
|
+
const line = `${issue.code}: ${issue.message}`;
|
|
109
|
+
if (issue.type === 'error') {
|
|
110
|
+
logger.error(line, context);
|
|
111
|
+
} else {
|
|
112
|
+
logger.warn(line, context);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Build, report, and return an Error. The caller throws it.
|
|
118
|
+
*
|
|
119
|
+
* Cancellation carve-out (CODE_STANDARDS.md §3): an AbortError carries an
|
|
120
|
+
* identity the runtime depends on, so it is passed straight back rather than
|
|
121
|
+
* being wrapped and stripped of its `name`.
|
|
122
|
+
*
|
|
123
|
+
* @param {string} type - an ErrorType member
|
|
124
|
+
* @param {object} [info] - interpolation detail, also attached as `details`
|
|
125
|
+
* @returns {Error}
|
|
126
|
+
*/
|
|
127
|
+
function getError(type, info) {
|
|
128
|
+
if (info instanceof Error && info.name === 'AbortError') return info;
|
|
129
|
+
|
|
130
|
+
const message = resolveMessage(ERROR_MESSAGES, type, info);
|
|
131
|
+
const issue = { type: 'error', code: type, message, details: info };
|
|
132
|
+
report(issue);
|
|
133
|
+
|
|
134
|
+
const err = new Error(message);
|
|
135
|
+
err.code = type;
|
|
136
|
+
err.details = info;
|
|
137
|
+
return err;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Report a non-fatal issue. Never throws, returns nothing.
|
|
142
|
+
* @param {string} type - a WarningType member
|
|
143
|
+
* @param {object} [info]
|
|
144
|
+
*/
|
|
145
|
+
function logWarning(type, info) {
|
|
146
|
+
const message = resolveMessage(WARNING_MESSAGES, type, info);
|
|
147
|
+
report({ type: 'warning', code: type, message, details: info });
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
module.exports = {
|
|
151
|
+
ErrorType,
|
|
152
|
+
WarningType,
|
|
153
|
+
ERROR_MESSAGES,
|
|
154
|
+
WARNING_MESSAGES,
|
|
155
|
+
getError,
|
|
156
|
+
logWarning,
|
|
157
|
+
};
|
package/utils/ipUtils.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client IP derivation and validation.
|
|
3
|
+
*
|
|
4
|
+
* Split out of middleware/validation.js because it is used by the route layer,
|
|
5
|
+
* the privacy layer, and the middleware layer (CODE_STANDARDS.md §2).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const net = require('net');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Validate an IP address.
|
|
12
|
+
*
|
|
13
|
+
* Uses Node's built-in parser rather than a hand-written regex. The previous
|
|
14
|
+
* IPv4 pattern (`^(\d{1,3}\.){3}\d{1,3}$`) had no octet range check, so
|
|
15
|
+
* `999.999.999.999` validated and was stored as a masked "address" that never
|
|
16
|
+
* existed. `net.isIP` range-checks properly and cannot backtrack.
|
|
17
|
+
*
|
|
18
|
+
* @param {string} ip
|
|
19
|
+
* @returns {boolean}
|
|
20
|
+
*/
|
|
21
|
+
function isValidIP(ip) {
|
|
22
|
+
if (!ip || typeof ip !== 'string') return false;
|
|
23
|
+
return net.isIP(ip) !== 0;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Resolve the client's IP for a request.
|
|
28
|
+
*
|
|
29
|
+
* Deliberately reads ONLY `req.ip`, which Express derives according to the
|
|
30
|
+
* app's `trust proxy` setting. The previous implementation read `x-real-ip`
|
|
31
|
+
* and `x-forwarded-for` straight off the request, so any caller could name
|
|
32
|
+
* their own address — forging geolocation, inflating unique-visitor counts,
|
|
33
|
+
* and rotating the rate-limiter key to bypass it entirely.
|
|
34
|
+
*
|
|
35
|
+
* If a reverse proxy in front of this service sets only `X-Real-IP`, configure
|
|
36
|
+
* it to also set `X-Forwarded-For`; that is the header Express understands.
|
|
37
|
+
*
|
|
38
|
+
* @param {import('express').Request} req
|
|
39
|
+
* @returns {string|undefined}
|
|
40
|
+
*/
|
|
41
|
+
function getClientIp(req) {
|
|
42
|
+
return req.ip;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Normalize an IPv4-mapped IPv6 address to its IPv4 form.
|
|
47
|
+
* Express reports `::ffff:1.2.3.4` on a dual-stack socket; geo lookup and
|
|
48
|
+
* masking both want the plain IPv4.
|
|
49
|
+
*
|
|
50
|
+
* @param {string} ip
|
|
51
|
+
* @returns {string}
|
|
52
|
+
*/
|
|
53
|
+
function normalizeIp(ip) {
|
|
54
|
+
if (typeof ip !== 'string') return ip;
|
|
55
|
+
const mapped = ip.match(/^::ffff:((?:\d{1,3}\.){3}\d{1,3})$/i);
|
|
56
|
+
return mapped && net.isIPv4(mapped[1]) ? mapped[1] : ip;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
module.exports = {
|
|
60
|
+
isValidIP,
|
|
61
|
+
getClientIp,
|
|
62
|
+
normalizeIp,
|
|
63
|
+
};
|
package/utils/logger.js
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Structured logger.
|
|
3
|
+
*
|
|
4
|
+
* Per agent-instructions LOGGING.md this is the ONLY module in the project
|
|
5
|
+
* permitted to write to the console. Application code calls debug/info/warn/
|
|
6
|
+
* error/audit; utils/errorUtils.js routes error and warning reporting here so
|
|
7
|
+
* there is exactly one sink, not two.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const { APP_SLUG } = require('../constants');
|
|
11
|
+
|
|
12
|
+
/** LOGGING.md §1. */
|
|
13
|
+
const LogLevel = {
|
|
14
|
+
DEBUG: 'debug',
|
|
15
|
+
INFO: 'info',
|
|
16
|
+
WARN: 'warn',
|
|
17
|
+
ERROR: 'error',
|
|
18
|
+
/** Emits nothing except the audit channel. Used by the test suite. */
|
|
19
|
+
SILENT: 'silent',
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const LOG_LEVEL_PRIORITY = {
|
|
23
|
+
[LogLevel.DEBUG]: 10,
|
|
24
|
+
[LogLevel.INFO]: 20,
|
|
25
|
+
[LogLevel.WARN]: 30,
|
|
26
|
+
[LogLevel.ERROR]: 40,
|
|
27
|
+
[LogLevel.SILENT]: Number.MAX_SAFE_INTEGER,
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/** Sensitive keys redacted from any diagnostic dump (LOGGING.md §5). */
|
|
31
|
+
const REDACTED_KEYS = new Set(['password', 'secret', 'apikey', 'apikeys', 'token', 'authorization']);
|
|
32
|
+
|
|
33
|
+
const REDACTED_PLACEHOLDER = '[SET]';
|
|
34
|
+
const ABSENT_PLACEHOLDER = '[NOT SET]';
|
|
35
|
+
|
|
36
|
+
let threshold = LogLevel.INFO;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The single underlying write. Swappable so tests can capture output without
|
|
40
|
+
* monkey-patching the console.
|
|
41
|
+
*/
|
|
42
|
+
let sink = (line) => process.stdout.write(`${line}\n`);
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Configure the logger. Called once from the server bootstrap after config is
|
|
46
|
+
* resolved, so this module never reads process.env itself (CONFIG.md §0).
|
|
47
|
+
* @param {{ level?: string, writer?: (line: string) => void }} options
|
|
48
|
+
*/
|
|
49
|
+
function configure({ level, writer } = {}) {
|
|
50
|
+
if (level && Object.prototype.hasOwnProperty.call(LOG_LEVEL_PRIORITY, level)) {
|
|
51
|
+
threshold = level;
|
|
52
|
+
}
|
|
53
|
+
if (typeof writer === 'function') {
|
|
54
|
+
sink = writer;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** @returns {boolean} whether a message at this level would be emitted. */
|
|
59
|
+
function isEnabled(level) {
|
|
60
|
+
return LOG_LEVEL_PRIORITY[level] >= LOG_LEVEL_PRIORITY[threshold];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Render the correlation context as fixed-position bracket fields so the
|
|
65
|
+
* stream stays greppable (LOGGING.md §2). Absent fields render as `-` rather
|
|
66
|
+
* than being omitted, so column positions never shift.
|
|
67
|
+
*/
|
|
68
|
+
function formatContext(context = {}) {
|
|
69
|
+
const ip = context.ip || '-';
|
|
70
|
+
const requestId = context.requestId || '-';
|
|
71
|
+
return `[${ip}] [${requestId}]`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* LOGGING.md §4: a failing sink degrades logging, it never takes down the
|
|
76
|
+
* request that triggered it.
|
|
77
|
+
*/
|
|
78
|
+
function write(level, message, context) {
|
|
79
|
+
try {
|
|
80
|
+
const timestamp = new Date().toISOString();
|
|
81
|
+
const line = `[${timestamp}] [${level.toUpperCase()}] ${formatContext(context)} ${message}`;
|
|
82
|
+
sink(line);
|
|
83
|
+
} catch {
|
|
84
|
+
// Intentionally swallowed. There is nowhere left to report to.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function log(level, message, context) {
|
|
89
|
+
if (!isEnabled(level)) return;
|
|
90
|
+
write(level, message, context);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const debug = (message, context) => log(LogLevel.DEBUG, message, context);
|
|
94
|
+
const info = (message, context) => log(LogLevel.INFO, message, context);
|
|
95
|
+
const warn = (message, context) => log(LogLevel.WARN, message, context);
|
|
96
|
+
const error = (message, context) => log(LogLevel.ERROR, message, context);
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Audit channel for state-mutating actions (LOGGING.md §3).
|
|
100
|
+
*
|
|
101
|
+
* Deliberately bypasses the level threshold: turning operational verbosity
|
|
102
|
+
* down must never silently discard the record of who changed what. Tagged
|
|
103
|
+
* distinctly so it can be split to its own destination downstream.
|
|
104
|
+
*/
|
|
105
|
+
function audit(action, context = {}) {
|
|
106
|
+
const actor = context.actor || context.appId || 'anonymous';
|
|
107
|
+
write(`audit:${APP_SLUG}`, `action=${action} actor=${actor}`, context);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Redact sensitive values for diagnostics output (LOGGING.md §5).
|
|
112
|
+
* Allowlist semantics: anything whose key matches a sensitive name is replaced
|
|
113
|
+
* with a presence marker, never its value.
|
|
114
|
+
*/
|
|
115
|
+
function redact(record = {}) {
|
|
116
|
+
const safe = {};
|
|
117
|
+
for (const [key, value] of Object.entries(record)) {
|
|
118
|
+
if (REDACTED_KEYS.has(key.toLowerCase())) {
|
|
119
|
+
safe[key] = value ? REDACTED_PLACEHOLDER : ABSENT_PLACEHOLDER;
|
|
120
|
+
} else {
|
|
121
|
+
safe[key] = value;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return safe;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
module.exports = {
|
|
128
|
+
LogLevel,
|
|
129
|
+
LOG_LEVEL_PRIORITY,
|
|
130
|
+
configure,
|
|
131
|
+
isEnabled,
|
|
132
|
+
debug,
|
|
133
|
+
info,
|
|
134
|
+
warn,
|
|
135
|
+
error,
|
|
136
|
+
audit,
|
|
137
|
+
redact,
|
|
138
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
const crypto = require('crypto');
|
|
2
|
+
|
|
3
|
+
const { SERVER } = require('../constants');
|
|
4
|
+
const { getError, ErrorType } = require('./errorUtils');
|
|
5
|
+
|
|
6
|
+
/** Rotation never runs slower than this, even if dedup is disabled. */
|
|
7
|
+
const MIN_ROTATION_HOURS = 1;
|
|
8
|
+
|
|
9
|
+
const MS_PER_HOUR = 60 * 60 * 1000;
|
|
10
|
+
|
|
11
|
+
class PrivacyUtils {
|
|
12
|
+
/**
|
|
13
|
+
* Masks the IP address to be non-identifiable.
|
|
14
|
+
* IPv4: Masks the last octet (e.g. 1.2.3.4 -> 1.2.3.0)
|
|
15
|
+
* IPv6: Masks the last 64 bits (interface identifier)
|
|
16
|
+
* @param {string} ip The raw IP address
|
|
17
|
+
* @returns {string} The masked IP
|
|
18
|
+
*/
|
|
19
|
+
static maskIP(ip) {
|
|
20
|
+
if (!ip) return '0.0.0.0';
|
|
21
|
+
|
|
22
|
+
// Handle IPv4-mapped IPv6 (::ffff:127.0.0.1)
|
|
23
|
+
if (ip.startsWith('::ffff:')) {
|
|
24
|
+
const ipv4 = ip.split(':').pop();
|
|
25
|
+
return `::ffff:${this.maskIPv4(ipv4)}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (ip.includes(':')) {
|
|
29
|
+
return this.maskIPv6(ip);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return this.maskIPv4(ip);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
static maskIPv4(ip) {
|
|
36
|
+
const parts = ip.split('.');
|
|
37
|
+
if (parts.length === 4) {
|
|
38
|
+
return `${parts[0]}.${parts[1]}.${parts[2]}.0`;
|
|
39
|
+
}
|
|
40
|
+
return ip;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
static maskIPv6(ip) {
|
|
44
|
+
const parts = ip.split(':');
|
|
45
|
+
// Mask the last 4 groups (64 bits)
|
|
46
|
+
if (parts.length >= 4) {
|
|
47
|
+
return parts.slice(0, 4).join(':') + ':0:0:0:0';
|
|
48
|
+
}
|
|
49
|
+
return ip;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Identifier for the current rotation window.
|
|
54
|
+
*
|
|
55
|
+
* Including this in the hash input means a visitor's hash changes on every
|
|
56
|
+
* window boundary, so two records from different windows cannot be linked
|
|
57
|
+
* back to the same person even by whoever holds the secret.
|
|
58
|
+
*
|
|
59
|
+
* @param {number} rotationHours
|
|
60
|
+
* @param {number} [now] epoch millis, injectable for tests
|
|
61
|
+
* @returns {number}
|
|
62
|
+
*/
|
|
63
|
+
static currentWindowId(rotationHours, now = Date.now()) {
|
|
64
|
+
const hours = Math.max(Number(rotationHours) || 0, MIN_ROTATION_HOURS);
|
|
65
|
+
return Math.floor(now / (hours * MS_PER_HOUR));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Generate the transient visitor identifier.
|
|
70
|
+
*
|
|
71
|
+
* This is a keyed HMAC, not a bare digest. Every non-secret input is
|
|
72
|
+
* public or guessable — the date is known, user agents come from a small
|
|
73
|
+
* population, and IPv4 is only 2^32 — so an unkeyed SHA-256 of them is
|
|
74
|
+
* reversible by exhaustive search in about an hour on one CPU core. The
|
|
75
|
+
* server secret is what makes that search infeasible; the window id is
|
|
76
|
+
* what stops hashes being linkable over time.
|
|
77
|
+
*
|
|
78
|
+
* @param {string} ip Raw IP address
|
|
79
|
+
* @param {string} userAgent Raw User-Agent string
|
|
80
|
+
* @param {string} secret Server secret from utils/secretStore.js
|
|
81
|
+
* @param {number} [rotationHours] Window length, defaults to the unique-visitor window
|
|
82
|
+
* @param {number} [now] epoch millis, injectable for tests
|
|
83
|
+
* @returns {string} HMAC-SHA-256 hex digest
|
|
84
|
+
* @throws {Error} ErrorType.SECRET_UNAVAILABLE when no secret is supplied
|
|
85
|
+
*/
|
|
86
|
+
static generateVisitorHash(
|
|
87
|
+
ip,
|
|
88
|
+
userAgent,
|
|
89
|
+
secret,
|
|
90
|
+
rotationHours = SERVER.DEFAULT_UNIQUE_VISITOR_WINDOW_HOURS,
|
|
91
|
+
now = Date.now(),
|
|
92
|
+
) {
|
|
93
|
+
if (!secret) {
|
|
94
|
+
// Failing closed is deliberate: silently hashing without the key
|
|
95
|
+
// would produce reversible identifiers that look indistinguishable
|
|
96
|
+
// from safe ones.
|
|
97
|
+
throw getError(ErrorType.SECRET_UNAVAILABLE);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const windowId = this.currentWindowId(rotationHours, now);
|
|
101
|
+
const input = `${ip}|${userAgent}|${windowId}`;
|
|
102
|
+
|
|
103
|
+
return crypto.createHmac('sha256', secret).update(input).digest('hex');
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = PrivacyUtils;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
const URL = require('url-parse');
|
|
2
|
+
|
|
3
|
+
const { FIELD_MAX_LENGTH, QUERY_LIMITS, SOURCE_TYPE } = require('../constants');
|
|
4
|
+
const { truncate } = require('./stringUtils');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Parse referrer URLs to extract domain and source type
|
|
8
|
+
*/
|
|
9
|
+
class ReferrerParser {
|
|
10
|
+
/**
|
|
11
|
+
* Parse referrer URL
|
|
12
|
+
* @param {string} referrer - Referrer URL from request headers
|
|
13
|
+
* @returns {object} Parsed referrer data
|
|
14
|
+
*/
|
|
15
|
+
static parse(referrer) {
|
|
16
|
+
if (!referrer || referrer === '') {
|
|
17
|
+
return {
|
|
18
|
+
referrer: null,
|
|
19
|
+
referrerDomain: null,
|
|
20
|
+
sourceType: SOURCE_TYPE.DIRECT
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
const url = new URL(referrer);
|
|
26
|
+
const domain = url.hostname || null;
|
|
27
|
+
const sourceType = this.getSourceType(domain, referrer);
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
referrer: truncate(referrer, FIELD_MAX_LENGTH.REFERRER),
|
|
31
|
+
referrerDomain: truncate(domain, FIELD_MAX_LENGTH.REFERRER_DOMAIN),
|
|
32
|
+
sourceType
|
|
33
|
+
};
|
|
34
|
+
} catch (error) {
|
|
35
|
+
return {
|
|
36
|
+
referrer: truncate(referrer, FIELD_MAX_LENGTH.REFERRER),
|
|
37
|
+
referrerDomain: null,
|
|
38
|
+
sourceType: SOURCE_TYPE.UNKNOWN
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Determine source type based on referrer domain
|
|
45
|
+
* @param {string} domain - Referrer domain
|
|
46
|
+
* @param {string} fullUrl - Full referrer URL
|
|
47
|
+
* @returns {string} Source type
|
|
48
|
+
*/
|
|
49
|
+
static getSourceType(domain, fullUrl) {
|
|
50
|
+
if (!domain) return SOURCE_TYPE.DIRECT;
|
|
51
|
+
|
|
52
|
+
const lowerDomain = domain.toLowerCase();
|
|
53
|
+
|
|
54
|
+
// Search engines
|
|
55
|
+
if (this.isSearchEngine(lowerDomain)) return SOURCE_TYPE.SEARCH;
|
|
56
|
+
|
|
57
|
+
// Social media
|
|
58
|
+
if (this.isSocialMedia(lowerDomain)) return SOURCE_TYPE.SOCIAL;
|
|
59
|
+
|
|
60
|
+
// Email clients
|
|
61
|
+
if (this.isEmail(lowerDomain)) return SOURCE_TYPE.EMAIL;
|
|
62
|
+
|
|
63
|
+
// Ads/campaigns (check for utm parameters)
|
|
64
|
+
if (fullUrl.includes('utm_source') || fullUrl.includes('utm_medium')) {
|
|
65
|
+
return SOURCE_TYPE.CAMPAIGN;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Everything else is referral
|
|
69
|
+
return SOURCE_TYPE.REFERRAL;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Check if domain is a search engine
|
|
74
|
+
*/
|
|
75
|
+
static isSearchEngine(domain) {
|
|
76
|
+
const searchEngines = [
|
|
77
|
+
'google', 'bing', 'yahoo', 'duckduckgo', 'baidu',
|
|
78
|
+
'yandex', 'ask', 'aol', 'ecosia', 'qwant'
|
|
79
|
+
];
|
|
80
|
+
return searchEngines.some(engine => domain.includes(engine));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Check if domain is social media
|
|
85
|
+
*/
|
|
86
|
+
static isSocialMedia(domain) {
|
|
87
|
+
const socialMedia = [
|
|
88
|
+
'facebook', 'twitter', 'x.com', 'instagram', 'linkedin',
|
|
89
|
+
'reddit', 'pinterest', 'tiktok', 'youtube', 'snapchat',
|
|
90
|
+
'whatsapp', 'telegram', 'discord', 'tumblr', 'vk.com',
|
|
91
|
+
'weibo', 'line.me', 'mastodon'
|
|
92
|
+
];
|
|
93
|
+
return socialMedia.some(social => domain.includes(social));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Check if domain is email client
|
|
98
|
+
*/
|
|
99
|
+
static isEmail(domain) {
|
|
100
|
+
const emailClients = [
|
|
101
|
+
'mail.google', 'outlook', 'yahoo.com/mail',
|
|
102
|
+
'mail.yahoo', 'protonmail', 'mail.aol'
|
|
103
|
+
];
|
|
104
|
+
return emailClients.some(email => domain.includes(email));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Get top referrers summary
|
|
109
|
+
* @param {Array} referrers - Array of referrer objects from DB
|
|
110
|
+
* @returns {object} Summarized referrer stats
|
|
111
|
+
*/
|
|
112
|
+
static summarizeReferrers(referrers) {
|
|
113
|
+
const bySource = {};
|
|
114
|
+
const byDomain = {};
|
|
115
|
+
|
|
116
|
+
referrers.forEach(ref => {
|
|
117
|
+
const sourceType = ref.sourceType || SOURCE_TYPE.UNKNOWN;
|
|
118
|
+
bySource[sourceType] = (bySource[sourceType] || 0) + 1;
|
|
119
|
+
|
|
120
|
+
if (ref.referrerDomain) {
|
|
121
|
+
byDomain[ref.referrerDomain] = (byDomain[ref.referrerDomain] || 0) + 1;
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
bySource: Object.entries(bySource)
|
|
127
|
+
.map(([source, count]) => ({ source, count }))
|
|
128
|
+
.sort((a, b) => b.count - a.count),
|
|
129
|
+
byDomain: Object.entries(byDomain)
|
|
130
|
+
.map(([domain, count]) => ({ domain, count }))
|
|
131
|
+
.sort((a, b) => b.count - a.count)
|
|
132
|
+
.slice(0, QUERY_LIMITS.LIST_LIMIT_DEFAULT)
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
module.exports = ReferrerParser;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persisted server secret for visitor hashing.
|
|
3
|
+
*
|
|
4
|
+
* agent-instructions SECURITY.md §1: never fall back to a weak, guessable
|
|
5
|
+
* default for a security-relevant value — generate one cryptographically and
|
|
6
|
+
* persist it. This module is that generator.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const crypto = require('crypto');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
|
|
13
|
+
const { PRIVACY } = require('../constants');
|
|
14
|
+
const { getError, logWarning, ErrorType, WarningType } = require('./errorUtils');
|
|
15
|
+
|
|
16
|
+
/** Matches a hex string of exactly the expected entropy. */
|
|
17
|
+
const SECRET_PATTERN = new RegExp(`^[0-9a-f]{${PRIVACY.SECRET_BYTES * 2}}$`);
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Load the visitor-hash secret, generating and persisting it on first run.
|
|
21
|
+
*
|
|
22
|
+
* The file is written with mode 0600 (owner read/write only): this secret is
|
|
23
|
+
* the sole reason a stored visitor hash cannot be brute-forced back to the raw
|
|
24
|
+
* IP that produced it, so it is as sensitive as a database password.
|
|
25
|
+
*
|
|
26
|
+
* @param {string} filePath - absolute path to the secret file
|
|
27
|
+
* @returns {string} hex-encoded secret
|
|
28
|
+
* @throws {Error} ErrorType.SECRET_PERSIST_FAILED if it cannot be written
|
|
29
|
+
*/
|
|
30
|
+
function loadOrCreate(filePath) {
|
|
31
|
+
if (fs.existsSync(filePath)) {
|
|
32
|
+
const existing = fs.readFileSync(filePath, 'utf8').trim();
|
|
33
|
+
if (SECRET_PATTERN.test(existing)) {
|
|
34
|
+
return existing;
|
|
35
|
+
}
|
|
36
|
+
// A malformed file is treated as absent and replaced. Keeping a short
|
|
37
|
+
// or corrupted secret would silently weaken every hash derived from it.
|
|
38
|
+
logWarning(WarningType.CONFIG_FILE_UNREADABLE, { file: filePath });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const secret = crypto.randomBytes(PRIVACY.SECRET_BYTES).toString('hex');
|
|
42
|
+
|
|
43
|
+
try {
|
|
44
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
45
|
+
fs.writeFileSync(filePath, `${secret}\n`, { mode: PRIVACY.SECRET_FILE_MODE });
|
|
46
|
+
// writeFileSync only applies `mode` when creating the file, so an
|
|
47
|
+
// existing file with looser permissions keeps them. Force it.
|
|
48
|
+
fs.chmodSync(filePath, PRIVACY.SECRET_FILE_MODE);
|
|
49
|
+
} catch (cause) {
|
|
50
|
+
throw getError(ErrorType.SECRET_PERSIST_FAILED, { path: filePath, cause: cause.message });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
logWarning(WarningType.SECRET_GENERATED, { path: filePath });
|
|
54
|
+
return secret;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
module.exports = { loadOrCreate, SECRET_PATTERN };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* String helpers shared by the persistence and parsing layers.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Truncate a value to a maximum length before it reaches a fixed-width column.
|
|
7
|
+
*
|
|
8
|
+
* Defence in depth behind boundary validation: validation rejects over-length
|
|
9
|
+
* input at the edge, but derived values (a browser version parsed out of a
|
|
10
|
+
* hostile User-Agent, for example) are produced internally and never pass
|
|
11
|
+
* through a validator. Under MySQL strict mode an over-long value is error
|
|
12
|
+
* 1406 and a 500; under non-strict mode it is a silent truncation.
|
|
13
|
+
*
|
|
14
|
+
* @param {unknown} value
|
|
15
|
+
* @param {number} max
|
|
16
|
+
* @returns {string|null} null for absent input, so callers can bind it directly
|
|
17
|
+
*/
|
|
18
|
+
function truncate(value, max) {
|
|
19
|
+
if (value === undefined || value === null || value === '') return null;
|
|
20
|
+
const str = String(value);
|
|
21
|
+
return str.length > max ? str.slice(0, max) : str;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Byte length of a value once serialized as JSON.
|
|
26
|
+
* Used to bound `eventData` before it is persisted.
|
|
27
|
+
*
|
|
28
|
+
* @param {unknown} value
|
|
29
|
+
* @returns {number}
|
|
30
|
+
*/
|
|
31
|
+
function jsonByteLength(value) {
|
|
32
|
+
if (value === undefined || value === null) return 0;
|
|
33
|
+
return Buffer.byteLength(JSON.stringify(value), 'utf8');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
module.exports = { truncate, jsonByteLength };
|