analytics-platform 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +247 -2
- package/index.d.ts +184 -0
- package/index.js +128 -0
- package/lib/analytics.js +105 -0
- package/lib/audit.js +57 -0
- package/lib/bans.js +78 -0
- package/lib/core.js +122 -0
- package/lib/crawlers.js +216 -0
- package/lib/errors.js +178 -0
- package/lib/firewall.js +262 -0
- package/lib/flags.js +41 -0
- package/lib/forms.js +81 -0
- package/lib/headers.js +22 -0
- package/lib/inventory.js +142 -0
- package/lib/ip.js +104 -0
- package/lib/maintenance.js +81 -0
- package/lib/next.js +251 -0
- package/lib/performance.js +120 -0
- package/lib/posture.js +226 -0
- package/lib/proxy.js +198 -0
- package/lib/requestRules.js +217 -0
- package/lib/resources.js +453 -0
- package/next.d.ts +36 -0
- package/package.json +41 -4
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// Request rules (virtual patching; protocol in sdk/protocols/firewall.md):
|
|
2
|
+
// the firewall has always matched addresses, these match the request — its
|
|
3
|
+
// method, path, query string and user agent — so an attack is turned away
|
|
4
|
+
// before the vulnerable code runs. The rules come with the others, from the
|
|
5
|
+
// collector, and are compiled once per fetch; matching is plain string work on
|
|
6
|
+
// text we already have, so it costs nothing per request.
|
|
7
|
+
//
|
|
8
|
+
// Shared by the Express/node:http middleware and the Next.js edge one, so
|
|
9
|
+
// both decide the same way: nothing here touches the filesystem or the
|
|
10
|
+
// network, which is what the edge runtime allows.
|
|
11
|
+
"use strict";
|
|
12
|
+
|
|
13
|
+
const MAX_RULES = 200;
|
|
14
|
+
// How much of each part a rule may look at. Patterns come from our rules, but
|
|
15
|
+
// the text comes from whoever is asking, and no regex engine we ship to can be
|
|
16
|
+
// timed out: a bounded slice keeps a bad pattern from being a hung request.
|
|
17
|
+
const MAX_TEXT = 2000;
|
|
18
|
+
const MAX_REGEX = 200;
|
|
19
|
+
|
|
20
|
+
const lower = (value) => String(value ?? "").slice(0, MAX_TEXT).toLowerCase();
|
|
21
|
+
// One pass only, as the protocol says: enough for %2e%2e, and no risk of
|
|
22
|
+
// decoding a visitor's own text into something that looks like an attack.
|
|
23
|
+
// Each run of escapes is decoded on its own, so a stray "%" elsewhere can't
|
|
24
|
+
// leave the whole text undecoded; a run that isn't valid UTF-8 still has its
|
|
25
|
+
// ASCII escapes (%2e, %2f) decoded.
|
|
26
|
+
function decodeRun(run) {
|
|
27
|
+
try {
|
|
28
|
+
return decodeURIComponent(run);
|
|
29
|
+
} catch {
|
|
30
|
+
return run.replace(/%([0-7][0-9a-f])/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function decodeOnce(text) {
|
|
34
|
+
if (!text.includes("%") && !text.includes("+")) return text;
|
|
35
|
+
return text.replace(/\+/g, " ").replace(/(?:%[0-9a-f]{2})+/gi, decodeRun);
|
|
36
|
+
}
|
|
37
|
+
const normalizePath = (path) => lower(path).replace(/\\/g, "/").replace(/\/{2,}/g, "/");
|
|
38
|
+
|
|
39
|
+
// A pattern can only come from us, but a careless one would be matched against
|
|
40
|
+
// text an attacker chooses, in this very request, and JavaScript cannot time a
|
|
41
|
+
// regex out. So the client checks the shape itself instead of trusting what it
|
|
42
|
+
// was handed: a repeated group that can match the same text more than one way
|
|
43
|
+
// takes exponential time on input that nearly matches. A group can do that when
|
|
44
|
+
// it holds a quantifier ("(a+)+", "(\w\d?)+") or an alternation ("(a|ab)*"),
|
|
45
|
+
// so a group followed by * + or { holds neither. Backreferences and lookbehind
|
|
46
|
+
// are refused for the same reason (and they aren't in every engine we ship to).
|
|
47
|
+
// The same check as backend/services/firewall/requestRules.js, freeRepeats
|
|
48
|
+
// included; sdk/protocols/request-rules-cases.json holds the shared lists.
|
|
49
|
+
function safePattern(pattern) {
|
|
50
|
+
if (typeof pattern !== "string" || !pattern || pattern.length > MAX_REGEX) return false;
|
|
51
|
+
if (/\\[1-9]/.test(pattern) || /\(\?</.test(pattern)) return false;
|
|
52
|
+
// Per open group: whether anything in it, nested groups too, repeats or branches.
|
|
53
|
+
const open = [];
|
|
54
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
55
|
+
const char = pattern[i];
|
|
56
|
+
if (char === "\\") {
|
|
57
|
+
i++;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (char === "[") {
|
|
61
|
+
// A character class: skip it whole, quantifiers inside it are literals.
|
|
62
|
+
while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (char === "(") {
|
|
66
|
+
open.push(false);
|
|
67
|
+
// "(?:", "(?=", "(?!": that ? names the kind of group, it repeats nothing.
|
|
68
|
+
if (pattern[i + 1] === "?") i += 2;
|
|
69
|
+
} else if (char === ")") {
|
|
70
|
+
if (!open.length) return false;
|
|
71
|
+
const ambiguous = open.pop();
|
|
72
|
+
const next = pattern[i + 1];
|
|
73
|
+
if (ambiguous && (next === "*" || next === "+" || next === "{")) return false;
|
|
74
|
+
if (ambiguous && open.length) open[open.length - 1] = true;
|
|
75
|
+
} else if (open.length && "|*+?{".includes(char)) {
|
|
76
|
+
open[open.length - 1] = true;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return open.length === 0 && freeRepeats(pattern) <= 1;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Two repeats that can both take the same text multiply each other's work:
|
|
83
|
+
// ".*=.*=.*x" against a query of "=" that never ends in "x" is polynomial,
|
|
84
|
+
// and the 2000 characters the clients match at most is plenty for that to
|
|
85
|
+
// hang a request. A repeat is pinned when the character straight after it is
|
|
86
|
+
// one it can't match ("[^/]+/" ends at the first "/", once); any other is
|
|
87
|
+
// free, and a pattern may hold one free repeat.
|
|
88
|
+
const literalAt = (pattern, i) => {
|
|
89
|
+
const char = pattern[i];
|
|
90
|
+
if (char === undefined) return null;
|
|
91
|
+
if (char === "\\") return pattern[i + 1] && !/[a-z0-9]/i.test(pattern[i + 1]) ? pattern[i + 1] : null;
|
|
92
|
+
return ".[](){}*+?|^$".includes(char) ? null : char;
|
|
93
|
+
};
|
|
94
|
+
function freeRepeats(pattern) {
|
|
95
|
+
let free = 0;
|
|
96
|
+
for (let i = 0; i < pattern.length;) {
|
|
97
|
+
const start = i;
|
|
98
|
+
const char = pattern[i];
|
|
99
|
+
if (char === "\\") i += 2;
|
|
100
|
+
else if (char === "[") {
|
|
101
|
+
i++;
|
|
102
|
+
while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1;
|
|
103
|
+
i++;
|
|
104
|
+
} else if (char === "(") {
|
|
105
|
+
i += pattern[i + 1] === "?" ? 3 : 1;
|
|
106
|
+
continue;
|
|
107
|
+
} else if (char === "|" || char === "^" || char === "$") {
|
|
108
|
+
i++;
|
|
109
|
+
continue;
|
|
110
|
+
} else i++;
|
|
111
|
+
// A group's text can't be tried against one character.
|
|
112
|
+
const atom = char === ")" ? null : pattern.slice(start, i);
|
|
113
|
+
const quantifier = pattern[i];
|
|
114
|
+
let repeats = quantifier === "*" || quantifier === "+";
|
|
115
|
+
if (quantifier === "{") {
|
|
116
|
+
const bounds = /^\{(\d*)(,(\d*))?\}/.exec(pattern.slice(i));
|
|
117
|
+
if (!bounds) continue;
|
|
118
|
+
// "{3}" is one length, not a choice; "{2,}" and "{2,9}" are choices.
|
|
119
|
+
repeats = bounds[2] !== undefined && bounds[3] !== bounds[1];
|
|
120
|
+
i += bounds[0].length - 1;
|
|
121
|
+
} else if (quantifier !== "?" && !repeats) continue;
|
|
122
|
+
i++;
|
|
123
|
+
if (pattern[i] === "?") i++;
|
|
124
|
+
if (!repeats) continue;
|
|
125
|
+
const next = literalAt(pattern, i);
|
|
126
|
+
let pinned = false;
|
|
127
|
+
if (atom && next !== null) {
|
|
128
|
+
try {
|
|
129
|
+
pinned = !new RegExp(`^(?:${atom})$`, "i").test(next);
|
|
130
|
+
} catch {
|
|
131
|
+
// not an atom on its own: free
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (!pinned) free++;
|
|
135
|
+
}
|
|
136
|
+
return free;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function compileSection(section) {
|
|
140
|
+
if (!section || typeof section !== "object") return null;
|
|
141
|
+
const out = {};
|
|
142
|
+
for (const matcher of ["exact", "prefix", "suffix", "contains"]) {
|
|
143
|
+
const list = Array.isArray(section[matcher]) ? section[matcher].filter((s) => typeof s === "string").map(lower) : [];
|
|
144
|
+
if (list.length) out[matcher] = list;
|
|
145
|
+
}
|
|
146
|
+
if (safePattern(section.regex)) {
|
|
147
|
+
try {
|
|
148
|
+
out.regex = new RegExp(section.regex, "i");
|
|
149
|
+
} catch {
|
|
150
|
+
// A pattern we can't compile is dropped; the rule's other parts stay.
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return Object.keys(out).length ? out : null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function compileRequestRules(data) {
|
|
157
|
+
const given = Array.isArray(data?.rules) ? data.rules.slice(0, MAX_RULES) : [];
|
|
158
|
+
const rules = [];
|
|
159
|
+
for (const rule of given) {
|
|
160
|
+
if (!rule || typeof rule.id !== "string" || !rule.id) continue;
|
|
161
|
+
// The middleware runs before the body is parsed, so we never see it: a
|
|
162
|
+
// rule that needs the body is skipped whole, not matched on the rest.
|
|
163
|
+
if (rule.body) continue;
|
|
164
|
+
const compiled = {
|
|
165
|
+
id: rule.id,
|
|
166
|
+
action: rule.action === "block" ? "block" : "monitor",
|
|
167
|
+
ban: Number(rule.ban) > 0 ? Number(rule.ban) : 0,
|
|
168
|
+
methods: Array.isArray(rule.methods) && rule.methods.length
|
|
169
|
+
? new Set(rule.methods.map((method) => String(method).toUpperCase())) : null,
|
|
170
|
+
path: compileSection(rule.path),
|
|
171
|
+
query: compileSection(rule.query),
|
|
172
|
+
ua: compileSection(rule.ua),
|
|
173
|
+
};
|
|
174
|
+
// A rule with nothing to match would match everything.
|
|
175
|
+
if (!compiled.path && !compiled.query && !compiled.ua) continue;
|
|
176
|
+
rules.push(compiled);
|
|
177
|
+
}
|
|
178
|
+
return rules;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Any one entry of the section, against the text as it came and decoded once.
|
|
182
|
+
function sectionHits(section, raw, decoded) {
|
|
183
|
+
const texts = decoded === raw ? [raw] : [raw, decoded];
|
|
184
|
+
for (const text of texts) {
|
|
185
|
+
if (section.exact?.includes(text)) return true;
|
|
186
|
+
if (section.prefix?.some((entry) => text.startsWith(entry))) return true;
|
|
187
|
+
if (section.suffix?.some((entry) => text.endsWith(entry))) return true;
|
|
188
|
+
if (section.contains?.some((entry) => text.includes(entry))) return true;
|
|
189
|
+
if (section.regex?.test(text)) return true;
|
|
190
|
+
}
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// The first rule that matches, or null. `path` is the full target as the
|
|
195
|
+
// server received it, query string and all.
|
|
196
|
+
function matchRequest(rules, { method = "", path = "", userAgent = "" } = {}) {
|
|
197
|
+
if (!rules || !rules.length) return null;
|
|
198
|
+
const target = String(path ?? "");
|
|
199
|
+
const mark = target.indexOf("?");
|
|
200
|
+
const rawPath = normalizePath(mark === -1 ? target : target.slice(0, mark));
|
|
201
|
+
const rawQuery = lower(mark === -1 ? "" : target.slice(mark + 1));
|
|
202
|
+
const ua = lower(userAgent);
|
|
203
|
+
const decodedPath = normalizePath(decodeOnce(mark === -1 ? target : target.slice(0, mark)));
|
|
204
|
+
// Lowered again after decoding: %55NION is UNION, and the patterns are lower case.
|
|
205
|
+
const decodedQuery = decodeOnce(rawQuery).toLowerCase();
|
|
206
|
+
const upper = String(method ?? "").toUpperCase();
|
|
207
|
+
for (const rule of rules) {
|
|
208
|
+
if (rule.methods && !rule.methods.has(upper)) continue;
|
|
209
|
+
if (rule.path && !sectionHits(rule.path, rawPath, decodedPath)) continue;
|
|
210
|
+
if (rule.query && !sectionHits(rule.query, rawQuery, decodedQuery)) continue;
|
|
211
|
+
if (rule.ua && !sectionHits(rule.ua, ua, ua)) continue;
|
|
212
|
+
return rule;
|
|
213
|
+
}
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
module.exports = { compileRequestRules, matchRequest, safePattern, MAX_RULES, MAX_TEXT };
|
package/lib/resources.js
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
// Resources module (protocol in sdk/protocols/resources.md): how the machine
|
|
2
|
+
// is doing — CPU, load, memory, swap, disks — sampled every minute and sent to
|
|
3
|
+
// POST /events/resources every few minutes. The companion of the performance
|
|
4
|
+
// module: that says how the requests went, this says whether the box can still
|
|
5
|
+
// serve them, so the dashboard shows what you would otherwise run
|
|
6
|
+
// node_exporter, Prometheus and Grafana for.
|
|
7
|
+
//
|
|
8
|
+
// Nothing is guessed. A reading this machine won't give us is left out of the
|
|
9
|
+
// row, never sent as a zero: the collector stores it as NULL and the dashboard
|
|
10
|
+
// shows "—", because "0% CPU" and "we couldn't measure CPU" are different
|
|
11
|
+
// statements and only one of them means healthy.
|
|
12
|
+
"use strict";
|
|
13
|
+
|
|
14
|
+
const fs = require("node:fs");
|
|
15
|
+
const os = require("node:os");
|
|
16
|
+
const path = require("node:path");
|
|
17
|
+
|
|
18
|
+
// Keep in step with the collector (expresscollect/Analytics/resources.js) and
|
|
19
|
+
// the agent (sdk/vps-agent).
|
|
20
|
+
const SAMPLE_MS = 60 * 1000;
|
|
21
|
+
// Below this, two CPU readings are too close together to divide: the counters
|
|
22
|
+
// move in ticks, so a percentage over a few milliseconds is quantisation noise
|
|
23
|
+
// (0% or 100%, nothing between). A sample taken that soon reports no CPU
|
|
24
|
+
// figure rather than a made-up one.
|
|
25
|
+
const MIN_CPU_MS = 1000;
|
|
26
|
+
const EVERY_S = 5 * 60;
|
|
27
|
+
const RETRY_S = 60;
|
|
28
|
+
// About 8 hours of samples: enough to ride out a collector outage, bounded so
|
|
29
|
+
// a long one can't grow in memory.
|
|
30
|
+
const MAX_ROWS = 500;
|
|
31
|
+
const MAX_DISKS = 20;
|
|
32
|
+
// Nobody watches a sub-gigabyte partition fill up; this drops the EFI
|
|
33
|
+
// partition and other small mounts without needing a list of paths.
|
|
34
|
+
const MIN_DISK_MB = 1024;
|
|
35
|
+
// Real on-disk filesystems only: no tmpfs, devtmpfs, proc, sysfs, cgroup or
|
|
36
|
+
// squashfs (snaps), none of which are the customer's disk. overlay is here so
|
|
37
|
+
// a containerised app still reports something sensible.
|
|
38
|
+
const FSTYPES = new Set(["ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "f2fs", "jfs",
|
|
39
|
+
"reiserfs", "ufs", "vfat", "exfat", "ntfs", "ntfs3", "overlay", "apfs", "hfs"]);
|
|
40
|
+
|
|
41
|
+
const MB = 1024 * 1024;
|
|
42
|
+
const read = (file) => {
|
|
43
|
+
try {
|
|
44
|
+
return fs.readFileSync(file, "utf8");
|
|
45
|
+
} catch {
|
|
46
|
+
return "";
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
// (busy, total) across every core. idle is the machine doing nothing;
|
|
51
|
+
// everything else, including time stolen by the hypervisor, is time it wasn't
|
|
52
|
+
// available to us. os.cpus() rather than /proc/stat, so this works everywhere.
|
|
53
|
+
function cpuTimes(cpus = os.cpus()) {
|
|
54
|
+
let busy = 0;
|
|
55
|
+
let total = 0;
|
|
56
|
+
for (const cpu of cpus || []) {
|
|
57
|
+
const t = cpu?.times || {};
|
|
58
|
+
const idle = Number(t.idle) || 0;
|
|
59
|
+
const sum = idle + (Number(t.user) || 0) + (Number(t.nice) || 0) + (Number(t.sys) || 0) + (Number(t.irq) || 0);
|
|
60
|
+
busy += sum - idle;
|
|
61
|
+
total += sum;
|
|
62
|
+
}
|
|
63
|
+
return total ? { busy, total } : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// How busy the machine was between two readings, or null when the pair can't
|
|
67
|
+
// say: the first sample, a counter reset, or no time passed.
|
|
68
|
+
function cpuPercent(before, after) {
|
|
69
|
+
if (!before || !after) return null;
|
|
70
|
+
const busy = after.busy - before.busy;
|
|
71
|
+
const total = after.total - before.total;
|
|
72
|
+
if (total <= 0 || busy < 0) return null;
|
|
73
|
+
return Math.round(Math.min(100, (100 * busy) / total) * 10) / 10;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// MemTotal/MemAvailable/Swap from /proc/meminfo, in whole megabytes.
|
|
77
|
+
//
|
|
78
|
+
// Used is total - MemAvailable, never total - MemFree. Page cache is not
|
|
79
|
+
// pressure, and counting it as used makes every healthy Linux box look full —
|
|
80
|
+
// which is also why os.freemem() is not used here: on Linux it is MemFree.
|
|
81
|
+
function memoryMb(text) {
|
|
82
|
+
const values = {};
|
|
83
|
+
for (const line of String(text).split("\n")) {
|
|
84
|
+
const [name, rest] = line.split(":");
|
|
85
|
+
const kb = Number(String(rest || "").trim().split(/\s+/)[0]);
|
|
86
|
+
if (name && Number.isFinite(kb)) values[name] = Math.floor(kb / 1024);
|
|
87
|
+
}
|
|
88
|
+
const out = {};
|
|
89
|
+
if (values.MemTotal) {
|
|
90
|
+
out.mem_total_mb = values.MemTotal;
|
|
91
|
+
if (values.MemAvailable !== undefined) out.mem_used_mb = Math.max(0, values.MemTotal - values.MemAvailable);
|
|
92
|
+
}
|
|
93
|
+
if (values.SwapTotal !== undefined) {
|
|
94
|
+
out.swap_total_mb = values.SwapTotal;
|
|
95
|
+
if (values.SwapFree !== undefined) out.swap_used_mb = Math.max(0, values.SwapTotal - values.SwapFree);
|
|
96
|
+
}
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Where this machine keeps its filesystems. On Linux that's /proc/mounts; off
|
|
101
|
+
// it, the one filesystem the application is running from, which is the only
|
|
102
|
+
// one we can name without guessing.
|
|
103
|
+
// Whether a mount point is a directory, never throwing: a path this process
|
|
104
|
+
// cannot stat is simply unconfirmed, and mountPoints falls back to the
|
|
105
|
+
// device's shortest mount rather than dropping the disk.
|
|
106
|
+
function defaultIsDir(point) {
|
|
107
|
+
try {
|
|
108
|
+
return fs.statSync(point).isDirectory();
|
|
109
|
+
} catch {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// A mount point has to be a directory. Docker bind-mounts single files, so a
|
|
115
|
+
// container lists /etc/hosts, /etc/hostname and /etc/resolv.conf on the host's
|
|
116
|
+
// own device, and those paths are short enough to win the shortest-path race
|
|
117
|
+
// against the real mount. The test is per-device and advisory: a device whose
|
|
118
|
+
// mounts cannot be confirmed keeps its shortest one, because reporting no
|
|
119
|
+
// disks is worse than reporting an odd mount point.
|
|
120
|
+
function mountPoints(mountsText, platform = process.platform, isDir = defaultIsDir) {
|
|
121
|
+
if (platform !== "linux") return [path.parse(process.cwd()).root || "/"];
|
|
122
|
+
const dirs = new Map();
|
|
123
|
+
const any = new Map();
|
|
124
|
+
for (const line of String(mountsText).split("\n")) {
|
|
125
|
+
const [device, mount, fstype] = line.split(/\s+/);
|
|
126
|
+
if (!device || !mount || !FSTYPES.has(fstype) || !mount.startsWith("/")) continue;
|
|
127
|
+
const point = mount.replace(/\\040/g, " ");
|
|
128
|
+
// The shortest path wins: "/" over "/var" for one device, so a bind mount
|
|
129
|
+
// isn't reported as a second disk.
|
|
130
|
+
if (!any.has(device) || point.length < any.get(device).length) any.set(device, point);
|
|
131
|
+
if (isDir(point) && (!dirs.has(device) || point.length < dirs.get(device).length)) dirs.set(device, point);
|
|
132
|
+
}
|
|
133
|
+
const points = [...any].map(([device, point]) => dirs.get(device) ?? point);
|
|
134
|
+
return [...new Set(points)].sort();
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Both figures are df's, so a customer can check them: used is total minus
|
|
138
|
+
// free, and total is used plus what's available to us. The blocks reserved for
|
|
139
|
+
// root are in neither, which is what makes used/total the percentage df prints
|
|
140
|
+
// — and that percentage is what the disk alert is about.
|
|
141
|
+
function diskRows(points, statfs = fs.statfsSync) {
|
|
142
|
+
if (typeof statfs !== "function") return [];
|
|
143
|
+
// One filesystem can sit behind two devices the kernel calls different: in a
|
|
144
|
+
// container / is the overlay and a bind mount exposes the host disk the
|
|
145
|
+
// overlay's upperdir lives on, so both report the same size and the same
|
|
146
|
+
// free space down to the byte. No device-based rule separates them, so the
|
|
147
|
+
// sizes are what gives it away. Raw bytes, not rounded megabytes, so disks
|
|
148
|
+
// that merely round alike stay apart; two separate but identically sized and
|
|
149
|
+
// identically empty filesystems do collapse into one until something is
|
|
150
|
+
// written to either, and being empty they owe the fullness alert nothing.
|
|
151
|
+
const seen = new Set();
|
|
152
|
+
const rows = [];
|
|
153
|
+
for (const mount of points) {
|
|
154
|
+
let st;
|
|
155
|
+
try {
|
|
156
|
+
st = statfs(mount);
|
|
157
|
+
} catch {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const block = Number(st.bsize) || 0;
|
|
161
|
+
const blocks = Number(st.blocks) || 0;
|
|
162
|
+
if (!block || !blocks) continue;
|
|
163
|
+
const used = Math.max(0, blocks - Number(st.bfree || 0)) * block;
|
|
164
|
+
const total = used + Number(st.bavail || 0) * block;
|
|
165
|
+
const row = { mount, total_mb: Math.floor(total / MB), used_mb: Math.floor(used / MB) };
|
|
166
|
+
if (row.total_mb < MIN_DISK_MB) continue;
|
|
167
|
+
// points arrives sorted, so the shortest path wins the collision.
|
|
168
|
+
const key = `${total}:${used}`;
|
|
169
|
+
if (seen.has(key)) continue;
|
|
170
|
+
seen.add(key);
|
|
171
|
+
// A filesystem can refuse a write with gigabytes free once these meet.
|
|
172
|
+
const nodes = Number(st.files) || 0;
|
|
173
|
+
if (nodes) {
|
|
174
|
+
row.inodes_total = nodes;
|
|
175
|
+
row.inodes_used = Math.max(0, nodes - (Number(st.ffree) || 0));
|
|
176
|
+
}
|
|
177
|
+
rows.push(row);
|
|
178
|
+
if (rows.length >= MAX_DISKS) break;
|
|
179
|
+
}
|
|
180
|
+
return rows;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Sectors in /proc/diskstats are always 512 bytes: it is the unit of that
|
|
184
|
+
// interface, not of the drive.
|
|
185
|
+
const IO_SECTOR = 512;
|
|
186
|
+
// Two readings further apart than this say nothing about this minute.
|
|
187
|
+
const IO_MAX_PAIR_MS = 300 * 1000;
|
|
188
|
+
// Not disks: loop and ram devices, optical drives, device-mapper and md, each
|
|
189
|
+
// of which would count the devices underneath it a second time.
|
|
190
|
+
const NOT_A_DISK = /^(?:loop|ram|zram|sr|fd|dm-|md\d)/;
|
|
191
|
+
// Partitions, which must not be counted alongside the disk they sit on.
|
|
192
|
+
const PARTITION = /^(?:(?:sd|vd|xvd|hd)[a-z]+\d+|(?:nvme\d+n\d+|mmcblk\d+)p\d+)$/;
|
|
193
|
+
// Loopback, and the virtual ends of container and VM networking: counting
|
|
194
|
+
// these would count someone else's traffic, or ours twice.
|
|
195
|
+
const NOT_REAL_NET = /^(?:lo|veth|docker|br-|virbr|tap|tun|vnet)/;
|
|
196
|
+
|
|
197
|
+
// The jiffie counters from /proc/stat's first line, or null. In order: user,
|
|
198
|
+
// nice, system, idle, iowait, irq, softirq, steal. Linux only — os.cpus()
|
|
199
|
+
// carries no iowait and no steal, which are the two worth having.
|
|
200
|
+
function cpuFields(text) {
|
|
201
|
+
for (const line of String(text).split("\n")) {
|
|
202
|
+
if (!line.startsWith("cpu ")) continue;
|
|
203
|
+
const fields = line.trim().split(/\s+/).slice(1).map(Number);
|
|
204
|
+
if (fields.length < 4 || fields.some((value) => !Number.isFinite(value))) return null;
|
|
205
|
+
return fields;
|
|
206
|
+
}
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// user folds nice in, as top adds them together. Each is a share of the whole
|
|
211
|
+
// interval, so with idle and the interrupt buckets they come to 100.
|
|
212
|
+
const CPU_SHARES = [
|
|
213
|
+
["cpu_user_pct", [0, 1]],
|
|
214
|
+
["cpu_system_pct", [2]],
|
|
215
|
+
["cpu_iowait_pct", [4]],
|
|
216
|
+
["cpu_steal_pct", [7]],
|
|
217
|
+
];
|
|
218
|
+
|
|
219
|
+
// Where the machine's time went between two readings. A column the kernel is
|
|
220
|
+
// too old to carry is left out rather than sent as a zero. steal is the one
|
|
221
|
+
// worth surfacing: time the hypervisor gave to another tenant, which no amount
|
|
222
|
+
// of tuning the app will change.
|
|
223
|
+
function cpuShares(before, after) {
|
|
224
|
+
if (!before || !after) return {};
|
|
225
|
+
const total = after.reduce((sum, v) => sum + v, 0) - before.reduce((sum, v) => sum + v, 0);
|
|
226
|
+
if (total <= 0) return {};
|
|
227
|
+
const out = {};
|
|
228
|
+
for (const [name, columns] of CPU_SHARES) {
|
|
229
|
+
if (columns.some((i) => i >= before.length || i >= after.length)) continue;
|
|
230
|
+
const moved = columns.reduce((sum, i) => sum + (after[i] - before[i]), 0);
|
|
231
|
+
if (moved < 0) return {};
|
|
232
|
+
out[name] = Math.round(Math.min(100, (100 * moved) / total) * 10) / 10;
|
|
233
|
+
}
|
|
234
|
+
return out;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Bytes read and written since boot over the real disks, or null.
|
|
238
|
+
function diskCounters(text) {
|
|
239
|
+
let read_ = 0;
|
|
240
|
+
let written = 0;
|
|
241
|
+
let found = false;
|
|
242
|
+
for (const line of String(text).split("\n")) {
|
|
243
|
+
const fields = line.trim().split(/\s+/);
|
|
244
|
+
if (fields.length < 10) continue;
|
|
245
|
+
const name = fields[2];
|
|
246
|
+
if (!name || NOT_A_DISK.test(name) || PARTITION.test(name)) continue;
|
|
247
|
+
const sectorsRead = Number(fields[5]);
|
|
248
|
+
const sectorsWritten = Number(fields[9]);
|
|
249
|
+
if (!Number.isFinite(sectorsRead) || !Number.isFinite(sectorsWritten)) continue;
|
|
250
|
+
read_ += sectorsRead * IO_SECTOR;
|
|
251
|
+
written += sectorsWritten * IO_SECTOR;
|
|
252
|
+
found = true;
|
|
253
|
+
}
|
|
254
|
+
return found ? [read_, written] : null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Bytes in and out since boot over the real interfaces, or null.
|
|
258
|
+
function netCounters(text) {
|
|
259
|
+
let rx = 0;
|
|
260
|
+
let tx = 0;
|
|
261
|
+
let found = false;
|
|
262
|
+
for (const line of String(text).split("\n")) {
|
|
263
|
+
const at = line.indexOf(":");
|
|
264
|
+
if (at < 0) continue;
|
|
265
|
+
const name = line.slice(0, at).trim();
|
|
266
|
+
if (!name || NOT_REAL_NET.test(name)) continue;
|
|
267
|
+
const fields = line.slice(at + 1).trim().split(/\s+/).map(Number);
|
|
268
|
+
if (fields.length < 9) continue;
|
|
269
|
+
if (!Number.isFinite(fields[0]) || !Number.isFinite(fields[8])) continue;
|
|
270
|
+
rx += fields[0];
|
|
271
|
+
tx += fields[8];
|
|
272
|
+
found = true;
|
|
273
|
+
}
|
|
274
|
+
return found ? [rx, tx] : null;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Megabytes a second between two byte counters, or null when the pair can't
|
|
278
|
+
// say: no pair, a reboot that reset them, no time, or too long a gap.
|
|
279
|
+
function rateMbs(before, after, ms) {
|
|
280
|
+
if (before === null || before === undefined || after === null || after === undefined) return null;
|
|
281
|
+
if (!(ms > 0) || ms > IO_MAX_PAIR_MS) return null;
|
|
282
|
+
const moved = after - before;
|
|
283
|
+
if (moved < 0) return null;
|
|
284
|
+
return Math.round((moved / (ms / 1000) / MB) * 10) / 10;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function createResources(core, options = {}) {
|
|
288
|
+
const log = (...args) => core.log("resources", ...args);
|
|
289
|
+
const serverName = String(options.serverName ?? os.hostname()).slice(0, 100);
|
|
290
|
+
const runtime = `Node ${process.versions.node}`;
|
|
291
|
+
// minute (ISO) -> row, so a minute sampled twice is kept once.
|
|
292
|
+
const rows = new Map();
|
|
293
|
+
let lastCpu = cpuTimes();
|
|
294
|
+
let lastFields = cpuFields(read("/proc/stat"));
|
|
295
|
+
let lastDisk = diskCounters(read("/proc/diskstats"));
|
|
296
|
+
let lastNet = netCounters(read("/proc/net/dev"));
|
|
297
|
+
let lastAt = Date.now();
|
|
298
|
+
let every = EVERY_S;
|
|
299
|
+
let sampleTimer = null;
|
|
300
|
+
let postTimer = null;
|
|
301
|
+
let closed = false;
|
|
302
|
+
let loop = null;
|
|
303
|
+
// The event loop's own delay: how long the application itself was blocked,
|
|
304
|
+
// which no outside measurement of the machine can see.
|
|
305
|
+
try {
|
|
306
|
+
const { monitorEventLoopDelay } = require("node:perf_hooks");
|
|
307
|
+
if (typeof monitorEventLoopDelay === "function") {
|
|
308
|
+
loop = monitorEventLoopDelay({ resolution: 20 });
|
|
309
|
+
loop.enable();
|
|
310
|
+
}
|
|
311
|
+
} catch {
|
|
312
|
+
loop = null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// One sample, covering the time since the last one.
|
|
316
|
+
function sample(at = Date.now()) {
|
|
317
|
+
const cpu = cpuTimes();
|
|
318
|
+
const percent = cpuPercent(lastCpu, cpu);
|
|
319
|
+
const elapsed = at - lastAt;
|
|
320
|
+
// Everything below needs two readings, like cpu_pct does.
|
|
321
|
+
const fields = cpuFields(read("/proc/stat"));
|
|
322
|
+
const disk = diskCounters(read("/proc/diskstats"));
|
|
323
|
+
const net = netCounters(read("/proc/net/dev"));
|
|
324
|
+
// The minute the interval started in, not the one it ended in.
|
|
325
|
+
const row = { minute: new Date(Math.floor(lastAt / 60000) * 60000).toISOString() };
|
|
326
|
+
lastCpu = cpu;
|
|
327
|
+
lastAt = at;
|
|
328
|
+
if (percent !== null && elapsed >= MIN_CPU_MS) row.cpu_pct = percent;
|
|
329
|
+
const cores = os.cpus()?.length;
|
|
330
|
+
if (cores) row.cpu_cores = cores;
|
|
331
|
+
const load = os.loadavg?.() || [];
|
|
332
|
+
// Windows has no load average and reports zeroes; sending them would be a
|
|
333
|
+
// measurement we never took.
|
|
334
|
+
if (load.length === 3 && load.some((value) => value > 0)) {
|
|
335
|
+
row.load1 = Math.round(load[0] * 100) / 100;
|
|
336
|
+
row.load5 = Math.round(load[1] * 100) / 100;
|
|
337
|
+
row.load15 = Math.round(load[2] * 100) / 100;
|
|
338
|
+
}
|
|
339
|
+
const memory = memoryMb(read("/proc/meminfo"));
|
|
340
|
+
if (memory.mem_total_mb) {
|
|
341
|
+
Object.assign(row, memory);
|
|
342
|
+
} else if (os.totalmem()) {
|
|
343
|
+
// Off Linux: os.freemem() is the best this platform offers.
|
|
344
|
+
row.mem_total_mb = Math.floor(os.totalmem() / MB);
|
|
345
|
+
row.mem_used_mb = Math.max(0, Math.floor((os.totalmem() - os.freemem()) / MB));
|
|
346
|
+
}
|
|
347
|
+
const disks = diskRows(mountPoints(read("/proc/mounts")));
|
|
348
|
+
if (disks.length) row.disks = disks;
|
|
349
|
+
Object.assign(row, cpuShares(lastFields, fields));
|
|
350
|
+
const rates = {
|
|
351
|
+
disk_read_mbs: lastDisk && disk ? rateMbs(lastDisk[0], disk[0], elapsed) : null,
|
|
352
|
+
disk_write_mbs: lastDisk && disk ? rateMbs(lastDisk[1], disk[1], elapsed) : null,
|
|
353
|
+
net_rx_mbs: lastNet && net ? rateMbs(lastNet[0], net[0], elapsed) : null,
|
|
354
|
+
net_tx_mbs: lastNet && net ? rateMbs(lastNet[1], net[1], elapsed) : null,
|
|
355
|
+
};
|
|
356
|
+
for (const [name, value] of Object.entries(rates)) {
|
|
357
|
+
if (value !== null) row[name] = value;
|
|
358
|
+
}
|
|
359
|
+
lastFields = fields;
|
|
360
|
+
lastDisk = disk;
|
|
361
|
+
lastNet = net;
|
|
362
|
+
const rss = process.memoryUsage?.().rss;
|
|
363
|
+
if (rss) row.proc_rss_mb = Math.max(1, Math.floor(rss / MB));
|
|
364
|
+
if (loop) {
|
|
365
|
+
row.loop_lag_ms = Math.round((loop.mean / 1e6) * 100) / 100;
|
|
366
|
+
loop.reset();
|
|
367
|
+
}
|
|
368
|
+
const up = os.uptime?.();
|
|
369
|
+
if (Number.isFinite(up) && up > 0) row.host_uptime_s = Math.floor(up);
|
|
370
|
+
rows.set(row.minute, row);
|
|
371
|
+
// Oldest first out: a long outage keeps the recent minutes.
|
|
372
|
+
while (rows.size > MAX_ROWS) rows.delete(rows.keys().next().value);
|
|
373
|
+
return row;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Sends what's buffered. Rows are kept only when the collector couldn't take
|
|
377
|
+
// them and retrying may work.
|
|
378
|
+
async function flush() {
|
|
379
|
+
if (!rows.size) return true;
|
|
380
|
+
const batch = [...rows.keys()].sort();
|
|
381
|
+
const body = { site_id: core.siteId, server_name: serverName, runtime, rows: batch.map((minute) => rows.get(minute)) };
|
|
382
|
+
const res = await core.send("resources", "/events/resources", { auth: "bearer", body });
|
|
383
|
+
if (!res) return false;
|
|
384
|
+
if (res.ok) {
|
|
385
|
+
const data = await res.json().catch(() => null);
|
|
386
|
+
if (Number(data?.every) > 0) every = Math.max(RETRY_S, Number(data.every));
|
|
387
|
+
if (data?.enabled === false) log("server resources are off in the dashboard (Security → Server package)");
|
|
388
|
+
for (const minute of batch) rows.delete(minute);
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
// Unreachable or storage down: these rows are still good. A 4xx means
|
|
392
|
+
// resending won't make them acceptable.
|
|
393
|
+
if (res.status >= 500) return false;
|
|
394
|
+
log("collector answered %s to the resources report", res.status);
|
|
395
|
+
for (const minute of batch) rows.delete(minute);
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
async function report() {
|
|
400
|
+
let wait = every;
|
|
401
|
+
try {
|
|
402
|
+
if (core.isOn("resources")) {
|
|
403
|
+
if (!(await flush())) wait = RETRY_S;
|
|
404
|
+
} else {
|
|
405
|
+
// Switched off: keep nothing, and ask again later.
|
|
406
|
+
rows.clear();
|
|
407
|
+
wait = Math.max(600, every);
|
|
408
|
+
}
|
|
409
|
+
} catch (error) {
|
|
410
|
+
log("could not send the resources report:", error?.message);
|
|
411
|
+
wait = RETRY_S;
|
|
412
|
+
}
|
|
413
|
+
schedule(wait * 1000);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function schedule(ms) {
|
|
417
|
+
if (closed) return;
|
|
418
|
+
postTimer = setTimeout(report, ms);
|
|
419
|
+
postTimer.unref?.();
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
if (core.valid) {
|
|
423
|
+
sampleTimer = setInterval(() => {
|
|
424
|
+
try {
|
|
425
|
+
sample();
|
|
426
|
+
} catch (error) {
|
|
427
|
+
log("could not sample the machine:", error?.message);
|
|
428
|
+
}
|
|
429
|
+
}, SAMPLE_MS);
|
|
430
|
+
sampleTimer.unref?.();
|
|
431
|
+
schedule(EVERY_S * 1000);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
return {
|
|
435
|
+
// One sample now (for your own checks and tests).
|
|
436
|
+
sample,
|
|
437
|
+
// Send what's buffered now.
|
|
438
|
+
flush,
|
|
439
|
+
report,
|
|
440
|
+
close() {
|
|
441
|
+
closed = true;
|
|
442
|
+
clearInterval(sampleTimer);
|
|
443
|
+
clearTimeout(postTimer);
|
|
444
|
+
loop?.disable?.();
|
|
445
|
+
return flush().catch(() => false);
|
|
446
|
+
},
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
module.exports = {
|
|
451
|
+
createResources, cpuTimes, cpuPercent, memoryMb, mountPoints, diskRows, MIN_DISK_MB,
|
|
452
|
+
cpuFields, cpuShares, diskCounters, netCounters, rateMbs,
|
|
453
|
+
};
|