@blamejs/core 0.7.18 → 0.7.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +427 -423
- package/README.md +150 -150
- package/bin/blamejs.js +0 -0
- package/index.js +310 -308
- package/lib/api-key.js +660 -660
- package/lib/api-snapshot.js +338 -338
- package/lib/app-shutdown.js +385 -385
- package/lib/app.js +365 -365
- package/lib/archive.js +250 -250
- package/lib/atomic-file.js +544 -544
- package/lib/audit-chain.js +177 -177
- package/lib/audit-sign.js +344 -344
- package/lib/audit-tools.js +677 -677
- package/lib/audit.js +766 -766
- package/lib/auth/jwt-external.js +365 -0
- package/lib/auth/jwt.js +337 -311
- package/lib/auth/lockout.js +436 -436
- package/lib/auth/oauth.js +721 -721
- package/lib/auth/passkey.js +181 -181
- package/lib/auth/password.js +628 -594
- package/lib/backup/bundle.js +217 -217
- package/lib/backup/crypto.js +176 -176
- package/lib/backup/index.js +515 -515
- package/lib/backup/manifest.js +282 -282
- package/lib/break-glass.js +1338 -1338
- package/lib/bundler.js +441 -441
- package/lib/cache-redis.js +256 -256
- package/lib/cache.js +1206 -1206
- package/lib/canonical-json.js +115 -115
- package/lib/chain-writer.js +234 -234
- package/lib/cli-helpers.js +206 -206
- package/lib/cli.js +2334 -2334
- package/lib/cluster-provider-db.js +317 -317
- package/lib/cluster-storage.js +226 -226
- package/lib/cluster.js +703 -703
- package/lib/config-drift.js +301 -301
- package/lib/consent.js +222 -222
- package/lib/constants.js +191 -191
- package/lib/cookies.js +350 -315
- package/lib/credential-hash.js +322 -322
- package/lib/crypto.js +266 -266
- package/lib/csv.js +275 -275
- package/lib/db-declare-row-policy.js +267 -267
- package/lib/db-declare-view.js +420 -420
- package/lib/db-query.js +406 -406
- package/lib/db-schema.js +319 -319
- package/lib/db.js +1288 -1288
- package/lib/deprecate.js +222 -222
- package/lib/dev.js +335 -335
- package/lib/dual-control.js +473 -473
- package/lib/error-page.js +420 -420
- package/lib/external-db-migrate.js +441 -441
- package/lib/external-db.js +1061 -1061
- package/lib/file-type.js +273 -273
- package/lib/forms.js +422 -422
- package/lib/framework-error.js +293 -293
- package/lib/framework-schema.js +717 -717
- package/lib/handlers.js +350 -350
- package/lib/http-client-cookie-jar.js +508 -508
- package/lib/http-client.js +1195 -1195
- package/lib/i18n.js +878 -878
- package/lib/jobs.js +185 -185
- package/lib/log-stream-cloudwatch.js +369 -369
- package/lib/log-stream-local.js +146 -146
- package/lib/log-stream-otlp-grpc.js +410 -410
- package/lib/log-stream-otlp.js +286 -286
- package/lib/log-stream-syslog.js +302 -302
- package/lib/log-stream-webhook.js +199 -199
- package/lib/log-stream.js +330 -330
- package/lib/log.js +500 -500
- package/lib/mail-bounce.js +528 -528
- package/lib/mail-dkim.js +369 -369
- package/lib/mail.js +981 -981
- package/lib/metrics.js +683 -683
- package/lib/middleware/api-encrypt.js +936 -936
- package/lib/middleware/attach-user.js +157 -157
- package/lib/middleware/bearer-auth.js +152 -0
- package/lib/middleware/body-parser.js +1170 -1170
- package/lib/middleware/bot-guard.js +178 -178
- package/lib/middleware/compression.js +452 -452
- package/lib/middleware/cors.js +314 -314
- package/lib/middleware/csp-nonce.js +348 -348
- package/lib/middleware/csrf-protect.js +399 -316
- package/lib/middleware/db-role-for.js +264 -264
- package/lib/middleware/fetch-metadata.js +129 -0
- package/lib/middleware/health.js +392 -392
- package/lib/middleware/index.js +85 -79
- package/lib/middleware/rate-limit.js +358 -358
- package/lib/middleware/request-id.js +61 -61
- package/lib/middleware/request-log.js +168 -168
- package/lib/middleware/require-auth.js +104 -104
- package/lib/middleware/security-headers.js +121 -116
- package/lib/middleware/sse.js +166 -166
- package/lib/migrations.js +383 -383
- package/lib/mtls-ca.js +518 -518
- package/lib/mtls-engine-default.js +481 -481
- package/lib/network-dns.js +632 -632
- package/lib/network-heartbeat.js +290 -290
- package/lib/network-nts.js +574 -574
- package/lib/network-proxy.js +265 -265
- package/lib/network-tls.js +328 -328
- package/lib/network.js +233 -233
- package/lib/notify.js +612 -612
- package/lib/ntp-check.js +229 -229
- package/lib/numeric-bounds.js +111 -111
- package/lib/object-store/azure-blob-bucket-ops.js +349 -349
- package/lib/object-store/azure-blob.js +488 -488
- package/lib/object-store/gcs-bucket-ops.js +351 -351
- package/lib/object-store/gcs.js +519 -519
- package/lib/object-store/http-put.js +153 -153
- package/lib/object-store/index.js +197 -197
- package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
- package/lib/object-store/sigv4.js +903 -903
- package/lib/observability.js +151 -151
- package/lib/otel-export.js +269 -269
- package/lib/pagination.js +464 -464
- package/lib/parsers/index.js +80 -80
- package/lib/parsers/safe-env.js +642 -642
- package/lib/parsers/safe-ini.js +292 -292
- package/lib/parsers/safe-toml.js +784 -784
- package/lib/parsers/safe-xml.js +390 -390
- package/lib/parsers/safe-yaml.js +1015 -1015
- package/lib/permissions.js +708 -708
- package/lib/pqc-agent.js +87 -87
- package/lib/pqc-gate.js +279 -279
- package/lib/protobuf-encoder.js +190 -190
- package/lib/protocol-dispatcher.js +161 -161
- package/lib/pubsub-redis.js +167 -167
- package/lib/pubsub.js +429 -429
- package/lib/queue-local.js +476 -476
- package/lib/queue-redis.js +745 -745
- package/lib/queue-sqs.js +319 -319
- package/lib/queue.js +695 -695
- package/lib/redis-client.js +519 -519
- package/lib/request-helpers.js +340 -340
- package/lib/restore-bundle.js +237 -237
- package/lib/restore-rollback.js +259 -259
- package/lib/restore.js +409 -409
- package/lib/retry.js +376 -376
- package/lib/router.js +748 -748
- package/lib/safe-async.js +735 -735
- package/lib/safe-buffer.js +237 -237
- package/lib/safe-json.js +541 -541
- package/lib/safe-schema.js +1266 -1266
- package/lib/safe-url.js +159 -159
- package/lib/scheduler.js +706 -706
- package/lib/security-assert.js +373 -373
- package/lib/seeders.js +618 -618
- package/lib/session.js +535 -478
- package/lib/slug.js +269 -269
- package/lib/ssrf-guard.js +401 -401
- package/lib/static.js +7 -5
- package/lib/storage.js +471 -471
- package/lib/subject.js +281 -281
- package/lib/template.js +791 -791
- package/lib/testing.js +798 -798
- package/lib/time.js +310 -310
- package/lib/totp.js +302 -302
- package/lib/tracing.js +494 -494
- package/lib/uuid.js +132 -132
- package/lib/validate-opts.js +340 -340
- package/lib/vault/index.js +308 -308
- package/lib/vault/rotate.js +784 -784
- package/lib/vault/wrap.js +296 -296
- package/lib/vendor/noble-ciphers.cjs +9 -9
- package/lib/webhook.js +595 -595
- package/lib/websocket.js +1048 -1048
- package/package.json +77 -77
- package/sbom.cyclonedx.json +7 -7
package/lib/metrics.js
CHANGED
|
@@ -1,683 +1,683 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* metrics — Prometheus-format counters, gauges, and histograms.
|
|
4
|
-
*
|
|
5
|
-
* Production deployments need numbers. Without a metrics layer ops
|
|
6
|
-
* teams are half-blind on every incident: "Was that p99 spike real?
|
|
7
|
-
* What's the queue depth? How fast are audit emits going?". This
|
|
8
|
-
* module ships the standard Prometheus types with framework
|
|
9
|
-
* auto-instrumentation already wired into audit / vault / queue
|
|
10
|
-
* hot paths so operators get the framework's vital signs for free.
|
|
11
|
-
*
|
|
12
|
-
* Public API:
|
|
13
|
-
*
|
|
14
|
-
* var m = b.metrics.create({
|
|
15
|
-
* namespace: "myapp", // prepended to every metric name
|
|
16
|
-
* defaultLabels: { service: "api", version: "1.2.3" },
|
|
17
|
-
* labelCardinalityCap: 10000, // per-metric ceiling
|
|
18
|
-
* });
|
|
19
|
-
*
|
|
20
|
-
* var requests = m.counter("http_requests_total", {
|
|
21
|
-
* help: "Total HTTP requests",
|
|
22
|
-
* labelNames: ["method", "route", "status"],
|
|
23
|
-
* });
|
|
24
|
-
* requests.inc({ method: "GET", route: "/users", status: "200" });
|
|
25
|
-
* requests.inc({ method: "GET", route: "/users", status: "200" }, 5);
|
|
26
|
-
*
|
|
27
|
-
* var latency = m.histogram("http_request_duration_seconds", {
|
|
28
|
-
* help: "HTTP request latency",
|
|
29
|
-
* labelNames: ["method", "route"],
|
|
30
|
-
* buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
|
|
31
|
-
* });
|
|
32
|
-
* latency.observe({ method: "GET", route: "/users" }, 0.123);
|
|
33
|
-
*
|
|
34
|
-
* var queueDepth = m.gauge("queue_depth", { labelNames: ["queueName"] });
|
|
35
|
-
* queueDepth.set({ queueName: "default" }, 42);
|
|
36
|
-
* queueDepth.inc({ queueName: "default" });
|
|
37
|
-
* queueDepth.dec({ queueName: "default" });
|
|
38
|
-
*
|
|
39
|
-
* router.use(m.requestMiddleware()); // auto-times every request
|
|
40
|
-
* router.get("/metrics", m.expositionHandler());
|
|
41
|
-
*
|
|
42
|
-
* Framework auto-instrumentation:
|
|
43
|
-
* When metrics.create() runs, framework hot paths (audit.record,
|
|
44
|
-
* vault.seal, vault.unseal, queue ops) call metrics.tap() — a
|
|
45
|
-
* global no-op stub that the active registry replaces with real
|
|
46
|
-
* counters. Modules don't import the registry directly; the tap
|
|
47
|
-
* pattern keeps them decoupled and lets operators with no metrics
|
|
48
|
-
* pay zero cost.
|
|
49
|
-
*
|
|
50
|
-
* Built-in metrics surfaced:
|
|
51
|
-
* framework_audit_events_total{action, outcome} counter
|
|
52
|
-
* framework_vault_seal_total counter
|
|
53
|
-
* framework_vault_unseal_total counter
|
|
54
|
-
* framework_queue_enqueue_total{queueName} counter
|
|
55
|
-
* framework_queue_complete_total{queueName} counter
|
|
56
|
-
* framework_queue_fail_total{queueName} counter
|
|
57
|
-
* framework_queue_depth{queueName} gauge
|
|
58
|
-
* framework_jobs_inflight{queueName} gauge
|
|
59
|
-
* framework_errors_total{class} counter
|
|
60
|
-
* framework_http_requests_total{method,route,status} counter
|
|
61
|
-
* framework_http_request_duration_seconds{method,route} histogram
|
|
62
|
-
*
|
|
63
|
-
* Cardinality control:
|
|
64
|
-
* Every metric has a per-instance ceiling on distinct label
|
|
65
|
-
* combinations (default 10,000). When a request's label set would
|
|
66
|
-
* create the 10,001st unique combination, the increment is dropped
|
|
67
|
-
* and a warning is logged ONCE per metric. The bound is high enough
|
|
68
|
-
* that legitimate apps don't hit it; low enough that runaway
|
|
69
|
-
* labels (a label per request id, per user id, per full URL with
|
|
70
|
-
* query string) can't OOM the process. Operators size up via
|
|
71
|
-
* labelCardinalityCap when they have a legitimate need.
|
|
72
|
-
*
|
|
73
|
-
* Best practice: route labels are the route TEMPLATE (`/users/:id`),
|
|
74
|
-
* not the actual path (`/users/123`). The framework's
|
|
75
|
-
* requestMiddleware uses req.routePattern when set; otherwise falls
|
|
76
|
-
* back to req.url stripped of query string.
|
|
77
|
-
*
|
|
78
|
-
* Exposition format:
|
|
79
|
-
* The text/plain exposition follows the Prometheus 0.0.4 text format:
|
|
80
|
-
* `# HELP <name> <description>` and `# TYPE <name> <counter|gauge|
|
|
81
|
-
* histogram>` headers, one sample per line with serialized labels
|
|
82
|
-
* in `{key="value",key2="value2"}` form. Buckets and _sum / _count
|
|
83
|
-
* for histograms.
|
|
84
|
-
*
|
|
85
|
-
* Out of scope (with structural reasons):
|
|
86
|
-
* - Summary type (client-side quantiles): generally inferior to
|
|
87
|
-
* histogram for aggregation across instances. Prometheus team
|
|
88
|
-
* recommends histogram. Add later if a real demand emerges.
|
|
89
|
-
* - Push gateway: pull-only monitoring is the simpler architecture.
|
|
90
|
-
* Operators with batch jobs that want push wire it themselves.
|
|
91
|
-
* - Native histogram (Prometheus 2.40+): not yet broadly supported
|
|
92
|
-
* by tooling; classic histogram is universal.
|
|
93
|
-
* - Per-process labels at scrape time (instance, hostname): operators
|
|
94
|
-
* pass via defaultLabels. The framework doesn't auto-inject —
|
|
95
|
-
* deploy environments differ on what to use (k8s pod name,
|
|
96
|
-
* hostname, container id) and the operator knows best.
|
|
97
|
-
*/
|
|
98
|
-
|
|
99
|
-
var C = require("./constants");
|
|
100
|
-
var canonicalJson = require("./canonical-json");
|
|
101
|
-
var { defineClass } = require("./framework-error");
|
|
102
|
-
var { boot } = require("./log");
|
|
103
|
-
var nb = require("./numeric-bounds");
|
|
104
|
-
var { resolveRoute, captureResponseStatus, HTTP_STATUS } = require("./request-helpers");
|
|
105
|
-
var validateOpts = require("./validate-opts");
|
|
106
|
-
|
|
107
|
-
var MetricsError = defineClass("MetricsError", { alwaysPermanent: true });
|
|
108
|
-
var log = boot("metrics");
|
|
109
|
-
|
|
110
|
-
// Default histogram buckets for HTTP latency in seconds.
|
|
111
|
-
var DEFAULT_HTTP_BUCKETS = Object.freeze([
|
|
112
|
-
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
|
113
|
-
]);
|
|
114
|
-
|
|
115
|
-
var DEFAULT_CARDINALITY_CAP = C.BYTES.bytes(10000);
|
|
116
|
-
// Bound metric / label names before regex test — DoS shape if an
|
|
117
|
-
// operator passed a multi-megabyte string. Prometheus exposition
|
|
118
|
-
// recommends short ascii names; 200 is a generous ceiling.
|
|
119
|
-
var MAX_METRIC_NAME_LEN = 200;
|
|
120
|
-
var METRIC_NAME_RE = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/;
|
|
121
|
-
var LABEL_NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
122
|
-
|
|
123
|
-
// ---- tap (global no-op stub for framework module taps) ----
|
|
124
|
-
//
|
|
125
|
-
// Framework modules call metrics.tap("name", value, labels) at hot
|
|
126
|
-
// paths. Until a registry is active, calls are no-ops (zero cost).
|
|
127
|
-
// metrics.create() replaces _activeTap with the registry-driven path,
|
|
128
|
-
// flowing taps into pre-registered counters / gauges / histograms.
|
|
129
|
-
//
|
|
130
|
-
// Drop-silent by design: the wrapping try/catch is intentional. tap is
|
|
131
|
-
// called from request hot paths where throwing on a misnamed metric
|
|
132
|
-
// (or a label whose value coerced unexpectedly) would crash the
|
|
133
|
-
// request that triggered it. The metric-registration path (counter /
|
|
134
|
-
// gauge / histogram) DOES throw on bad name/label-name regex — that's
|
|
135
|
-
// a config-time call where operators want fast failure; the runtime
|
|
136
|
-
// tap() is the drop-silent boundary.
|
|
137
|
-
|
|
138
|
-
var _activeTap = null;
|
|
139
|
-
|
|
140
|
-
function tap(name, value, labels) {
|
|
141
|
-
if (_activeTap === null) return;
|
|
142
|
-
try { _activeTap(name, value, labels); }
|
|
143
|
-
catch (_e) { /* tap path errors must not break callers */ }
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// ---- helpers ----
|
|
147
|
-
|
|
148
|
-
function _validateMetricName(name) {
|
|
149
|
-
if (typeof name !== "string" || name.length > MAX_METRIC_NAME_LEN ||
|
|
150
|
-
!METRIC_NAME_RE.test(name)) {
|
|
151
|
-
throw new MetricsError("metrics/bad-name",
|
|
152
|
-
"metric name '" + name + "' must match " + METRIC_NAME_RE +
|
|
153
|
-
" (max " + MAX_METRIC_NAME_LEN + " chars)");
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
function _validateLabelName(name) {
|
|
157
|
-
if (typeof name !== "string" || name.length > MAX_METRIC_NAME_LEN ||
|
|
158
|
-
!LABEL_NAME_RE.test(name)) {
|
|
159
|
-
throw new MetricsError("metrics/bad-label",
|
|
160
|
-
"label name '" + name + "' must match " + LABEL_NAME_RE +
|
|
161
|
-
" (max " + MAX_METRIC_NAME_LEN + " chars)");
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
// Counter / gauge / histogram methods all accept (callLabels, value) but
|
|
165
|
-
// degrade to (value) when no labels are passed. Centralized here so we
|
|
166
|
-
// don't repeat the swap-and-coerce dance four times.
|
|
167
|
-
//
|
|
168
|
-
// _normalizeLabelArg(arg1, arg2, defaultValue) → { labels, value }
|
|
169
|
-
function _normalizeLabelArg(callLabels, value, defaultValue) {
|
|
170
|
-
if (typeof callLabels === "number") {
|
|
171
|
-
return { labels: null, value: callLabels };
|
|
172
|
-
}
|
|
173
|
-
return {
|
|
174
|
-
labels: callLabels,
|
|
175
|
-
value: typeof value === "number" ? value : defaultValue,
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
function _validateLabelValue(value) {
|
|
180
|
-
// Prometheus exposition: label values are quoted strings; backslash,
|
|
181
|
-
// newline, double-quote get escaped at serialize time. Coerce here so
|
|
182
|
-
// counters indexed by various input types still work.
|
|
183
|
-
if (value === null || value === undefined) return "";
|
|
184
|
-
return String(value);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Serialize a labels object to a canonical Map key. Routed through
|
|
188
|
-
// canonical-json so the framework has one canonical-sort source of
|
|
189
|
-
// truth for sorted-keys serialization (avoiding the silent-data-loss
|
|
190
|
-
// class on Date / Buffer / Map / Set / BigInt values).
|
|
191
|
-
function _labelsKey(labels) {
|
|
192
|
-
if (!labels) return "";
|
|
193
|
-
// Coerce values to strings first so canonical-json's primitive
|
|
194
|
-
// serialization matches the Prometheus wire format.
|
|
195
|
-
var coerced = {};
|
|
196
|
-
var keys = Object.keys(labels);
|
|
197
|
-
for (var i = 0; i < keys.length; i++) {
|
|
198
|
-
coerced[keys[i]] = _validateLabelValue(labels[keys[i]]);
|
|
199
|
-
}
|
|
200
|
-
return canonicalJson.stringify(coerced);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
// Escape a label value per the Prometheus exposition format.
|
|
204
|
-
function _escapeLabelValue(v) {
|
|
205
|
-
return String(v).replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, "\\\"");
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
// Wire-format renderer for Prometheus exposition. Determinism is
|
|
209
|
-
// shared with _labelsKey: both walk keys in lexicographic order via
|
|
210
|
-
// the same _sortedLabelKeys helper, so the canonical-sort source of
|
|
211
|
-
// truth is one helper instead of two duplicated walks.
|
|
212
|
-
function _renderLabels(labelObj) {
|
|
213
|
-
if (!labelObj) return "";
|
|
214
|
-
var sortedKeys = _sortedLabelKeys(labelObj);
|
|
215
|
-
if (sortedKeys.length === 0) return "";
|
|
216
|
-
var parts = [];
|
|
217
|
-
for (var k = 0; k < sortedKeys.length; k++) {
|
|
218
|
-
parts.push(sortedKeys[k] + '="' + _escapeLabelValue(labelObj[sortedKeys[k]]) + '"');
|
|
219
|
-
}
|
|
220
|
-
return "{" + parts.join(",") + "}";
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
// Single canonical-sort source of truth for label keys. Both
|
|
224
|
-
// _labelsKey (Map key) and _renderLabels (Prometheus exposition)
|
|
225
|
-
// route through here so the framework has one walk shape.
|
|
226
|
-
function _sortedLabelKeys(labelObj) {
|
|
227
|
-
var keys = Object.keys(labelObj);
|
|
228
|
-
keys.sort();
|
|
229
|
-
return keys;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
// Combine default + per-call labels, validating against the metric's
|
|
233
|
-
// declared labelNames. Throws if a label name isn't declared.
|
|
234
|
-
function _resolveLabels(defaultLabels, declaredNames, callLabels) {
|
|
235
|
-
var out = {};
|
|
236
|
-
if (defaultLabels) {
|
|
237
|
-
var dk = Object.keys(defaultLabels);
|
|
238
|
-
for (var i = 0; i < dk.length; i++) out[dk[i]] = defaultLabels[dk[i]];
|
|
239
|
-
}
|
|
240
|
-
if (callLabels) {
|
|
241
|
-
var ck = Object.keys(callLabels);
|
|
242
|
-
for (var j = 0; j < ck.length; j++) {
|
|
243
|
-
var k = ck[j];
|
|
244
|
-
if (declaredNames.indexOf(k) === -1 && !(defaultLabels && Object.prototype.hasOwnProperty.call(defaultLabels, k))) {
|
|
245
|
-
throw new MetricsError("metrics/undeclared-label",
|
|
246
|
-
"label '" + k + "' not declared in labelNames " + JSON.stringify(declaredNames));
|
|
247
|
-
}
|
|
248
|
-
out[k] = callLabels[k];
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
// Verify all declared labels are present.
|
|
252
|
-
for (var n = 0; n < declaredNames.length; n++) {
|
|
253
|
-
if (!Object.prototype.hasOwnProperty.call(out, declaredNames[n])) {
|
|
254
|
-
throw new MetricsError("metrics/missing-label",
|
|
255
|
-
"label '" + declaredNames[n] + "' is required (declared in labelNames)");
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
return out;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
// ---- registry factory ----
|
|
262
|
-
|
|
263
|
-
function create(opts) {
|
|
264
|
-
opts = opts || {};
|
|
265
|
-
validateOpts(opts, [
|
|
266
|
-
"namespace", "defaultLabels", "labelCardinalityCap",
|
|
267
|
-
], "b.metrics");
|
|
268
|
-
var namespace = opts.namespace || "";
|
|
269
|
-
var defaultLabels = opts.defaultLabels || {};
|
|
270
|
-
nb.requirePositiveFiniteIntIfPresent(opts.labelCardinalityCap,
|
|
271
|
-
"labelCardinalityCap", MetricsError, "metrics/bad-opt");
|
|
272
|
-
var cardinalityCap = opts.labelCardinalityCap || DEFAULT_CARDINALITY_CAP;
|
|
273
|
-
|
|
274
|
-
// Validate defaultLabels names.
|
|
275
|
-
var dlk = Object.keys(defaultLabels);
|
|
276
|
-
for (var i = 0; i < dlk.length; i++) _validateLabelName(dlk[i]);
|
|
277
|
-
|
|
278
|
-
var metrics = new Map(); // fullName → metric instance
|
|
279
|
-
|
|
280
|
-
function _qualifyName(name) {
|
|
281
|
-
return namespace ? namespace + "_" + name : name;
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
function _registerMetric(metric) {
|
|
285
|
-
if (metrics.has(metric.name)) {
|
|
286
|
-
throw new MetricsError("metrics/duplicate",
|
|
287
|
-
"metric '" + metric.name + "' already registered");
|
|
288
|
-
}
|
|
289
|
-
metrics.set(metric.name, metric);
|
|
290
|
-
return metric;
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
// ---- counter ----
|
|
294
|
-
|
|
295
|
-
function counter(name, copts) {
|
|
296
|
-
copts = copts || {};
|
|
297
|
-
var fullName = _qualifyName(name);
|
|
298
|
-
_validateMetricName(fullName);
|
|
299
|
-
var labelNames = (copts.labelNames || []).slice();
|
|
300
|
-
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
301
|
-
var help = copts.help || "";
|
|
302
|
-
var values = new Map(); // labelsKey → { labels, value }
|
|
303
|
-
var capWarned = false;
|
|
304
|
-
|
|
305
|
-
var instance = {
|
|
306
|
-
type: "counter",
|
|
307
|
-
name: fullName,
|
|
308
|
-
help: help,
|
|
309
|
-
labelNames: labelNames,
|
|
310
|
-
values: values,
|
|
311
|
-
inc: function (callLabels, n) {
|
|
312
|
-
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
313
|
-
if (arg.value < 0) {
|
|
314
|
-
throw new MetricsError("metrics/counter-decrement",
|
|
315
|
-
"counter.inc value must be >= 0 (got " + arg.value + ") — counters never decrease");
|
|
316
|
-
}
|
|
317
|
-
var resolved = _resolveLabels(defaultLabels, labelNames, arg.labels);
|
|
318
|
-
var key = _labelsKey(resolved);
|
|
319
|
-
var entry = values.get(key);
|
|
320
|
-
if (!entry) {
|
|
321
|
-
if (values.size >= cardinalityCap) {
|
|
322
|
-
if (!capWarned) {
|
|
323
|
-
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap +
|
|
324
|
-
") — dropping new label combinations. Reduce label cardinality or raise the cap.");
|
|
325
|
-
capWarned = true;
|
|
326
|
-
}
|
|
327
|
-
return;
|
|
328
|
-
}
|
|
329
|
-
entry = { labels: resolved, value: 0 };
|
|
330
|
-
values.set(key, entry);
|
|
331
|
-
}
|
|
332
|
-
entry.value += arg.value;
|
|
333
|
-
},
|
|
334
|
-
reset: function () { values.clear(); capWarned = false; },
|
|
335
|
-
get: function (callLabels) {
|
|
336
|
-
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
337
|
-
var entry = values.get(_labelsKey(resolved));
|
|
338
|
-
return entry ? entry.value : 0;
|
|
339
|
-
},
|
|
340
|
-
};
|
|
341
|
-
return _registerMetric(instance);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
// ---- gauge ----
|
|
345
|
-
|
|
346
|
-
function gauge(name, copts) {
|
|
347
|
-
copts = copts || {};
|
|
348
|
-
var fullName = _qualifyName(name);
|
|
349
|
-
_validateMetricName(fullName);
|
|
350
|
-
var labelNames = (copts.labelNames || []).slice();
|
|
351
|
-
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
352
|
-
var help = copts.help || "";
|
|
353
|
-
var values = new Map();
|
|
354
|
-
var capWarned = false;
|
|
355
|
-
|
|
356
|
-
function _ensure(callLabels) {
|
|
357
|
-
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
358
|
-
var key = _labelsKey(resolved);
|
|
359
|
-
var entry = values.get(key);
|
|
360
|
-
if (!entry) {
|
|
361
|
-
if (values.size >= cardinalityCap) {
|
|
362
|
-
if (!capWarned) {
|
|
363
|
-
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap + ")");
|
|
364
|
-
capWarned = true;
|
|
365
|
-
}
|
|
366
|
-
return null;
|
|
367
|
-
}
|
|
368
|
-
entry = { labels: resolved, value: 0 };
|
|
369
|
-
values.set(key, entry);
|
|
370
|
-
}
|
|
371
|
-
return entry;
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
var instance = {
|
|
375
|
-
type: "gauge",
|
|
376
|
-
name: fullName,
|
|
377
|
-
help: help,
|
|
378
|
-
labelNames: labelNames,
|
|
379
|
-
values: values,
|
|
380
|
-
set: function (callLabels, v) {
|
|
381
|
-
var arg = _normalizeLabelArg(callLabels, v, NaN);
|
|
382
|
-
if (typeof arg.value !== "number" || isNaN(arg.value)) {
|
|
383
|
-
throw new MetricsError("metrics/gauge-bad-value",
|
|
384
|
-
"gauge.set value must be a finite number");
|
|
385
|
-
}
|
|
386
|
-
var entry = _ensure(arg.labels);
|
|
387
|
-
if (entry) entry.value = arg.value;
|
|
388
|
-
},
|
|
389
|
-
inc: function (callLabels, n) {
|
|
390
|
-
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
391
|
-
var entry = _ensure(arg.labels);
|
|
392
|
-
if (entry) entry.value += arg.value;
|
|
393
|
-
},
|
|
394
|
-
dec: function (callLabels, n) {
|
|
395
|
-
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
396
|
-
var entry = _ensure(arg.labels);
|
|
397
|
-
if (entry) entry.value -= arg.value;
|
|
398
|
-
},
|
|
399
|
-
reset: function () { values.clear(); capWarned = false; },
|
|
400
|
-
get: function (callLabels) {
|
|
401
|
-
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
402
|
-
var entry = values.get(_labelsKey(resolved));
|
|
403
|
-
return entry ? entry.value : 0;
|
|
404
|
-
},
|
|
405
|
-
};
|
|
406
|
-
return _registerMetric(instance);
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
// ---- histogram ----
|
|
410
|
-
|
|
411
|
-
function histogram(name, copts) {
|
|
412
|
-
copts = copts || {};
|
|
413
|
-
var fullName = _qualifyName(name);
|
|
414
|
-
_validateMetricName(fullName);
|
|
415
|
-
var labelNames = (copts.labelNames || []).slice();
|
|
416
|
-
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
417
|
-
var help = copts.help || "";
|
|
418
|
-
var rawBuckets = copts.buckets || DEFAULT_HTTP_BUCKETS;
|
|
419
|
-
if (!Array.isArray(rawBuckets) || rawBuckets.length === 0) {
|
|
420
|
-
throw new MetricsError("metrics/bad-buckets",
|
|
421
|
-
"histogram buckets must be a non-empty array of ascending numbers");
|
|
422
|
-
}
|
|
423
|
-
// Verify ascending + numeric.
|
|
424
|
-
for (var b = 0; b < rawBuckets.length; b++) {
|
|
425
|
-
if (typeof rawBuckets[b] !== "number" || isNaN(rawBuckets[b])) {
|
|
426
|
-
throw new MetricsError("metrics/bad-buckets",
|
|
427
|
-
"histogram bucket boundary " + b + " is not a number");
|
|
428
|
-
}
|
|
429
|
-
if (b > 0 && rawBuckets[b] <= rawBuckets[b - 1]) {
|
|
430
|
-
throw new MetricsError("metrics/bad-buckets",
|
|
431
|
-
"histogram buckets must be strictly ascending");
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
var buckets = rawBuckets.slice();
|
|
435
|
-
var values = new Map();
|
|
436
|
-
var capWarned = false;
|
|
437
|
-
|
|
438
|
-
var instance = {
|
|
439
|
-
type: "histogram",
|
|
440
|
-
name: fullName,
|
|
441
|
-
help: help,
|
|
442
|
-
labelNames: labelNames,
|
|
443
|
-
buckets: buckets,
|
|
444
|
-
values: values,
|
|
445
|
-
observe: function (callLabels, v) {
|
|
446
|
-
var arg = _normalizeLabelArg(callLabels, v, NaN);
|
|
447
|
-
if (typeof arg.value !== "number" || isNaN(arg.value)) {
|
|
448
|
-
throw new MetricsError("metrics/histogram-bad-value",
|
|
449
|
-
"histogram.observe value must be a finite number");
|
|
450
|
-
}
|
|
451
|
-
var resolved = _resolveLabels(defaultLabels, labelNames, arg.labels);
|
|
452
|
-
var key = _labelsKey(resolved);
|
|
453
|
-
var entry = values.get(key);
|
|
454
|
-
if (!entry) {
|
|
455
|
-
if (values.size >= cardinalityCap) {
|
|
456
|
-
if (!capWarned) {
|
|
457
|
-
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap + ")");
|
|
458
|
-
capWarned = true;
|
|
459
|
-
}
|
|
460
|
-
return;
|
|
461
|
-
}
|
|
462
|
-
// counts[i] is the count for the [<=buckets[i]] bucket; counts[buckets.length] is +Inf.
|
|
463
|
-
entry = {
|
|
464
|
-
labels: resolved,
|
|
465
|
-
counts: new Array(buckets.length + 1).fill(0),
|
|
466
|
-
sum: 0,
|
|
467
|
-
count: 0,
|
|
468
|
-
};
|
|
469
|
-
values.set(key, entry);
|
|
470
|
-
}
|
|
471
|
-
for (var i = 0; i < buckets.length; i++) {
|
|
472
|
-
if (arg.value <= buckets[i]) entry.counts[i]++;
|
|
473
|
-
}
|
|
474
|
-
entry.counts[buckets.length]++; // +Inf bucket is everything
|
|
475
|
-
entry.sum += arg.value;
|
|
476
|
-
entry.count += 1;
|
|
477
|
-
},
|
|
478
|
-
reset: function () { values.clear(); capWarned = false; },
|
|
479
|
-
};
|
|
480
|
-
return _registerMetric(instance);
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
// ---- exposition ----
|
|
484
|
-
|
|
485
|
-
function exposition() {
|
|
486
|
-
var lines = [];
|
|
487
|
-
var sortedNames = Array.from(metrics.keys()).sort();
|
|
488
|
-
for (var i = 0; i < sortedNames.length; i++) {
|
|
489
|
-
var m = metrics.get(sortedNames[i]);
|
|
490
|
-
if (m.help) lines.push("# HELP " + m.name + " " + m.help);
|
|
491
|
-
lines.push("# TYPE " + m.name + " " + m.type);
|
|
492
|
-
var keys = Array.from(m.values.keys()).sort();
|
|
493
|
-
if (m.type === "histogram") {
|
|
494
|
-
for (var k = 0; k < keys.length; k++) {
|
|
495
|
-
var entry = m.values.get(keys[k]);
|
|
496
|
-
for (var bi = 0; bi < m.buckets.length; bi++) {
|
|
497
|
-
var bLabels = Object.assign({}, entry.labels, { le: String(m.buckets[bi]) });
|
|
498
|
-
lines.push(m.name + "_bucket" + _renderLabels(bLabels) + " " + entry.counts[bi]);
|
|
499
|
-
}
|
|
500
|
-
var infLabels = Object.assign({}, entry.labels, { le: "+Inf" });
|
|
501
|
-
lines.push(m.name + "_bucket" + _renderLabels(infLabels) + " " + entry.counts[m.buckets.length]);
|
|
502
|
-
lines.push(m.name + "_sum" + _renderLabels(entry.labels) + " " + entry.sum);
|
|
503
|
-
lines.push(m.name + "_count" + _renderLabels(entry.labels) + " " + entry.count);
|
|
504
|
-
}
|
|
505
|
-
} else {
|
|
506
|
-
for (var v = 0; v < keys.length; v++) {
|
|
507
|
-
var ent = m.values.get(keys[v]);
|
|
508
|
-
lines.push(m.name + _renderLabels(ent.labels) + " " + ent.value);
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
lines.push("");
|
|
512
|
-
}
|
|
513
|
-
return lines.join("\n") + (lines.length ? "" : "\n");
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
function expositionHandler() {
|
|
517
|
-
return function metricsHandler(req, res) {
|
|
518
|
-
var body = exposition();
|
|
519
|
-
res.writeHead(HTTP_STATUS.OK, {
|
|
520
|
-
"Content-Type": "text/plain; version=0.0.4; charset=utf-8",
|
|
521
|
-
"Content-Length": Buffer.byteLength(body),
|
|
522
|
-
"Cache-Control": "no-store",
|
|
523
|
-
});
|
|
524
|
-
res.end(body);
|
|
525
|
-
};
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
// ---- request middleware (auto-time + auto-count) ----
|
|
529
|
-
|
|
530
|
-
// Built-in metrics created on first requestMiddleware call so apps
|
|
531
|
-
// that don't use the middleware don't pay the registration cost.
|
|
532
|
-
var requestsTotal = null;
|
|
533
|
-
var requestDuration = null;
|
|
534
|
-
function _ensureRequestMetrics() {
|
|
535
|
-
if (requestsTotal && requestDuration) return;
|
|
536
|
-
requestsTotal = counter("framework_http_requests_total", {
|
|
537
|
-
help: "Total HTTP requests handled by the framework",
|
|
538
|
-
labelNames: ["method", "route", "status"],
|
|
539
|
-
});
|
|
540
|
-
requestDuration = histogram("framework_http_request_duration_seconds", {
|
|
541
|
-
help: "HTTP request latency in seconds",
|
|
542
|
-
labelNames: ["method", "route"],
|
|
543
|
-
buckets: DEFAULT_HTTP_BUCKETS,
|
|
544
|
-
});
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function requestMiddleware() {
|
|
548
|
-
_ensureRequestMetrics();
|
|
549
|
-
return function metricsRequest(req, res, next) {
|
|
550
|
-
var start = process.hrtime.bigint();
|
|
551
|
-
captureResponseStatus(res, function (status) {
|
|
552
|
-
var elapsedSec = Number(process.hrtime.bigint() - start) / 1e9;
|
|
553
|
-
var route = resolveRoute(req);
|
|
554
|
-
var labels = { method: req.method || "GET", route: route, status: String(status) };
|
|
555
|
-
var durLabels = { method: labels.method, route: route };
|
|
556
|
-
// Best-effort tap path — a label coercion or registry race
|
|
557
|
-
// must not crash the request that triggered it. Log so
|
|
558
|
-
// the operator sees the failure in the same channel as
|
|
559
|
-
// the rest of the framework's diagnostics.
|
|
560
|
-
try { requestsTotal.inc(labels); }
|
|
561
|
-
catch (e) { log.warn("metrics/request-counter-failed: " + e.message); }
|
|
562
|
-
try { requestDuration.observe(durLabels, elapsedSec); }
|
|
563
|
-
catch (e) { log.warn("metrics/request-duration-failed: " + e.message); }
|
|
564
|
-
});
|
|
565
|
-
return next();
|
|
566
|
-
};
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
// ---- framework auto-instrumentation taps ----
|
|
570
|
-
//
|
|
571
|
-
// tap() calls in audit / vault / queue route through here when this
|
|
572
|
-
// registry is active. Pre-register the counters so we don't create
|
|
573
|
-
// them lazily inside the tap path (which would bind allocator cost
|
|
574
|
-
// to per-event hot paths).
|
|
575
|
-
|
|
576
|
-
var auditEventsTotal = counter("framework_audit_events_total", {
|
|
577
|
-
help: "Audit events recorded by the framework",
|
|
578
|
-
labelNames: ["action", "outcome"],
|
|
579
|
-
});
|
|
580
|
-
var vaultSealTotal = counter("framework_vault_seal_total", {
|
|
581
|
-
help: "Vault seal calls",
|
|
582
|
-
});
|
|
583
|
-
var vaultUnsealTotal = counter("framework_vault_unseal_total", {
|
|
584
|
-
help: "Vault unseal calls",
|
|
585
|
-
});
|
|
586
|
-
var queueEnqueueTotal = counter("framework_queue_enqueue_total", {
|
|
587
|
-
help: "Queue enqueue operations",
|
|
588
|
-
labelNames: ["queueName"],
|
|
589
|
-
});
|
|
590
|
-
var queueCompleteTotal = counter("framework_queue_complete_total", {
|
|
591
|
-
help: "Queue job completions",
|
|
592
|
-
labelNames: ["queueName"],
|
|
593
|
-
});
|
|
594
|
-
var queueFailTotal = counter("framework_queue_fail_total", {
|
|
595
|
-
help: "Queue job failures",
|
|
596
|
-
labelNames: ["queueName"],
|
|
597
|
-
});
|
|
598
|
-
var queueDepthGauge = gauge("framework_queue_depth", {
|
|
599
|
-
help: "Queue depth — pending + inflight jobs",
|
|
600
|
-
labelNames: ["queueName"],
|
|
601
|
-
});
|
|
602
|
-
var jobsInflightGauge = gauge("framework_jobs_inflight", {
|
|
603
|
-
help: "Jobs currently leased to a consumer",
|
|
604
|
-
labelNames: ["queueName"],
|
|
605
|
-
});
|
|
606
|
-
var errorsTotal = counter("framework_errors_total", {
|
|
607
|
-
help: "Framework-error class constructions",
|
|
608
|
-
labelNames: ["class"],
|
|
609
|
-
});
|
|
610
|
-
|
|
611
|
-
function _tapHandler(name, value, labels) {
|
|
612
|
-
var qn = (labels && labels.queueName) || "default";
|
|
613
|
-
if (name === "audit.record") {
|
|
614
|
-
auditEventsTotal.inc({
|
|
615
|
-
action: (labels && labels.action) || "unknown",
|
|
616
|
-
outcome: (labels && labels.outcome) || "unknown",
|
|
617
|
-
}, value || 1);
|
|
618
|
-
} else if (name === "vault.seal") { vaultSealTotal.inc(value || 1); }
|
|
619
|
-
else if (name === "vault.unseal") { vaultUnsealTotal.inc(value || 1); }
|
|
620
|
-
else if (name === "queue.enqueue") {
|
|
621
|
-
queueEnqueueTotal.inc({ queueName: qn }, value || 1);
|
|
622
|
-
queueDepthGauge.inc({ queueName: qn }, value || 1);
|
|
623
|
-
}
|
|
624
|
-
else if (name === "queue.lease") {
|
|
625
|
-
jobsInflightGauge.inc({ queueName: qn }, value || 1);
|
|
626
|
-
}
|
|
627
|
-
else if (name === "queue.complete") {
|
|
628
|
-
queueCompleteTotal.inc({ queueName: qn }, value || 1);
|
|
629
|
-
queueDepthGauge.dec({ queueName: qn }, value || 1);
|
|
630
|
-
jobsInflightGauge.dec({ queueName: qn }, value || 1);
|
|
631
|
-
}
|
|
632
|
-
else if (name === "queue.fail") {
|
|
633
|
-
queueFailTotal.inc({ queueName: qn }, value || 1);
|
|
634
|
-
jobsInflightGauge.dec({ queueName: qn }, value || 1);
|
|
635
|
-
// Depth = pending + inflight. Retry transitions inflight→pending
|
|
636
|
-
// (depth unchanged); terminal failure exits both buckets.
|
|
637
|
-
if (labels && labels.willRetry === false) {
|
|
638
|
-
queueDepthGauge.dec({ queueName: qn }, value || 1);
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
else if (name === "error.construct") {
|
|
642
|
-
errorsTotal.inc({ class: (labels && labels.class) || "unknown" }, value || 1);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
_activeTap = _tapHandler;
|
|
646
|
-
|
|
647
|
-
// ---- registry surface ----
|
|
648
|
-
|
|
649
|
-
var registry = {
|
|
650
|
-
counter: counter,
|
|
651
|
-
gauge: gauge,
|
|
652
|
-
histogram: histogram,
|
|
653
|
-
requestMiddleware: requestMiddleware,
|
|
654
|
-
expositionHandler: expositionHandler,
|
|
655
|
-
exposition: exposition,
|
|
656
|
-
metrics: metrics, // diagnostic; operators rarely need it
|
|
657
|
-
namespace: namespace,
|
|
658
|
-
defaultLabels: defaultLabels,
|
|
659
|
-
cardinalityCap: cardinalityCap,
|
|
660
|
-
deactivate: function () {
|
|
661
|
-
// Release the global tap handler. Operator-driven test reset.
|
|
662
|
-
if (_activeTap === _tapHandler) _activeTap = null;
|
|
663
|
-
},
|
|
664
|
-
};
|
|
665
|
-
return registry;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
function _resetForTest() {
|
|
669
|
-
_activeTap = null;
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
module.exports = {
|
|
673
|
-
create: create,
|
|
674
|
-
tap: tap,
|
|
675
|
-
MetricsError: MetricsError,
|
|
676
|
-
DEFAULT_HTTP_BUCKETS: DEFAULT_HTTP_BUCKETS,
|
|
677
|
-
DEFAULT_CARDINALITY_CAP: DEFAULT_CARDINALITY_CAP,
|
|
678
|
-
_resetForTest: _resetForTest,
|
|
679
|
-
// Internal helpers for tests
|
|
680
|
-
_labelsKey: _labelsKey,
|
|
681
|
-
_renderLabels: _renderLabels,
|
|
682
|
-
_escapeLabelValue: _escapeLabelValue,
|
|
683
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* metrics — Prometheus-format counters, gauges, and histograms.
|
|
4
|
+
*
|
|
5
|
+
* Production deployments need numbers. Without a metrics layer ops
|
|
6
|
+
* teams are half-blind on every incident: "Was that p99 spike real?
|
|
7
|
+
* What's the queue depth? How fast are audit emits going?". This
|
|
8
|
+
* module ships the standard Prometheus types with framework
|
|
9
|
+
* auto-instrumentation already wired into audit / vault / queue
|
|
10
|
+
* hot paths so operators get the framework's vital signs for free.
|
|
11
|
+
*
|
|
12
|
+
* Public API:
|
|
13
|
+
*
|
|
14
|
+
* var m = b.metrics.create({
|
|
15
|
+
* namespace: "myapp", // prepended to every metric name
|
|
16
|
+
* defaultLabels: { service: "api", version: "1.2.3" },
|
|
17
|
+
* labelCardinalityCap: 10000, // per-metric ceiling
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* var requests = m.counter("http_requests_total", {
|
|
21
|
+
* help: "Total HTTP requests",
|
|
22
|
+
* labelNames: ["method", "route", "status"],
|
|
23
|
+
* });
|
|
24
|
+
* requests.inc({ method: "GET", route: "/users", status: "200" });
|
|
25
|
+
* requests.inc({ method: "GET", route: "/users", status: "200" }, 5);
|
|
26
|
+
*
|
|
27
|
+
* var latency = m.histogram("http_request_duration_seconds", {
|
|
28
|
+
* help: "HTTP request latency",
|
|
29
|
+
* labelNames: ["method", "route"],
|
|
30
|
+
* buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
|
|
31
|
+
* });
|
|
32
|
+
* latency.observe({ method: "GET", route: "/users" }, 0.123);
|
|
33
|
+
*
|
|
34
|
+
* var queueDepth = m.gauge("queue_depth", { labelNames: ["queueName"] });
|
|
35
|
+
* queueDepth.set({ queueName: "default" }, 42);
|
|
36
|
+
* queueDepth.inc({ queueName: "default" });
|
|
37
|
+
* queueDepth.dec({ queueName: "default" });
|
|
38
|
+
*
|
|
39
|
+
* router.use(m.requestMiddleware()); // auto-times every request
|
|
40
|
+
* router.get("/metrics", m.expositionHandler());
|
|
41
|
+
*
|
|
42
|
+
* Framework auto-instrumentation:
|
|
43
|
+
* When metrics.create() runs, framework hot paths (audit.record,
|
|
44
|
+
* vault.seal, vault.unseal, queue ops) call metrics.tap() — a
|
|
45
|
+
* global no-op stub that the active registry replaces with real
|
|
46
|
+
* counters. Modules don't import the registry directly; the tap
|
|
47
|
+
* pattern keeps them decoupled and lets operators with no metrics
|
|
48
|
+
* pay zero cost.
|
|
49
|
+
*
|
|
50
|
+
* Built-in metrics surfaced:
|
|
51
|
+
* framework_audit_events_total{action, outcome} counter
|
|
52
|
+
* framework_vault_seal_total counter
|
|
53
|
+
* framework_vault_unseal_total counter
|
|
54
|
+
* framework_queue_enqueue_total{queueName} counter
|
|
55
|
+
* framework_queue_complete_total{queueName} counter
|
|
56
|
+
* framework_queue_fail_total{queueName} counter
|
|
57
|
+
* framework_queue_depth{queueName} gauge
|
|
58
|
+
* framework_jobs_inflight{queueName} gauge
|
|
59
|
+
* framework_errors_total{class} counter
|
|
60
|
+
* framework_http_requests_total{method,route,status} counter
|
|
61
|
+
* framework_http_request_duration_seconds{method,route} histogram
|
|
62
|
+
*
|
|
63
|
+
* Cardinality control:
|
|
64
|
+
* Every metric has a per-instance ceiling on distinct label
|
|
65
|
+
* combinations (default 10,000). When a request's label set would
|
|
66
|
+
* create the 10,001st unique combination, the increment is dropped
|
|
67
|
+
* and a warning is logged ONCE per metric. The bound is high enough
|
|
68
|
+
* that legitimate apps don't hit it; low enough that runaway
|
|
69
|
+
* labels (a label per request id, per user id, per full URL with
|
|
70
|
+
* query string) can't OOM the process. Operators size up via
|
|
71
|
+
* labelCardinalityCap when they have a legitimate need.
|
|
72
|
+
*
|
|
73
|
+
* Best practice: route labels are the route TEMPLATE (`/users/:id`),
|
|
74
|
+
* not the actual path (`/users/123`). The framework's
|
|
75
|
+
* requestMiddleware uses req.routePattern when set; otherwise falls
|
|
76
|
+
* back to req.url stripped of query string.
|
|
77
|
+
*
|
|
78
|
+
* Exposition format:
|
|
79
|
+
* The text/plain exposition follows the Prometheus 0.0.4 text format:
|
|
80
|
+
* `# HELP <name> <description>` and `# TYPE <name> <counter|gauge|
|
|
81
|
+
* histogram>` headers, one sample per line with serialized labels
|
|
82
|
+
* in `{key="value",key2="value2"}` form. Buckets and _sum / _count
|
|
83
|
+
* for histograms.
|
|
84
|
+
*
|
|
85
|
+
* Out of scope (with structural reasons):
|
|
86
|
+
* - Summary type (client-side quantiles): generally inferior to
|
|
87
|
+
* histogram for aggregation across instances. Prometheus team
|
|
88
|
+
* recommends histogram. Add later if a real demand emerges.
|
|
89
|
+
* - Push gateway: pull-only monitoring is the simpler architecture.
|
|
90
|
+
* Operators with batch jobs that want push wire it themselves.
|
|
91
|
+
* - Native histogram (Prometheus 2.40+): not yet broadly supported
|
|
92
|
+
* by tooling; classic histogram is universal.
|
|
93
|
+
* - Per-process labels at scrape time (instance, hostname): operators
|
|
94
|
+
* pass via defaultLabels. The framework doesn't auto-inject —
|
|
95
|
+
* deploy environments differ on what to use (k8s pod name,
|
|
96
|
+
* hostname, container id) and the operator knows best.
|
|
97
|
+
*/
|
|
98
|
+
|
|
99
|
+
var C = require("./constants");
|
|
100
|
+
var canonicalJson = require("./canonical-json");
|
|
101
|
+
var { defineClass } = require("./framework-error");
|
|
102
|
+
var { boot } = require("./log");
|
|
103
|
+
var nb = require("./numeric-bounds");
|
|
104
|
+
var { resolveRoute, captureResponseStatus, HTTP_STATUS } = require("./request-helpers");
|
|
105
|
+
var validateOpts = require("./validate-opts");
|
|
106
|
+
|
|
107
|
+
var MetricsError = defineClass("MetricsError", { alwaysPermanent: true });
|
|
108
|
+
var log = boot("metrics");
|
|
109
|
+
|
|
110
|
+
// Default histogram buckets for HTTP latency in seconds.
|
|
111
|
+
var DEFAULT_HTTP_BUCKETS = Object.freeze([
|
|
112
|
+
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
var DEFAULT_CARDINALITY_CAP = C.BYTES.bytes(10000);
|
|
116
|
+
// Bound metric / label names before regex test — DoS shape if an
|
|
117
|
+
// operator passed a multi-megabyte string. Prometheus exposition
|
|
118
|
+
// recommends short ascii names; 200 is a generous ceiling.
|
|
119
|
+
var MAX_METRIC_NAME_LEN = 200;
|
|
120
|
+
var METRIC_NAME_RE = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/;
|
|
121
|
+
var LABEL_NAME_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
122
|
+
|
|
123
|
+
// ---- tap (global no-op stub for framework module taps) ----
|
|
124
|
+
//
|
|
125
|
+
// Framework modules call metrics.tap("name", value, labels) at hot
|
|
126
|
+
// paths. Until a registry is active, calls are no-ops (zero cost).
|
|
127
|
+
// metrics.create() replaces _activeTap with the registry-driven path,
|
|
128
|
+
// flowing taps into pre-registered counters / gauges / histograms.
|
|
129
|
+
//
|
|
130
|
+
// Drop-silent by design: the wrapping try/catch is intentional. tap is
|
|
131
|
+
// called from request hot paths where throwing on a misnamed metric
|
|
132
|
+
// (or a label whose value coerced unexpectedly) would crash the
|
|
133
|
+
// request that triggered it. The metric-registration path (counter /
|
|
134
|
+
// gauge / histogram) DOES throw on bad name/label-name regex — that's
|
|
135
|
+
// a config-time call where operators want fast failure; the runtime
|
|
136
|
+
// tap() is the drop-silent boundary.
|
|
137
|
+
|
|
138
|
+
var _activeTap = null;
|
|
139
|
+
|
|
140
|
+
function tap(name, value, labels) {
|
|
141
|
+
if (_activeTap === null) return;
|
|
142
|
+
try { _activeTap(name, value, labels); }
|
|
143
|
+
catch (_e) { /* tap path errors must not break callers */ }
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ---- helpers ----
|
|
147
|
+
|
|
148
|
+
function _validateMetricName(name) {
|
|
149
|
+
if (typeof name !== "string" || name.length > MAX_METRIC_NAME_LEN ||
|
|
150
|
+
!METRIC_NAME_RE.test(name)) {
|
|
151
|
+
throw new MetricsError("metrics/bad-name",
|
|
152
|
+
"metric name '" + name + "' must match " + METRIC_NAME_RE +
|
|
153
|
+
" (max " + MAX_METRIC_NAME_LEN + " chars)");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function _validateLabelName(name) {
|
|
157
|
+
if (typeof name !== "string" || name.length > MAX_METRIC_NAME_LEN ||
|
|
158
|
+
!LABEL_NAME_RE.test(name)) {
|
|
159
|
+
throw new MetricsError("metrics/bad-label",
|
|
160
|
+
"label name '" + name + "' must match " + LABEL_NAME_RE +
|
|
161
|
+
" (max " + MAX_METRIC_NAME_LEN + " chars)");
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// Counter / gauge / histogram methods all accept (callLabels, value) but
|
|
165
|
+
// degrade to (value) when no labels are passed. Centralized here so we
|
|
166
|
+
// don't repeat the swap-and-coerce dance four times.
|
|
167
|
+
//
|
|
168
|
+
// _normalizeLabelArg(arg1, arg2, defaultValue) → { labels, value }
|
|
169
|
+
function _normalizeLabelArg(callLabels, value, defaultValue) {
|
|
170
|
+
if (typeof callLabels === "number") {
|
|
171
|
+
return { labels: null, value: callLabels };
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
labels: callLabels,
|
|
175
|
+
value: typeof value === "number" ? value : defaultValue,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function _validateLabelValue(value) {
|
|
180
|
+
// Prometheus exposition: label values are quoted strings; backslash,
|
|
181
|
+
// newline, double-quote get escaped at serialize time. Coerce here so
|
|
182
|
+
// counters indexed by various input types still work.
|
|
183
|
+
if (value === null || value === undefined) return "";
|
|
184
|
+
return String(value);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Serialize a labels object to a canonical Map key. Routed through
|
|
188
|
+
// canonical-json so the framework has one canonical-sort source of
|
|
189
|
+
// truth for sorted-keys serialization (avoiding the silent-data-loss
|
|
190
|
+
// class on Date / Buffer / Map / Set / BigInt values).
|
|
191
|
+
function _labelsKey(labels) {
|
|
192
|
+
if (!labels) return "";
|
|
193
|
+
// Coerce values to strings first so canonical-json's primitive
|
|
194
|
+
// serialization matches the Prometheus wire format.
|
|
195
|
+
var coerced = {};
|
|
196
|
+
var keys = Object.keys(labels);
|
|
197
|
+
for (var i = 0; i < keys.length; i++) {
|
|
198
|
+
coerced[keys[i]] = _validateLabelValue(labels[keys[i]]);
|
|
199
|
+
}
|
|
200
|
+
return canonicalJson.stringify(coerced);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Escape a label value per the Prometheus exposition format.
|
|
204
|
+
function _escapeLabelValue(v) {
|
|
205
|
+
return String(v).replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, "\\\"");
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Wire-format renderer for Prometheus exposition. Determinism is
|
|
209
|
+
// shared with _labelsKey: both walk keys in lexicographic order via
|
|
210
|
+
// the same _sortedLabelKeys helper, so the canonical-sort source of
|
|
211
|
+
// truth is one helper instead of two duplicated walks.
|
|
212
|
+
function _renderLabels(labelObj) {
|
|
213
|
+
if (!labelObj) return "";
|
|
214
|
+
var sortedKeys = _sortedLabelKeys(labelObj);
|
|
215
|
+
if (sortedKeys.length === 0) return "";
|
|
216
|
+
var parts = [];
|
|
217
|
+
for (var k = 0; k < sortedKeys.length; k++) {
|
|
218
|
+
parts.push(sortedKeys[k] + '="' + _escapeLabelValue(labelObj[sortedKeys[k]]) + '"');
|
|
219
|
+
}
|
|
220
|
+
return "{" + parts.join(",") + "}";
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Single canonical-sort source of truth for label keys. Both
|
|
224
|
+
// _labelsKey (Map key) and _renderLabels (Prometheus exposition)
|
|
225
|
+
// route through here so the framework has one walk shape.
|
|
226
|
+
function _sortedLabelKeys(labelObj) {
|
|
227
|
+
var keys = Object.keys(labelObj);
|
|
228
|
+
keys.sort();
|
|
229
|
+
return keys;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Combine default + per-call labels, validating against the metric's
|
|
233
|
+
// declared labelNames. Throws if a label name isn't declared.
|
|
234
|
+
function _resolveLabels(defaultLabels, declaredNames, callLabels) {
|
|
235
|
+
var out = {};
|
|
236
|
+
if (defaultLabels) {
|
|
237
|
+
var dk = Object.keys(defaultLabels);
|
|
238
|
+
for (var i = 0; i < dk.length; i++) out[dk[i]] = defaultLabels[dk[i]];
|
|
239
|
+
}
|
|
240
|
+
if (callLabels) {
|
|
241
|
+
var ck = Object.keys(callLabels);
|
|
242
|
+
for (var j = 0; j < ck.length; j++) {
|
|
243
|
+
var k = ck[j];
|
|
244
|
+
if (declaredNames.indexOf(k) === -1 && !(defaultLabels && Object.prototype.hasOwnProperty.call(defaultLabels, k))) {
|
|
245
|
+
throw new MetricsError("metrics/undeclared-label",
|
|
246
|
+
"label '" + k + "' not declared in labelNames " + JSON.stringify(declaredNames));
|
|
247
|
+
}
|
|
248
|
+
out[k] = callLabels[k];
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
// Verify all declared labels are present.
|
|
252
|
+
for (var n = 0; n < declaredNames.length; n++) {
|
|
253
|
+
if (!Object.prototype.hasOwnProperty.call(out, declaredNames[n])) {
|
|
254
|
+
throw new MetricsError("metrics/missing-label",
|
|
255
|
+
"label '" + declaredNames[n] + "' is required (declared in labelNames)");
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return out;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// ---- registry factory ----
|
|
262
|
+
|
|
263
|
+
function create(opts) {
|
|
264
|
+
opts = opts || {};
|
|
265
|
+
validateOpts(opts, [
|
|
266
|
+
"namespace", "defaultLabels", "labelCardinalityCap",
|
|
267
|
+
], "b.metrics");
|
|
268
|
+
var namespace = opts.namespace || "";
|
|
269
|
+
var defaultLabels = opts.defaultLabels || {};
|
|
270
|
+
nb.requirePositiveFiniteIntIfPresent(opts.labelCardinalityCap,
|
|
271
|
+
"labelCardinalityCap", MetricsError, "metrics/bad-opt");
|
|
272
|
+
var cardinalityCap = opts.labelCardinalityCap || DEFAULT_CARDINALITY_CAP;
|
|
273
|
+
|
|
274
|
+
// Validate defaultLabels names.
|
|
275
|
+
var dlk = Object.keys(defaultLabels);
|
|
276
|
+
for (var i = 0; i < dlk.length; i++) _validateLabelName(dlk[i]);
|
|
277
|
+
|
|
278
|
+
var metrics = new Map(); // fullName → metric instance
|
|
279
|
+
|
|
280
|
+
function _qualifyName(name) {
|
|
281
|
+
return namespace ? namespace + "_" + name : name;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function _registerMetric(metric) {
|
|
285
|
+
if (metrics.has(metric.name)) {
|
|
286
|
+
throw new MetricsError("metrics/duplicate",
|
|
287
|
+
"metric '" + metric.name + "' already registered");
|
|
288
|
+
}
|
|
289
|
+
metrics.set(metric.name, metric);
|
|
290
|
+
return metric;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// ---- counter ----
|
|
294
|
+
|
|
295
|
+
function counter(name, copts) {
|
|
296
|
+
copts = copts || {};
|
|
297
|
+
var fullName = _qualifyName(name);
|
|
298
|
+
_validateMetricName(fullName);
|
|
299
|
+
var labelNames = (copts.labelNames || []).slice();
|
|
300
|
+
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
301
|
+
var help = copts.help || "";
|
|
302
|
+
var values = new Map(); // labelsKey → { labels, value }
|
|
303
|
+
var capWarned = false;
|
|
304
|
+
|
|
305
|
+
var instance = {
|
|
306
|
+
type: "counter",
|
|
307
|
+
name: fullName,
|
|
308
|
+
help: help,
|
|
309
|
+
labelNames: labelNames,
|
|
310
|
+
values: values,
|
|
311
|
+
inc: function (callLabels, n) {
|
|
312
|
+
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
313
|
+
if (arg.value < 0) {
|
|
314
|
+
throw new MetricsError("metrics/counter-decrement",
|
|
315
|
+
"counter.inc value must be >= 0 (got " + arg.value + ") — counters never decrease");
|
|
316
|
+
}
|
|
317
|
+
var resolved = _resolveLabels(defaultLabels, labelNames, arg.labels);
|
|
318
|
+
var key = _labelsKey(resolved);
|
|
319
|
+
var entry = values.get(key);
|
|
320
|
+
if (!entry) {
|
|
321
|
+
if (values.size >= cardinalityCap) {
|
|
322
|
+
if (!capWarned) {
|
|
323
|
+
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap +
|
|
324
|
+
") — dropping new label combinations. Reduce label cardinality or raise the cap.");
|
|
325
|
+
capWarned = true;
|
|
326
|
+
}
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
entry = { labels: resolved, value: 0 };
|
|
330
|
+
values.set(key, entry);
|
|
331
|
+
}
|
|
332
|
+
entry.value += arg.value;
|
|
333
|
+
},
|
|
334
|
+
reset: function () { values.clear(); capWarned = false; },
|
|
335
|
+
get: function (callLabels) {
|
|
336
|
+
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
337
|
+
var entry = values.get(_labelsKey(resolved));
|
|
338
|
+
return entry ? entry.value : 0;
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
return _registerMetric(instance);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// ---- gauge ----
|
|
345
|
+
|
|
346
|
+
function gauge(name, copts) {
|
|
347
|
+
copts = copts || {};
|
|
348
|
+
var fullName = _qualifyName(name);
|
|
349
|
+
_validateMetricName(fullName);
|
|
350
|
+
var labelNames = (copts.labelNames || []).slice();
|
|
351
|
+
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
352
|
+
var help = copts.help || "";
|
|
353
|
+
var values = new Map();
|
|
354
|
+
var capWarned = false;
|
|
355
|
+
|
|
356
|
+
function _ensure(callLabels) {
|
|
357
|
+
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
358
|
+
var key = _labelsKey(resolved);
|
|
359
|
+
var entry = values.get(key);
|
|
360
|
+
if (!entry) {
|
|
361
|
+
if (values.size >= cardinalityCap) {
|
|
362
|
+
if (!capWarned) {
|
|
363
|
+
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap + ")");
|
|
364
|
+
capWarned = true;
|
|
365
|
+
}
|
|
366
|
+
return null;
|
|
367
|
+
}
|
|
368
|
+
entry = { labels: resolved, value: 0 };
|
|
369
|
+
values.set(key, entry);
|
|
370
|
+
}
|
|
371
|
+
return entry;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
var instance = {
|
|
375
|
+
type: "gauge",
|
|
376
|
+
name: fullName,
|
|
377
|
+
help: help,
|
|
378
|
+
labelNames: labelNames,
|
|
379
|
+
values: values,
|
|
380
|
+
set: function (callLabels, v) {
|
|
381
|
+
var arg = _normalizeLabelArg(callLabels, v, NaN);
|
|
382
|
+
if (typeof arg.value !== "number" || isNaN(arg.value)) {
|
|
383
|
+
throw new MetricsError("metrics/gauge-bad-value",
|
|
384
|
+
"gauge.set value must be a finite number");
|
|
385
|
+
}
|
|
386
|
+
var entry = _ensure(arg.labels);
|
|
387
|
+
if (entry) entry.value = arg.value;
|
|
388
|
+
},
|
|
389
|
+
inc: function (callLabels, n) {
|
|
390
|
+
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
391
|
+
var entry = _ensure(arg.labels);
|
|
392
|
+
if (entry) entry.value += arg.value;
|
|
393
|
+
},
|
|
394
|
+
dec: function (callLabels, n) {
|
|
395
|
+
var arg = _normalizeLabelArg(callLabels, n, 1);
|
|
396
|
+
var entry = _ensure(arg.labels);
|
|
397
|
+
if (entry) entry.value -= arg.value;
|
|
398
|
+
},
|
|
399
|
+
reset: function () { values.clear(); capWarned = false; },
|
|
400
|
+
get: function (callLabels) {
|
|
401
|
+
var resolved = _resolveLabels(defaultLabels, labelNames, callLabels);
|
|
402
|
+
var entry = values.get(_labelsKey(resolved));
|
|
403
|
+
return entry ? entry.value : 0;
|
|
404
|
+
},
|
|
405
|
+
};
|
|
406
|
+
return _registerMetric(instance);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// ---- histogram ----
|
|
410
|
+
|
|
411
|
+
function histogram(name, copts) {
|
|
412
|
+
copts = copts || {};
|
|
413
|
+
var fullName = _qualifyName(name);
|
|
414
|
+
_validateMetricName(fullName);
|
|
415
|
+
var labelNames = (copts.labelNames || []).slice();
|
|
416
|
+
for (var i = 0; i < labelNames.length; i++) _validateLabelName(labelNames[i]);
|
|
417
|
+
var help = copts.help || "";
|
|
418
|
+
var rawBuckets = copts.buckets || DEFAULT_HTTP_BUCKETS;
|
|
419
|
+
if (!Array.isArray(rawBuckets) || rawBuckets.length === 0) {
|
|
420
|
+
throw new MetricsError("metrics/bad-buckets",
|
|
421
|
+
"histogram buckets must be a non-empty array of ascending numbers");
|
|
422
|
+
}
|
|
423
|
+
// Verify ascending + numeric.
|
|
424
|
+
for (var b = 0; b < rawBuckets.length; b++) {
|
|
425
|
+
if (typeof rawBuckets[b] !== "number" || isNaN(rawBuckets[b])) {
|
|
426
|
+
throw new MetricsError("metrics/bad-buckets",
|
|
427
|
+
"histogram bucket boundary " + b + " is not a number");
|
|
428
|
+
}
|
|
429
|
+
if (b > 0 && rawBuckets[b] <= rawBuckets[b - 1]) {
|
|
430
|
+
throw new MetricsError("metrics/bad-buckets",
|
|
431
|
+
"histogram buckets must be strictly ascending");
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
var buckets = rawBuckets.slice();
|
|
435
|
+
var values = new Map();
|
|
436
|
+
var capWarned = false;
|
|
437
|
+
|
|
438
|
+
var instance = {
|
|
439
|
+
type: "histogram",
|
|
440
|
+
name: fullName,
|
|
441
|
+
help: help,
|
|
442
|
+
labelNames: labelNames,
|
|
443
|
+
buckets: buckets,
|
|
444
|
+
values: values,
|
|
445
|
+
observe: function (callLabels, v) {
|
|
446
|
+
var arg = _normalizeLabelArg(callLabels, v, NaN);
|
|
447
|
+
if (typeof arg.value !== "number" || isNaN(arg.value)) {
|
|
448
|
+
throw new MetricsError("metrics/histogram-bad-value",
|
|
449
|
+
"histogram.observe value must be a finite number");
|
|
450
|
+
}
|
|
451
|
+
var resolved = _resolveLabels(defaultLabels, labelNames, arg.labels);
|
|
452
|
+
var key = _labelsKey(resolved);
|
|
453
|
+
var entry = values.get(key);
|
|
454
|
+
if (!entry) {
|
|
455
|
+
if (values.size >= cardinalityCap) {
|
|
456
|
+
if (!capWarned) {
|
|
457
|
+
log("metric '" + fullName + "' hit labelCardinalityCap (" + cardinalityCap + ")");
|
|
458
|
+
capWarned = true;
|
|
459
|
+
}
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
// counts[i] is the count for the [<=buckets[i]] bucket; counts[buckets.length] is +Inf.
|
|
463
|
+
entry = {
|
|
464
|
+
labels: resolved,
|
|
465
|
+
counts: new Array(buckets.length + 1).fill(0),
|
|
466
|
+
sum: 0,
|
|
467
|
+
count: 0,
|
|
468
|
+
};
|
|
469
|
+
values.set(key, entry);
|
|
470
|
+
}
|
|
471
|
+
for (var i = 0; i < buckets.length; i++) {
|
|
472
|
+
if (arg.value <= buckets[i]) entry.counts[i]++;
|
|
473
|
+
}
|
|
474
|
+
entry.counts[buckets.length]++; // +Inf bucket is everything
|
|
475
|
+
entry.sum += arg.value;
|
|
476
|
+
entry.count += 1;
|
|
477
|
+
},
|
|
478
|
+
reset: function () { values.clear(); capWarned = false; },
|
|
479
|
+
};
|
|
480
|
+
return _registerMetric(instance);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// ---- exposition ----
|
|
484
|
+
|
|
485
|
+
function exposition() {
|
|
486
|
+
var lines = [];
|
|
487
|
+
var sortedNames = Array.from(metrics.keys()).sort();
|
|
488
|
+
for (var i = 0; i < sortedNames.length; i++) {
|
|
489
|
+
var m = metrics.get(sortedNames[i]);
|
|
490
|
+
if (m.help) lines.push("# HELP " + m.name + " " + m.help);
|
|
491
|
+
lines.push("# TYPE " + m.name + " " + m.type);
|
|
492
|
+
var keys = Array.from(m.values.keys()).sort();
|
|
493
|
+
if (m.type === "histogram") {
|
|
494
|
+
for (var k = 0; k < keys.length; k++) {
|
|
495
|
+
var entry = m.values.get(keys[k]);
|
|
496
|
+
for (var bi = 0; bi < m.buckets.length; bi++) {
|
|
497
|
+
var bLabels = Object.assign({}, entry.labels, { le: String(m.buckets[bi]) });
|
|
498
|
+
lines.push(m.name + "_bucket" + _renderLabels(bLabels) + " " + entry.counts[bi]);
|
|
499
|
+
}
|
|
500
|
+
var infLabels = Object.assign({}, entry.labels, { le: "+Inf" });
|
|
501
|
+
lines.push(m.name + "_bucket" + _renderLabels(infLabels) + " " + entry.counts[m.buckets.length]);
|
|
502
|
+
lines.push(m.name + "_sum" + _renderLabels(entry.labels) + " " + entry.sum);
|
|
503
|
+
lines.push(m.name + "_count" + _renderLabels(entry.labels) + " " + entry.count);
|
|
504
|
+
}
|
|
505
|
+
} else {
|
|
506
|
+
for (var v = 0; v < keys.length; v++) {
|
|
507
|
+
var ent = m.values.get(keys[v]);
|
|
508
|
+
lines.push(m.name + _renderLabels(ent.labels) + " " + ent.value);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
lines.push("");
|
|
512
|
+
}
|
|
513
|
+
return lines.join("\n") + (lines.length ? "" : "\n");
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function expositionHandler() {
|
|
517
|
+
return function metricsHandler(req, res) {
|
|
518
|
+
var body = exposition();
|
|
519
|
+
res.writeHead(HTTP_STATUS.OK, {
|
|
520
|
+
"Content-Type": "text/plain; version=0.0.4; charset=utf-8",
|
|
521
|
+
"Content-Length": Buffer.byteLength(body),
|
|
522
|
+
"Cache-Control": "no-store",
|
|
523
|
+
});
|
|
524
|
+
res.end(body);
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// ---- request middleware (auto-time + auto-count) ----
|
|
529
|
+
|
|
530
|
+
// Built-in metrics created on first requestMiddleware call so apps
|
|
531
|
+
// that don't use the middleware don't pay the registration cost.
|
|
532
|
+
var requestsTotal = null;
|
|
533
|
+
var requestDuration = null;
|
|
534
|
+
function _ensureRequestMetrics() {
|
|
535
|
+
if (requestsTotal && requestDuration) return;
|
|
536
|
+
requestsTotal = counter("framework_http_requests_total", {
|
|
537
|
+
help: "Total HTTP requests handled by the framework",
|
|
538
|
+
labelNames: ["method", "route", "status"],
|
|
539
|
+
});
|
|
540
|
+
requestDuration = histogram("framework_http_request_duration_seconds", {
|
|
541
|
+
help: "HTTP request latency in seconds",
|
|
542
|
+
labelNames: ["method", "route"],
|
|
543
|
+
buckets: DEFAULT_HTTP_BUCKETS,
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function requestMiddleware() {
|
|
548
|
+
_ensureRequestMetrics();
|
|
549
|
+
return function metricsRequest(req, res, next) {
|
|
550
|
+
var start = process.hrtime.bigint();
|
|
551
|
+
captureResponseStatus(res, function (status) {
|
|
552
|
+
var elapsedSec = Number(process.hrtime.bigint() - start) / 1e9;
|
|
553
|
+
var route = resolveRoute(req);
|
|
554
|
+
var labels = { method: req.method || "GET", route: route, status: String(status) };
|
|
555
|
+
var durLabels = { method: labels.method, route: route };
|
|
556
|
+
// Best-effort tap path — a label coercion or registry race
|
|
557
|
+
// must not crash the request that triggered it. Log so
|
|
558
|
+
// the operator sees the failure in the same channel as
|
|
559
|
+
// the rest of the framework's diagnostics.
|
|
560
|
+
try { requestsTotal.inc(labels); }
|
|
561
|
+
catch (e) { log.warn("metrics/request-counter-failed: " + e.message); }
|
|
562
|
+
try { requestDuration.observe(durLabels, elapsedSec); }
|
|
563
|
+
catch (e) { log.warn("metrics/request-duration-failed: " + e.message); }
|
|
564
|
+
});
|
|
565
|
+
return next();
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
// ---- framework auto-instrumentation taps ----
|
|
570
|
+
//
|
|
571
|
+
// tap() calls in audit / vault / queue route through here when this
|
|
572
|
+
// registry is active. Pre-register the counters so we don't create
|
|
573
|
+
// them lazily inside the tap path (which would bind allocator cost
|
|
574
|
+
// to per-event hot paths).
|
|
575
|
+
|
|
576
|
+
var auditEventsTotal = counter("framework_audit_events_total", {
|
|
577
|
+
help: "Audit events recorded by the framework",
|
|
578
|
+
labelNames: ["action", "outcome"],
|
|
579
|
+
});
|
|
580
|
+
var vaultSealTotal = counter("framework_vault_seal_total", {
|
|
581
|
+
help: "Vault seal calls",
|
|
582
|
+
});
|
|
583
|
+
var vaultUnsealTotal = counter("framework_vault_unseal_total", {
|
|
584
|
+
help: "Vault unseal calls",
|
|
585
|
+
});
|
|
586
|
+
var queueEnqueueTotal = counter("framework_queue_enqueue_total", {
|
|
587
|
+
help: "Queue enqueue operations",
|
|
588
|
+
labelNames: ["queueName"],
|
|
589
|
+
});
|
|
590
|
+
var queueCompleteTotal = counter("framework_queue_complete_total", {
|
|
591
|
+
help: "Queue job completions",
|
|
592
|
+
labelNames: ["queueName"],
|
|
593
|
+
});
|
|
594
|
+
var queueFailTotal = counter("framework_queue_fail_total", {
|
|
595
|
+
help: "Queue job failures",
|
|
596
|
+
labelNames: ["queueName"],
|
|
597
|
+
});
|
|
598
|
+
var queueDepthGauge = gauge("framework_queue_depth", {
|
|
599
|
+
help: "Queue depth — pending + inflight jobs",
|
|
600
|
+
labelNames: ["queueName"],
|
|
601
|
+
});
|
|
602
|
+
var jobsInflightGauge = gauge("framework_jobs_inflight", {
|
|
603
|
+
help: "Jobs currently leased to a consumer",
|
|
604
|
+
labelNames: ["queueName"],
|
|
605
|
+
});
|
|
606
|
+
var errorsTotal = counter("framework_errors_total", {
|
|
607
|
+
help: "Framework-error class constructions",
|
|
608
|
+
labelNames: ["class"],
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
function _tapHandler(name, value, labels) {
|
|
612
|
+
var qn = (labels && labels.queueName) || "default";
|
|
613
|
+
if (name === "audit.record") {
|
|
614
|
+
auditEventsTotal.inc({
|
|
615
|
+
action: (labels && labels.action) || "unknown",
|
|
616
|
+
outcome: (labels && labels.outcome) || "unknown",
|
|
617
|
+
}, value || 1);
|
|
618
|
+
} else if (name === "vault.seal") { vaultSealTotal.inc(value || 1); }
|
|
619
|
+
else if (name === "vault.unseal") { vaultUnsealTotal.inc(value || 1); }
|
|
620
|
+
else if (name === "queue.enqueue") {
|
|
621
|
+
queueEnqueueTotal.inc({ queueName: qn }, value || 1);
|
|
622
|
+
queueDepthGauge.inc({ queueName: qn }, value || 1);
|
|
623
|
+
}
|
|
624
|
+
else if (name === "queue.lease") {
|
|
625
|
+
jobsInflightGauge.inc({ queueName: qn }, value || 1);
|
|
626
|
+
}
|
|
627
|
+
else if (name === "queue.complete") {
|
|
628
|
+
queueCompleteTotal.inc({ queueName: qn }, value || 1);
|
|
629
|
+
queueDepthGauge.dec({ queueName: qn }, value || 1);
|
|
630
|
+
jobsInflightGauge.dec({ queueName: qn }, value || 1);
|
|
631
|
+
}
|
|
632
|
+
else if (name === "queue.fail") {
|
|
633
|
+
queueFailTotal.inc({ queueName: qn }, value || 1);
|
|
634
|
+
jobsInflightGauge.dec({ queueName: qn }, value || 1);
|
|
635
|
+
// Depth = pending + inflight. Retry transitions inflight→pending
|
|
636
|
+
// (depth unchanged); terminal failure exits both buckets.
|
|
637
|
+
if (labels && labels.willRetry === false) {
|
|
638
|
+
queueDepthGauge.dec({ queueName: qn }, value || 1);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
else if (name === "error.construct") {
|
|
642
|
+
errorsTotal.inc({ class: (labels && labels.class) || "unknown" }, value || 1);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
_activeTap = _tapHandler;
|
|
646
|
+
|
|
647
|
+
// ---- registry surface ----
|
|
648
|
+
|
|
649
|
+
var registry = {
|
|
650
|
+
counter: counter,
|
|
651
|
+
gauge: gauge,
|
|
652
|
+
histogram: histogram,
|
|
653
|
+
requestMiddleware: requestMiddleware,
|
|
654
|
+
expositionHandler: expositionHandler,
|
|
655
|
+
exposition: exposition,
|
|
656
|
+
metrics: metrics, // diagnostic; operators rarely need it
|
|
657
|
+
namespace: namespace,
|
|
658
|
+
defaultLabels: defaultLabels,
|
|
659
|
+
cardinalityCap: cardinalityCap,
|
|
660
|
+
deactivate: function () {
|
|
661
|
+
// Release the global tap handler. Operator-driven test reset.
|
|
662
|
+
if (_activeTap === _tapHandler) _activeTap = null;
|
|
663
|
+
},
|
|
664
|
+
};
|
|
665
|
+
return registry;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
function _resetForTest() {
|
|
669
|
+
_activeTap = null;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
module.exports = {
|
|
673
|
+
create: create,
|
|
674
|
+
tap: tap,
|
|
675
|
+
MetricsError: MetricsError,
|
|
676
|
+
DEFAULT_HTTP_BUCKETS: DEFAULT_HTTP_BUCKETS,
|
|
677
|
+
DEFAULT_CARDINALITY_CAP: DEFAULT_CARDINALITY_CAP,
|
|
678
|
+
_resetForTest: _resetForTest,
|
|
679
|
+
// Internal helpers for tests
|
|
680
|
+
_labelsKey: _labelsKey,
|
|
681
|
+
_renderLabels: _renderLabels,
|
|
682
|
+
_escapeLabelValue: _escapeLabelValue,
|
|
683
|
+
};
|