@futurewindai/wotchi 0.1.0-beta.2 → 0.1.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -47
- package/SECURITY.md +2 -2
- package/dist/cjs/core/admission.js +22 -0
- package/dist/cjs/core/client.js +416 -21
- package/dist/cjs/core/config.js +193 -18
- package/dist/cjs/core/diagnostics.js +16 -1
- package/dist/cjs/core/group-store.js +22 -9
- package/dist/cjs/core/incident-builder.js +18 -0
- package/dist/cjs/core/incident-policy.js +2 -7
- package/dist/cjs/core/limits.js +17 -0
- package/dist/cjs/core/normalize.js +16 -6
- package/dist/cjs/core/notification-queue.js +92 -6
- package/dist/cjs/core/prometheus.js +129 -0
- package/dist/cjs/core/redact.js +57 -8
- package/dist/cjs/core/runtime-monitor.js +109 -0
- package/dist/cjs/index.js +8 -1
- package/dist/cjs/integrations/express/error-handler.js +21 -4
- package/dist/cjs/integrations/express/index.js +2 -1
- package/dist/cjs/integrations/express/status-observer.js +6 -0
- package/dist/cjs/integrations/nest/capture.js +21 -0
- package/dist/cjs/integrations/nest/exception-filter.js +64 -8
- package/dist/cjs/integrations/nest/filter-wrapper.js +17 -0
- package/dist/cjs/integrations/nest/index.js +7 -1
- package/dist/cjs/integrations/nest/module.js +44 -0
- package/dist/cjs/integrations/nest/register.js +15 -1
- package/dist/cjs/integrations/request-context.js +83 -1
- package/dist/cjs/notifiers/alert-payload.js +20 -0
- package/dist/cjs/notifiers/console.js +85 -15
- package/dist/cjs/notifiers/telegram-format.js +75 -5
- package/dist/cjs/notifiers/webhook-http.js +394 -0
- package/dist/cjs/notifiers/webhook.js +28 -0
- package/dist/esm/core/admission.js +19 -0
- package/dist/esm/core/client.js +416 -21
- package/dist/esm/core/config.js +193 -18
- package/dist/esm/core/diagnostics.js +16 -1
- package/dist/esm/core/group-store.js +22 -9
- package/dist/esm/core/incident-builder.js +18 -0
- package/dist/esm/core/incident-policy.js +2 -7
- package/dist/esm/core/limits.js +14 -0
- package/dist/esm/core/normalize.js +16 -6
- package/dist/esm/core/notification-queue.js +92 -6
- package/dist/esm/core/prometheus.js +125 -0
- package/dist/esm/core/redact.js +57 -8
- package/dist/esm/core/runtime-monitor.js +106 -0
- package/dist/esm/index.js +4 -1
- package/dist/esm/integrations/express/error-handler.js +21 -4
- package/dist/esm/integrations/express/index.js +1 -1
- package/dist/esm/integrations/express/status-observer.js +6 -0
- package/dist/esm/integrations/nest/capture.js +17 -0
- package/dist/esm/integrations/nest/exception-filter.js +65 -9
- package/dist/esm/integrations/nest/filter-wrapper.js +14 -0
- package/dist/esm/integrations/nest/index.js +3 -1
- package/dist/esm/integrations/nest/module.js +41 -0
- package/dist/esm/integrations/nest/register.js +15 -1
- package/dist/esm/integrations/request-context.js +83 -1
- package/dist/esm/notifiers/alert-payload.js +16 -0
- package/dist/esm/notifiers/console.js +85 -15
- package/dist/esm/notifiers/telegram-format.js +75 -5
- package/dist/esm/notifiers/webhook-http.js +387 -0
- package/dist/esm/notifiers/webhook.js +24 -0
- package/dist/types/core/admission.d.ts +10 -0
- package/dist/types/core/config.d.ts +17 -1
- package/dist/types/core/diagnostics.d.ts +7 -1
- package/dist/types/core/incident-policy.d.ts +1 -1
- package/dist/types/core/limits.d.ts +14 -0
- package/dist/types/core/notification-queue.d.ts +14 -1
- package/dist/types/core/prometheus.d.ts +7 -0
- package/dist/types/core/runtime-monitor.d.ts +15 -0
- package/dist/types/core/types.d.ts +111 -2
- package/dist/types/index.d.ts +8 -2
- package/dist/types/integrations/express/index.d.ts +2 -2
- package/dist/types/integrations/nest/capture.d.ts +4 -0
- package/dist/types/integrations/nest/exception-filter.d.ts +6 -2
- package/dist/types/integrations/nest/filter-wrapper.d.ts +7 -0
- package/dist/types/integrations/nest/index.d.ts +6 -2
- package/dist/types/integrations/nest/module.d.ts +9 -0
- package/dist/types/integrations/nest/register.d.ts +3 -0
- package/dist/types/integrations/request-context.d.ts +3 -0
- package/dist/types/notifiers/alert-payload.d.ts +5 -0
- package/dist/types/notifiers/webhook-http.d.ts +37 -0
- package/dist/types/notifiers/webhook.d.ts +4 -0
- package/dist/types-cjs/core/admission.d.cts +10 -0
- package/dist/types-cjs/core/config.d.cts +17 -1
- package/dist/types-cjs/core/diagnostics.d.cts +7 -1
- package/dist/types-cjs/core/incident-policy.d.cts +1 -1
- package/dist/types-cjs/core/limits.d.cts +14 -0
- package/dist/types-cjs/core/notification-queue.d.cts +14 -1
- package/dist/types-cjs/core/prometheus.d.cts +7 -0
- package/dist/types-cjs/core/runtime-monitor.d.cts +15 -0
- package/dist/types-cjs/core/types.d.cts +111 -2
- package/dist/types-cjs/index.d.cts +8 -2
- package/dist/types-cjs/integrations/express/index.d.cts +2 -2
- package/dist/types-cjs/integrations/nest/capture.d.cts +4 -0
- package/dist/types-cjs/integrations/nest/exception-filter.d.cts +6 -2
- package/dist/types-cjs/integrations/nest/filter-wrapper.d.cts +7 -0
- package/dist/types-cjs/integrations/nest/index.d.cts +6 -2
- package/dist/types-cjs/integrations/nest/module.d.cts +9 -0
- package/dist/types-cjs/integrations/nest/register.d.cts +3 -0
- package/dist/types-cjs/integrations/request-context.d.cts +3 -0
- package/dist/types-cjs/notifiers/alert-payload.d.cts +5 -0
- package/dist/types-cjs/notifiers/webhook-http.d.cts +37 -0
- package/dist/types-cjs/notifiers/webhook.d.cts +4 -0
- package/package.json +17 -4
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
export const PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8";
|
|
2
|
+
const METRICS = [
|
|
3
|
+
{
|
|
4
|
+
name: "captured_events_total",
|
|
5
|
+
help: "Total events accepted by Wotchi.",
|
|
6
|
+
type: "counter",
|
|
7
|
+
value: (diagnostics) => diagnostics.capturedEvents,
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
name: "capture_failures_total",
|
|
11
|
+
help: "Capture failures contained by Wotchi.",
|
|
12
|
+
type: "counter",
|
|
13
|
+
value: (diagnostics) => diagnostics.captureFailures,
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
name: "groups_evicted_total",
|
|
17
|
+
help: "Groups evicted from bounded memory.",
|
|
18
|
+
type: "counter",
|
|
19
|
+
value: (diagnostics) => diagnostics.groupsEvicted,
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
name: "alerts_queued_total",
|
|
23
|
+
help: "Alerts admitted to the notification queue.",
|
|
24
|
+
type: "counter",
|
|
25
|
+
value: (diagnostics) => diagnostics.alertsQueued,
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: "alerts_dropped_total",
|
|
29
|
+
help: "Alerts dropped because the notification queue was full.",
|
|
30
|
+
type: "counter",
|
|
31
|
+
value: (diagnostics) => diagnostics.alertsDropped,
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
name: "alerts_sent_total",
|
|
35
|
+
help: "Alerts delivered successfully to notifiers.",
|
|
36
|
+
type: "counter",
|
|
37
|
+
value: (diagnostics) => diagnostics.alertsSent,
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
name: "notifier_failures_total",
|
|
41
|
+
help: "Notifier delivery failures contained by Wotchi.",
|
|
42
|
+
type: "counter",
|
|
43
|
+
value: (diagnostics) => diagnostics.notifierFailures,
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
name: "fingerprint_callback_failures_total",
|
|
47
|
+
help: "Fingerprint callback failures contained by Wotchi.",
|
|
48
|
+
type: "counter",
|
|
49
|
+
value: (diagnostics) => diagnostics.fingerprintCallbackFailures,
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
name: "filter_failures_total",
|
|
53
|
+
help: "Event filter failures contained by Wotchi.",
|
|
54
|
+
type: "counter",
|
|
55
|
+
value: (diagnostics) => diagnostics.filterFailures,
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
name: "before_send_failures_total",
|
|
59
|
+
help: "Before-send callback failures contained by Wotchi.",
|
|
60
|
+
type: "counter",
|
|
61
|
+
value: (diagnostics) => diagnostics.beforeSendFailures,
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: "events_suppressed_total",
|
|
65
|
+
help: "Events suppressed by filtering, thresholds, or cooldowns.",
|
|
66
|
+
type: "counter",
|
|
67
|
+
value: (diagnostics) => diagnostics.eventsSuppressed,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: "events_dropped_overload_total",
|
|
71
|
+
help: "Events dropped before normalization by the optional overload admission limit.",
|
|
72
|
+
type: "counter",
|
|
73
|
+
value: (diagnostics) => diagnostics.eventsDroppedOverload,
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
name: "captures_after_shutdown_total",
|
|
77
|
+
help: "Capture calls ignored after client shutdown.",
|
|
78
|
+
type: "counter",
|
|
79
|
+
value: (diagnostics) => diagnostics.capturesAfterShutdown,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
name: "notifier_timeouts_total",
|
|
83
|
+
help: "Notifier deliveries that exceeded the configured timeout.",
|
|
84
|
+
type: "counter",
|
|
85
|
+
value: (diagnostics) => diagnostics.notifierTimeouts,
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
name: "notifier_circuit_open_skips_total",
|
|
89
|
+
help: "Notifier deliveries skipped while the notifier circuit was open.",
|
|
90
|
+
type: "counter",
|
|
91
|
+
value: (diagnostics) => diagnostics.notifierCircuitOpenSkips,
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
name: "active_groups",
|
|
95
|
+
help: "Current in-memory incident groups.",
|
|
96
|
+
type: "gauge",
|
|
97
|
+
value: (diagnostics) => diagnostics.activeGroups,
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
name: "pending_alerts",
|
|
101
|
+
help: "Current alerts waiting for notifier delivery.",
|
|
102
|
+
type: "gauge",
|
|
103
|
+
value: (diagnostics) => diagnostics.pendingAlerts,
|
|
104
|
+
},
|
|
105
|
+
];
|
|
106
|
+
const safeMetricValue = (value) => {
|
|
107
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
108
|
+
return 0;
|
|
109
|
+
}
|
|
110
|
+
return Math.min(Math.floor(value), Number.MAX_SAFE_INTEGER);
|
|
111
|
+
};
|
|
112
|
+
const renderMetrics = (diagnostics) => {
|
|
113
|
+
const lines = [];
|
|
114
|
+
for (const metric of METRICS) {
|
|
115
|
+
lines.push(`# HELP wotchi_${metric.name} ${metric.help}`, `# TYPE wotchi_${metric.name} ${metric.type}`, `wotchi_${metric.name} ${safeMetricValue(metric.value(diagnostics))}`);
|
|
116
|
+
}
|
|
117
|
+
lines.push("");
|
|
118
|
+
return lines.join("\n");
|
|
119
|
+
};
|
|
120
|
+
export function createWotchiPrometheusExporter(client) {
|
|
121
|
+
return Object.freeze({
|
|
122
|
+
contentType: PROMETHEUS_CONTENT_TYPE,
|
|
123
|
+
render: () => renderMetrics(client.getDiagnostics()),
|
|
124
|
+
});
|
|
125
|
+
}
|
package/dist/esm/core/redact.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { normalizeUnknown } from "./normalize.js";
|
|
2
|
+
import { MAX_NORMALIZATION_DEPTH, MAX_NORMALIZATION_KEYS, MAX_NORMALIZATION_STACK_LENGTH, MAX_NORMALIZATION_STRING_LENGTH, } from "./limits.js";
|
|
2
3
|
export const REDACTED = "[REDACTED]";
|
|
3
4
|
const DEFAULT_KEYS = [
|
|
4
5
|
"authorization",
|
|
@@ -9,6 +10,12 @@ const DEFAULT_KEYS = [
|
|
|
9
10
|
"secret",
|
|
10
11
|
"apiKey",
|
|
11
12
|
"api_key",
|
|
13
|
+
"xApiKey",
|
|
14
|
+
"x_api_key",
|
|
15
|
+
"dbPassword",
|
|
16
|
+
"db_password",
|
|
17
|
+
"databasePassword",
|
|
18
|
+
"database_password",
|
|
12
19
|
"token",
|
|
13
20
|
"accessToken",
|
|
14
21
|
"access_token",
|
|
@@ -18,12 +25,16 @@ const DEFAULT_KEYS = [
|
|
|
18
25
|
"private_key",
|
|
19
26
|
"clientSecret",
|
|
20
27
|
"client_secret",
|
|
28
|
+
"signature",
|
|
29
|
+
"session",
|
|
30
|
+
"webhookSecret",
|
|
31
|
+
"webhook_secret",
|
|
21
32
|
"cardNumber",
|
|
22
33
|
"cvv",
|
|
23
34
|
"cvc",
|
|
24
35
|
"pin",
|
|
25
36
|
];
|
|
26
|
-
const normalizeKey = (key) => key.toLowerCase().replace(/[-
|
|
37
|
+
const normalizeKey = (key) => key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
27
38
|
const defaultKeySet = new Set(DEFAULT_KEYS.map(normalizeKey));
|
|
28
39
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
29
40
|
const luhnValid = (digits) => {
|
|
@@ -49,6 +60,7 @@ const redactCardLikeValues = (value) => value.replace(/\b\d[\d -]{11,25}\d\b/g,
|
|
|
49
60
|
const CONNECTION_URL_PATTERN = /\b(?:postgres(?:ql)?|rediss?|mongodb(?:\+srv)?):\/\/[^\s"'<>]+/gi;
|
|
50
61
|
const CONNECTION_QUERY_SECRET_PATTERN = /([?&](?:password|passwd|pass|secret|token|api[-_]?key|access[-_]?token|refresh[-_]?token|sslpassword)=)[^&#\s]*/gi;
|
|
51
62
|
const URL_TRAILING_PUNCTUATION = /[),.;!?]+$/;
|
|
63
|
+
const AUTHORITY_USERINFO_PATTERN = /\b[^:\s/@]+:[^\s/@]+@(?=(?:[a-z0-9-]+(?:\.[a-z0-9-]+)*|\[[0-9a-f:.]+\])(?::\d+)?(?:[/?#\s,;.)]|$))/gi;
|
|
52
64
|
const redactConnectionUrl = (candidate) => {
|
|
53
65
|
let core = candidate;
|
|
54
66
|
let suffix = "";
|
|
@@ -72,20 +84,57 @@ const redactConnectionUrl = (candidate) => {
|
|
|
72
84
|
return `${core.slice(0, authorityStart)}${redactedAuthority}${remainder}${suffix}`;
|
|
73
85
|
};
|
|
74
86
|
const redactConnectionUrls = (value) => value.replace(CONNECTION_URL_PATTERN, redactConnectionUrl);
|
|
87
|
+
const redactAuthorityUserinfo = (value) => value.replace(AUTHORITY_USERINFO_PATTERN, `${REDACTED}@`);
|
|
88
|
+
const SENSITIVE_QUERY_KEYS = new Set([
|
|
89
|
+
"authorization",
|
|
90
|
+
"cookie",
|
|
91
|
+
"password",
|
|
92
|
+
"passwd",
|
|
93
|
+
"secret",
|
|
94
|
+
"signature",
|
|
95
|
+
"token",
|
|
96
|
+
"accesstoken",
|
|
97
|
+
"refreshtoken",
|
|
98
|
+
"apikey",
|
|
99
|
+
"xapikey",
|
|
100
|
+
"session",
|
|
101
|
+
]);
|
|
102
|
+
const decodeQueryKey = (value) => {
|
|
103
|
+
try {
|
|
104
|
+
return decodeURIComponent(value.replace(/\+/g, " "));
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return value;
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
const redactSensitiveQueryValues = (value) => value.replace(/([?&])([^=&#\s]+)=([^&#\s]*)/g, (match, prefix, key) => {
|
|
111
|
+
if (!SENSITIVE_QUERY_KEYS.has(normalizeKey(decodeQueryKey(key)))) {
|
|
112
|
+
return match;
|
|
113
|
+
}
|
|
114
|
+
return prefix + key + "=" + REDACTED;
|
|
115
|
+
});
|
|
116
|
+
const redactWebhookPathSecrets = (value) => value.replace(/(https?:\/\/[^/\s]+\/(?:services|hooks|webhooks?)\/)[^/?#\s]+/gi, (_match, prefix) => prefix + REDACTED);
|
|
75
117
|
const redactString = (value, maxStringLength) => {
|
|
76
118
|
let result = redactConnectionUrls(value);
|
|
119
|
+
result = redactAuthorityUserinfo(result);
|
|
120
|
+
result = redactWebhookPathSecrets(result);
|
|
121
|
+
result = redactSensitiveQueryValues(result);
|
|
77
122
|
result = result.replace(/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gi, REDACTED);
|
|
78
123
|
result = result.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED}`);
|
|
79
|
-
result = result.replace(/\
|
|
124
|
+
result = result.replace(/\bBasic\s+[A-Za-z0-9+/=]+/gi, `Basic ${REDACTED}`);
|
|
125
|
+
result = result.replace(/\b(?:Cookie|Set-Cookie)\s*:\s*[^\r\n]+/gi, "Cookie: [REDACTED]");
|
|
126
|
+
result = result.replace(/\b(?:authorization|password|passwd|secret|signature|session|token|x[-_]?api[-_]?key|db[-_]?password|database[-_]?password|api[-_]?key)\s*[:=]\s*[^\s,;]+/gi, (match) => {
|
|
80
127
|
const separatorIndex = match.search(/[:=]/);
|
|
81
128
|
return separatorIndex < 0 ? REDACTED : `${match.slice(0, separatorIndex + 1)}${REDACTED}`;
|
|
82
129
|
});
|
|
83
|
-
result = result.replace(/\b[A-Za-z0-9_-]{
|
|
130
|
+
result = result.replace(/\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, REDACTED);
|
|
84
131
|
result = result.replace(/\b(?:github_pat|ghp|xox[baprs]-|sk|pk|AKIA)[A-Za-z0-9_-]{8,}\b/gi, REDACTED);
|
|
85
132
|
result = redactCardLikeValues(result);
|
|
86
133
|
return result.slice(0, maxStringLength);
|
|
87
134
|
};
|
|
88
|
-
const normalizedOption = (value, fallback) => Number.isSafeInteger(value) && value !== undefined && value > 0
|
|
135
|
+
const normalizedOption = (value, fallback, maximum) => Number.isSafeInteger(value) && value !== undefined && value > 0 && value <= maximum
|
|
136
|
+
? value
|
|
137
|
+
: fallback;
|
|
89
138
|
const redactNormalized = (value, sensitiveKeys, options, depth, keysVisited) => {
|
|
90
139
|
if (typeof value === "string") {
|
|
91
140
|
return redactString(value, options.maxStringLength);
|
|
@@ -123,10 +172,10 @@ const redactNormalized = (value, sensitiveKeys, options, depth, keysVisited) =>
|
|
|
123
172
|
return result;
|
|
124
173
|
};
|
|
125
174
|
const redactionLimits = (options) => ({
|
|
126
|
-
maxDepth: normalizedOption(options.maxDepth, 5),
|
|
127
|
-
maxKeys: normalizedOption(options.maxKeys, 100),
|
|
128
|
-
maxStringLength: normalizedOption(options.maxStringLength, 500),
|
|
129
|
-
maxStackLength: normalizedOption(options.maxStackLength, 4_000),
|
|
175
|
+
maxDepth: normalizedOption(options.maxDepth, 5, MAX_NORMALIZATION_DEPTH),
|
|
176
|
+
maxKeys: normalizedOption(options.maxKeys, 100, MAX_NORMALIZATION_KEYS),
|
|
177
|
+
maxStringLength: normalizedOption(options.maxStringLength, 500, MAX_NORMALIZATION_STRING_LENGTH),
|
|
178
|
+
maxStackLength: normalizedOption(options.maxStackLength, 4_000, MAX_NORMALIZATION_STACK_LENGTH),
|
|
130
179
|
});
|
|
131
180
|
const sensitiveKeySet = (options) => {
|
|
132
181
|
const keys = new Set(defaultKeySet);
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
const DEFAULT_INTERVAL_MS = 5_000;
|
|
2
|
+
const MIN_INTERVAL_MS = 100;
|
|
3
|
+
const MAX_INTERVAL_MS = 60_000;
|
|
4
|
+
const MAX_THRESHOLD = Number.MAX_SAFE_INTEGER;
|
|
5
|
+
const positiveNumber = (value, fallback, field) => {
|
|
6
|
+
if (value === undefined) {
|
|
7
|
+
return fallback;
|
|
8
|
+
}
|
|
9
|
+
if (!Number.isFinite(value) || value <= 0 || value > MAX_THRESHOLD) {
|
|
10
|
+
throw new RangeError(`${field} must be a positive finite number`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
};
|
|
14
|
+
export function registerWotchiRuntimeWatcher(client, options = {}) {
|
|
15
|
+
const intervalMs = positiveNumber(options.intervalMs, DEFAULT_INTERVAL_MS, "intervalMs");
|
|
16
|
+
if (!Number.isSafeInteger(intervalMs) ||
|
|
17
|
+
intervalMs < MIN_INTERVAL_MS ||
|
|
18
|
+
intervalMs > MAX_INTERVAL_MS) {
|
|
19
|
+
throw new RangeError(`intervalMs must be an integer from ${MIN_INTERVAL_MS} to ${MAX_INTERVAL_MS}`);
|
|
20
|
+
}
|
|
21
|
+
const thresholds = {
|
|
22
|
+
cpuPercent: options.cpuPercent,
|
|
23
|
+
rssBytes: options.rssBytes,
|
|
24
|
+
heapUsedBytes: options.heapUsedBytes,
|
|
25
|
+
eventLoopDelayMs: options.eventLoopDelayMs,
|
|
26
|
+
pendingAlerts: options.pendingAlerts,
|
|
27
|
+
notifierFailures: options.notifierFailures,
|
|
28
|
+
};
|
|
29
|
+
for (const [field, value] of Object.entries(thresholds)) {
|
|
30
|
+
if (value !== undefined) {
|
|
31
|
+
positiveNumber(value, 1, field);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const alertThreshold = options.alertThreshold ?? 3;
|
|
35
|
+
if (!Number.isSafeInteger(alertThreshold) || alertThreshold <= 0 || alertThreshold > 1_000_000) {
|
|
36
|
+
throw new RangeError("alertThreshold must be a positive integer no greater than 1000000");
|
|
37
|
+
}
|
|
38
|
+
let previousCpu = process.cpuUsage();
|
|
39
|
+
let previousTime = process.hrtime.bigint();
|
|
40
|
+
let previousNotifierFailures = client.getDiagnostics().notifierFailures;
|
|
41
|
+
let expectedAt = Date.now() + intervalMs;
|
|
42
|
+
let registered = true;
|
|
43
|
+
const sample = () => {
|
|
44
|
+
if (!registered) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const timestamp = Date.now();
|
|
48
|
+
const elapsedMicros = Math.max(1, Number(process.hrtime.bigint() - previousTime) / 1_000);
|
|
49
|
+
const cpu = process.cpuUsage(previousCpu);
|
|
50
|
+
previousCpu = process.cpuUsage();
|
|
51
|
+
previousTime = process.hrtime.bigint();
|
|
52
|
+
const memory = process.memoryUsage();
|
|
53
|
+
const eventLoopDelayMs = Math.max(0, timestamp - expectedAt);
|
|
54
|
+
expectedAt = timestamp + intervalMs;
|
|
55
|
+
const cpuPercent = ((cpu.user + cpu.system) / elapsedMicros) * 100;
|
|
56
|
+
const diagnostics = client.getDiagnostics();
|
|
57
|
+
const notifierFailures = Math.max(0, diagnostics.notifierFailures - previousNotifierFailures);
|
|
58
|
+
previousNotifierFailures = diagnostics.notifierFailures;
|
|
59
|
+
const readings = [
|
|
60
|
+
{ metric: "cpu-percent", value: cpuPercent, threshold: thresholds.cpuPercent },
|
|
61
|
+
{ metric: "rss-bytes", value: memory.rss, threshold: thresholds.rssBytes },
|
|
62
|
+
{ metric: "heap-used-bytes", value: memory.heapUsed, threshold: thresholds.heapUsedBytes },
|
|
63
|
+
{
|
|
64
|
+
metric: "event-loop-delay-ms",
|
|
65
|
+
value: eventLoopDelayMs,
|
|
66
|
+
threshold: thresholds.eventLoopDelayMs,
|
|
67
|
+
},
|
|
68
|
+
{ metric: "pending-alerts", value: 0, threshold: thresholds.pendingAlerts },
|
|
69
|
+
{
|
|
70
|
+
metric: "notifier-failures",
|
|
71
|
+
value: notifierFailures,
|
|
72
|
+
threshold: thresholds.notifierFailures,
|
|
73
|
+
},
|
|
74
|
+
];
|
|
75
|
+
for (const reading of readings) {
|
|
76
|
+
const value = reading.metric === "pending-alerts" ? diagnostics.pendingAlerts : reading.value;
|
|
77
|
+
if (reading.threshold === undefined || value < reading.threshold) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
client.captureEvent({
|
|
81
|
+
level: "error",
|
|
82
|
+
kind: "runtime-monitor",
|
|
83
|
+
message: `Wotchi runtime pressure: ${reading.metric}`,
|
|
84
|
+
fingerprint: `wotchi.runtime.${reading.metric}`,
|
|
85
|
+
severity: "high",
|
|
86
|
+
alertThreshold,
|
|
87
|
+
context: {
|
|
88
|
+
metric: reading.metric,
|
|
89
|
+
value: Math.round(value * 100) / 100,
|
|
90
|
+
threshold: reading.threshold,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
const timer = setInterval(sample, intervalMs);
|
|
96
|
+
timer.unref?.();
|
|
97
|
+
return {
|
|
98
|
+
unregister: () => {
|
|
99
|
+
if (!registered) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
registered = false;
|
|
103
|
+
clearInterval(timer);
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
package/dist/esm/index.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { createWotchi } from "./core/client.js";
|
|
2
2
|
import { registerWotchiProcessMonitor } from "./core/process-monitor.js";
|
|
3
|
+
import { createWotchiPrometheusExporter, PROMETHEUS_CONTENT_TYPE } from "./core/prometheus.js";
|
|
4
|
+
import { registerWotchiRuntimeWatcher } from "./core/runtime-monitor.js";
|
|
3
5
|
import { consoleNotifier } from "./notifiers/console.js";
|
|
4
6
|
import { telegramNotifier } from "./notifiers/telegram.js";
|
|
7
|
+
import { webhookNotifier } from "./notifiers/webhook.js";
|
|
5
8
|
export { WotchiConfigurationError } from "./core/errors.js";
|
|
6
|
-
export { consoleNotifier, createWotchi, registerWotchiProcessMonitor, telegramNotifier };
|
|
9
|
+
export { consoleNotifier, createWotchi, createWotchiPrometheusExporter, PROMETHEUS_CONTENT_TYPE, registerWotchiProcessMonitor, registerWotchiRuntimeWatcher, telegramNotifier, webhookNotifier, };
|
|
@@ -3,11 +3,28 @@ import { markExpressErrorCaptured } from "./state.js";
|
|
|
3
3
|
export function createExpressErrorHandler(client, options) {
|
|
4
4
|
return (error, request, response, next) => {
|
|
5
5
|
markExpressErrorCaptured(request);
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
let captured = false;
|
|
7
|
+
const captureAfterResponse = () => {
|
|
8
|
+
if (captured) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
captured = true;
|
|
12
|
+
response.off("finish", captureAfterResponse);
|
|
13
|
+
response.off("close", captureAfterResponse);
|
|
14
|
+
try {
|
|
15
|
+
const requestContext = getExpressRequestContext(request, response, options);
|
|
16
|
+
client.captureException(error, undefined, requestContext === undefined ? undefined : { request: requestContext });
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// A capture failure must never change Express error handling.
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
if (response.writableEnded || response.finished) {
|
|
23
|
+
captureAfterResponse();
|
|
8
24
|
}
|
|
9
|
-
|
|
10
|
-
|
|
25
|
+
else {
|
|
26
|
+
response.once("finish", captureAfterResponse);
|
|
27
|
+
response.once("close", captureAfterResponse);
|
|
11
28
|
}
|
|
12
29
|
next(error);
|
|
13
30
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createExpressErrorHandler } from "./error-handler.js";
|
|
2
|
-
export { consoleNotifier, createWotchi, telegramNotifier } from "../../index.js";
|
|
2
|
+
export { consoleNotifier, createWotchi, telegramNotifier, webhookNotifier } from "../../index.js";
|
|
3
3
|
export { wotchiStatusObserver } from "./status-observer.js";
|
|
4
4
|
export function wotchiErrorHandler(client, options) {
|
|
5
5
|
return createExpressErrorHandler(client, options);
|
|
@@ -39,6 +39,12 @@ const normalizeOptions = (options) => {
|
|
|
39
39
|
...(options?.requestIdProperty === undefined
|
|
40
40
|
? {}
|
|
41
41
|
: { requestIdProperty: options.requestIdProperty }),
|
|
42
|
+
...(options?.correlationIdProperty === undefined
|
|
43
|
+
? {}
|
|
44
|
+
: { correlationIdProperty: options.correlationIdProperty }),
|
|
45
|
+
...(options?.traceContextProperty === undefined
|
|
46
|
+
? {}
|
|
47
|
+
: { traceContextProperty: options.traceContextProperty }),
|
|
42
48
|
statusCodes: normalizeStatuses(options?.statusCodes, DEFAULT_STATUS_CODES),
|
|
43
49
|
statusClasses: normalizeStatusClasses(options?.statusClasses),
|
|
44
50
|
ignoreStatusCodes: normalizeStatuses(options?.ignoreStatusCodes, []),
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { markExpressErrorCaptured } from "../express/state.js";
|
|
2
|
+
import { getNestRequestContext } from "./request-context.js";
|
|
3
|
+
export const captureWotchiNestException = (client, exception, host, options) => {
|
|
4
|
+
try {
|
|
5
|
+
markExpressErrorCaptured(host.switchToHttp().getRequest());
|
|
6
|
+
}
|
|
7
|
+
catch {
|
|
8
|
+
// Non-HTTP NestJS contexts do not expose an Express request to mark.
|
|
9
|
+
}
|
|
10
|
+
try {
|
|
11
|
+
const requestContext = getNestRequestContext(host, exception, options);
|
|
12
|
+
client.captureException(exception, undefined, requestContext === undefined ? undefined : { request: requestContext });
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// A capture failure must never change NestJS exception handling.
|
|
16
|
+
}
|
|
17
|
+
};
|
|
@@ -7,27 +7,83 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
-
import { Catch } from "@nestjs/common";
|
|
10
|
+
import { Catch, HttpException } from "@nestjs/common";
|
|
11
11
|
import { BaseExceptionFilter } from "@nestjs/core";
|
|
12
|
-
import {
|
|
12
|
+
import { captureWotchiNestException } from "./capture.js";
|
|
13
|
+
const FILTER_CATCH_EXCEPTIONS = "__filterCatchExceptions__";
|
|
14
|
+
const matchesException = (filter, exception) => {
|
|
15
|
+
const reflection = Reflect;
|
|
16
|
+
const exceptionTypes = reflection.getMetadata?.(FILTER_CATCH_EXCEPTIONS, filter.constructor);
|
|
17
|
+
if (!Array.isArray(exceptionTypes) || exceptionTypes.length === 0) {
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
return exceptionTypes.some((exceptionType) => typeof exceptionType === "function" &&
|
|
21
|
+
exception instanceof exceptionType);
|
|
22
|
+
};
|
|
13
23
|
let WotchiNestExceptionFilter = class WotchiNestExceptionFilter extends BaseExceptionFilter {
|
|
14
|
-
constructor(client, applicationRef, options) {
|
|
24
|
+
constructor(client, applicationRef, options, previousGlobalFilters = []) {
|
|
15
25
|
super(applicationRef);
|
|
16
26
|
this.client = client;
|
|
17
27
|
this.options = options;
|
|
28
|
+
this.previousGlobalFilters = previousGlobalFilters;
|
|
18
29
|
}
|
|
19
30
|
catch(exception, host) {
|
|
20
|
-
|
|
21
|
-
|
|
31
|
+
captureWotchiNestException(this.client, exception, host, this.options);
|
|
32
|
+
const existingFilter = this.previousGlobalFilters.find((filter) => matchesException(filter, exception));
|
|
33
|
+
if (existingFilter !== undefined) {
|
|
34
|
+
existingFilter.catch(exception, host);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
this.respondWithoutLoggingRawUnknownException(exception, host);
|
|
38
|
+
}
|
|
39
|
+
respondWithoutLoggingRawUnknownException(exception, host) {
|
|
40
|
+
if (this.applicationRef === undefined) {
|
|
41
|
+
this.respondThroughExpressAdapter(exception, host);
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
if (exception instanceof HttpException) {
|
|
45
|
+
super.catch(exception, host);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const response = host.getArgByIndex(1);
|
|
49
|
+
if (this.applicationRef.isHeadersSent(response)) {
|
|
50
|
+
this.applicationRef.end(response);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (this.isHttpError(exception)) {
|
|
54
|
+
this.applicationRef.reply(response, { statusCode: exception.statusCode, message: exception.message }, exception.statusCode);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
this.applicationRef.reply(response, { statusCode: 500, message: "Internal server error" }, 500);
|
|
58
|
+
}
|
|
59
|
+
respondThroughExpressAdapter(exception, host) {
|
|
60
|
+
const response = host.switchToHttp().getResponse();
|
|
61
|
+
if (response.headersSent) {
|
|
62
|
+
response.end();
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (exception instanceof HttpException) {
|
|
66
|
+
const statusCode = exception.getStatus();
|
|
67
|
+
const exceptionResponse = exception.getResponse();
|
|
68
|
+
response
|
|
69
|
+
.status(statusCode)
|
|
70
|
+
.json(typeof exceptionResponse === "object" && exceptionResponse !== null
|
|
71
|
+
? exceptionResponse
|
|
72
|
+
: { statusCode, message: exceptionResponse });
|
|
73
|
+
return;
|
|
22
74
|
}
|
|
23
|
-
|
|
24
|
-
|
|
75
|
+
if (this.isHttpError(exception)) {
|
|
76
|
+
response.status(exception.statusCode).json({
|
|
77
|
+
statusCode: exception.statusCode,
|
|
78
|
+
message: exception.message,
|
|
79
|
+
});
|
|
80
|
+
return;
|
|
25
81
|
}
|
|
26
|
-
|
|
82
|
+
response.status(500).json({ statusCode: 500, message: "Internal server error" });
|
|
27
83
|
}
|
|
28
84
|
};
|
|
29
85
|
WotchiNestExceptionFilter = __decorate([
|
|
30
86
|
Catch(),
|
|
31
|
-
__metadata("design:paramtypes", [Object, Object, Object])
|
|
87
|
+
__metadata("design:paramtypes", [Object, Object, Object, Array])
|
|
32
88
|
], WotchiNestExceptionFilter);
|
|
33
89
|
export { WotchiNestExceptionFilter };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { captureWotchiNestException } from "./capture.js";
|
|
2
|
+
export function withWotchiNestFilter(client, filter, options) {
|
|
3
|
+
const wrapped = Object.create(filter);
|
|
4
|
+
Object.defineProperty(wrapped, "catch", {
|
|
5
|
+
configurable: false,
|
|
6
|
+
enumerable: false,
|
|
7
|
+
value: (exception, host) => {
|
|
8
|
+
captureWotchiNestException(client, exception, host, options);
|
|
9
|
+
filter.catch(exception, host);
|
|
10
|
+
},
|
|
11
|
+
writable: false,
|
|
12
|
+
});
|
|
13
|
+
return wrapped;
|
|
14
|
+
}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
-
export { consoleNotifier, createWotchi, telegramNotifier } from "../../index.js";
|
|
1
|
+
export { consoleNotifier, createWotchi, telegramNotifier, webhookNotifier } from "../../index.js";
|
|
2
|
+
export { withWotchiNestFilter } from "./filter-wrapper.js";
|
|
3
|
+
export { WotchiModule, WOTCHI_CLIENT } from "./module.js";
|
|
2
4
|
export { registerWotchiNest, registerWotchiNestStatusObserver } from "./register.js";
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var WotchiModule_1;
|
|
8
|
+
import { Global, Module } from "@nestjs/common";
|
|
9
|
+
import { APP_FILTER } from "@nestjs/core";
|
|
10
|
+
import { createWotchi } from "../../core/client.js";
|
|
11
|
+
import { WotchiNestExceptionFilter } from "./exception-filter.js";
|
|
12
|
+
export const WOTCHI_CLIENT = Symbol("@futurewindai/wotchi/client");
|
|
13
|
+
let WotchiModule = WotchiModule_1 = class WotchiModule {
|
|
14
|
+
static forRoot(config, options) {
|
|
15
|
+
const registerGlobalFilter = options?.registerGlobalFilter ?? true;
|
|
16
|
+
const providers = [
|
|
17
|
+
{
|
|
18
|
+
provide: WOTCHI_CLIENT,
|
|
19
|
+
useFactory: () => createWotchi(config),
|
|
20
|
+
},
|
|
21
|
+
];
|
|
22
|
+
if (registerGlobalFilter) {
|
|
23
|
+
providers.push({
|
|
24
|
+
provide: APP_FILTER,
|
|
25
|
+
useFactory: (client) => new WotchiNestExceptionFilter(client),
|
|
26
|
+
inject: [WOTCHI_CLIENT],
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
module: WotchiModule_1,
|
|
31
|
+
global: true,
|
|
32
|
+
providers,
|
|
33
|
+
exports: [WOTCHI_CLIENT],
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
WotchiModule = WotchiModule_1 = __decorate([
|
|
38
|
+
Global(),
|
|
39
|
+
Module({})
|
|
40
|
+
], WotchiModule);
|
|
41
|
+
export { WotchiModule };
|
|
@@ -1,13 +1,27 @@
|
|
|
1
1
|
import { HttpAdapterHost } from "@nestjs/core";
|
|
2
2
|
import { WotchiNestExceptionFilter } from "./exception-filter.js";
|
|
3
3
|
import { wotchiStatusObserver } from "../express/status-observer.js";
|
|
4
|
+
const isNestExceptionFilter = (value) => typeof value === "object" &&
|
|
5
|
+
value !== null &&
|
|
6
|
+
"catch" in value &&
|
|
7
|
+
typeof value.catch === "function";
|
|
8
|
+
const getPreviousGlobalFilters = (application) => {
|
|
9
|
+
const configuredFilters = application.config?.getGlobalFilters?.();
|
|
10
|
+
if (!Array.isArray(configuredFilters)) {
|
|
11
|
+
return [];
|
|
12
|
+
}
|
|
13
|
+
return configuredFilters
|
|
14
|
+
.filter(isNestExceptionFilter)
|
|
15
|
+
.filter((filter) => !(filter instanceof WotchiNestExceptionFilter))
|
|
16
|
+
.reverse();
|
|
17
|
+
};
|
|
4
18
|
export function registerWotchiNest(app, client, options) {
|
|
5
19
|
const application = app;
|
|
6
20
|
const directAdapter = application.getHttpAdapter?.();
|
|
7
21
|
const httpAdapter = directAdapter === undefined
|
|
8
22
|
? application.get(HttpAdapterHost).httpAdapter
|
|
9
23
|
: directAdapter;
|
|
10
|
-
application.useGlobalFilters(new WotchiNestExceptionFilter(client, httpAdapter, options));
|
|
24
|
+
application.useGlobalFilters(new WotchiNestExceptionFilter(client, httpAdapter, options, getPreviousGlobalFilters(application)));
|
|
11
25
|
}
|
|
12
26
|
export function registerWotchiNestStatusObserver(app, client, options) {
|
|
13
27
|
const application = app;
|