@futurewindai/wotchi 0.1.0-beta.5 → 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -29
- package/SECURITY.md +3 -3
- package/dist/cjs/core/admission.js +22 -0
- package/dist/cjs/core/client.js +46 -0
- package/dist/cjs/core/config.js +16 -0
- package/dist/cjs/core/diagnostics.js +4 -0
- package/dist/cjs/core/group-store.js +7 -8
- package/dist/cjs/core/limits.js +5 -1
- package/dist/cjs/core/normalize.js +6 -1
- package/dist/cjs/core/notification-queue.js +77 -8
- package/dist/cjs/core/prometheus.js +129 -0
- package/dist/cjs/core/redact.js +4 -1
- package/dist/cjs/core/runtime-monitor.js +109 -0
- package/dist/cjs/index.js +6 -1
- package/dist/cjs/integrations/express/error-handler.js +24 -6
- package/dist/cjs/integrations/express/state.js +7 -20
- package/dist/cjs/integrations/express/status-observer.js +3 -9
- package/dist/cjs/integrations/nest/capture.js +21 -0
- package/dist/cjs/integrations/nest/exception-filter.js +64 -15
- package/dist/cjs/integrations/nest/filter-wrapper.js +19 -0
- package/dist/cjs/integrations/nest/index.js +6 -1
- package/dist/cjs/integrations/nest/module.js +44 -0
- package/dist/cjs/integrations/nest/register.js +17 -1
- package/dist/cjs/integrations/request-context.js +15 -23
- package/dist/esm/core/admission.js +19 -0
- package/dist/esm/core/client.js +46 -0
- package/dist/esm/core/config.js +17 -1
- package/dist/esm/core/diagnostics.js +4 -0
- package/dist/esm/core/group-store.js +7 -8
- package/dist/esm/core/limits.js +4 -0
- package/dist/esm/core/normalize.js +6 -1
- package/dist/esm/core/notification-queue.js +78 -9
- package/dist/esm/core/prometheus.js +125 -0
- package/dist/esm/core/redact.js +4 -1
- package/dist/esm/core/runtime-monitor.js +106 -0
- package/dist/esm/index.js +3 -1
- package/dist/esm/integrations/express/error-handler.js +23 -5
- package/dist/esm/integrations/express/state.js +7 -20
- package/dist/esm/integrations/express/status-observer.js +3 -9
- package/dist/esm/integrations/nest/capture.js +17 -0
- package/dist/esm/integrations/nest/exception-filter.js +65 -16
- package/dist/esm/integrations/nest/filter-wrapper.js +16 -0
- package/dist/esm/integrations/nest/index.js +2 -0
- package/dist/esm/integrations/nest/module.js +41 -0
- package/dist/esm/integrations/nest/register.js +17 -1
- package/dist/esm/integrations/request-context.js +14 -23
- package/dist/types/core/admission.d.ts +10 -0
- package/dist/types/core/config.d.ts +10 -0
- package/dist/types/core/diagnostics.d.ts +3 -1
- package/dist/types/core/incident-policy.d.ts +1 -1
- package/dist/types/core/limits.d.ts +4 -0
- package/dist/types/core/notification-queue.d.ts +9 -0
- package/dist/types/core/prometheus.d.ts +7 -0
- package/dist/types/core/runtime-monitor.d.ts +15 -0
- package/dist/types/core/types.d.ts +16 -1
- package/dist/types/index.d.ts +5 -1
- package/dist/types/integrations/express/status-observer.d.ts +1 -1
- package/dist/types/integrations/nest/capture.d.ts +4 -0
- package/dist/types/integrations/nest/exception-filter.d.ts +7 -3
- package/dist/types/integrations/nest/filter-wrapper.d.ts +7 -0
- package/dist/types/integrations/nest/index.d.ts +4 -0
- package/dist/types/integrations/nest/module.d.ts +9 -0
- package/dist/types/integrations/nest/register.d.ts +3 -0
- package/dist/types/integrations/request-context.d.ts +1 -0
- package/dist/types-cjs/core/admission.d.cts +10 -0
- package/dist/types-cjs/core/client.d.cts +1 -1
- package/dist/types-cjs/core/config.d.cts +12 -2
- package/dist/types-cjs/core/diagnostics.d.cts +4 -2
- package/dist/types-cjs/core/fingerprint.d.cts +1 -1
- package/dist/types-cjs/core/group-store.d.cts +1 -1
- package/dist/types-cjs/core/incident-builder.d.cts +2 -2
- package/dist/types-cjs/core/incident-policy.d.cts +2 -2
- package/dist/types-cjs/core/limits.d.cts +4 -0
- package/dist/types-cjs/core/notification-queue.d.cts +10 -1
- package/dist/types-cjs/core/process-monitor.d.cts +1 -1
- package/dist/types-cjs/core/prometheus.d.cts +7 -0
- package/dist/types-cjs/core/redact.d.cts +1 -1
- package/dist/types-cjs/core/runtime-monitor.d.cts +15 -0
- package/dist/types-cjs/core/types.d.cts +16 -1
- package/dist/types-cjs/index.d.cts +15 -11
- package/dist/types-cjs/integrations/express/error-handler.d.cts +2 -2
- package/dist/types-cjs/integrations/express/index.d.cts +8 -8
- package/dist/types-cjs/integrations/express/request-context.d.cts +2 -2
- package/dist/types-cjs/integrations/express/status-observer.d.cts +2 -2
- package/dist/types-cjs/integrations/nest/capture.d.cts +4 -0
- package/dist/types-cjs/integrations/nest/exception-filter.d.cts +8 -4
- package/dist/types-cjs/integrations/nest/filter-wrapper.d.cts +7 -0
- package/dist/types-cjs/integrations/nest/index.d.cts +9 -5
- package/dist/types-cjs/integrations/nest/module.d.cts +9 -0
- package/dist/types-cjs/integrations/nest/register.d.cts +7 -4
- package/dist/types-cjs/integrations/nest/request-context.d.cts +2 -2
- package/dist/types-cjs/integrations/request-context.d.cts +2 -1
- package/dist/types-cjs/notifiers/alert-payload.d.cts +2 -2
- package/dist/types-cjs/notifiers/console.d.cts +1 -1
- package/dist/types-cjs/notifiers/telegram-format.d.cts +1 -1
- package/dist/types-cjs/notifiers/telegram.d.cts +2 -2
- package/dist/types-cjs/notifiers/webhook-http.d.cts +1 -1
- package/dist/types-cjs/notifiers/webhook.d.cts +2 -2
- package/package.json +23 -12
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
const DEFAULT_INTERVAL_MS = 5_000;
|
|
2
|
+
const MIN_INTERVAL_MS = 100;
|
|
3
|
+
const MAX_INTERVAL_MS = 60_000;
|
|
4
|
+
const MAX_THRESHOLD = Number.MAX_SAFE_INTEGER;
|
|
5
|
+
const positiveNumber = (value, fallback, field) => {
|
|
6
|
+
if (value === undefined) {
|
|
7
|
+
return fallback;
|
|
8
|
+
}
|
|
9
|
+
if (!Number.isFinite(value) || value <= 0 || value > MAX_THRESHOLD) {
|
|
10
|
+
throw new RangeError(`${field} must be a positive finite number`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
};
|
|
14
|
+
export function registerWotchiRuntimeWatcher(client, options = {}) {
|
|
15
|
+
const intervalMs = positiveNumber(options.intervalMs, DEFAULT_INTERVAL_MS, "intervalMs");
|
|
16
|
+
if (!Number.isSafeInteger(intervalMs) ||
|
|
17
|
+
intervalMs < MIN_INTERVAL_MS ||
|
|
18
|
+
intervalMs > MAX_INTERVAL_MS) {
|
|
19
|
+
throw new RangeError(`intervalMs must be an integer from ${MIN_INTERVAL_MS} to ${MAX_INTERVAL_MS}`);
|
|
20
|
+
}
|
|
21
|
+
const thresholds = {
|
|
22
|
+
cpuPercent: options.cpuPercent,
|
|
23
|
+
rssBytes: options.rssBytes,
|
|
24
|
+
heapUsedBytes: options.heapUsedBytes,
|
|
25
|
+
eventLoopDelayMs: options.eventLoopDelayMs,
|
|
26
|
+
pendingAlerts: options.pendingAlerts,
|
|
27
|
+
notifierFailures: options.notifierFailures,
|
|
28
|
+
};
|
|
29
|
+
for (const [field, value] of Object.entries(thresholds)) {
|
|
30
|
+
if (value !== undefined) {
|
|
31
|
+
positiveNumber(value, 1, field);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const alertThreshold = options.alertThreshold ?? 3;
|
|
35
|
+
if (!Number.isSafeInteger(alertThreshold) || alertThreshold <= 0 || alertThreshold > 1_000_000) {
|
|
36
|
+
throw new RangeError("alertThreshold must be a positive integer no greater than 1000000");
|
|
37
|
+
}
|
|
38
|
+
let previousCpu = process.cpuUsage();
|
|
39
|
+
let previousTime = process.hrtime.bigint();
|
|
40
|
+
let previousNotifierFailures = client.getDiagnostics().notifierFailures;
|
|
41
|
+
let expectedAt = Date.now() + intervalMs;
|
|
42
|
+
let registered = true;
|
|
43
|
+
const sample = () => {
|
|
44
|
+
if (!registered) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const timestamp = Date.now();
|
|
48
|
+
const elapsedMicros = Math.max(1, Number(process.hrtime.bigint() - previousTime) / 1_000);
|
|
49
|
+
const cpu = process.cpuUsage(previousCpu);
|
|
50
|
+
previousCpu = process.cpuUsage();
|
|
51
|
+
previousTime = process.hrtime.bigint();
|
|
52
|
+
const memory = process.memoryUsage();
|
|
53
|
+
const eventLoopDelayMs = Math.max(0, timestamp - expectedAt);
|
|
54
|
+
expectedAt = timestamp + intervalMs;
|
|
55
|
+
const cpuPercent = ((cpu.user + cpu.system) / elapsedMicros) * 100;
|
|
56
|
+
const diagnostics = client.getDiagnostics();
|
|
57
|
+
const notifierFailures = Math.max(0, diagnostics.notifierFailures - previousNotifierFailures);
|
|
58
|
+
previousNotifierFailures = diagnostics.notifierFailures;
|
|
59
|
+
const readings = [
|
|
60
|
+
{ metric: "cpu-percent", value: cpuPercent, threshold: thresholds.cpuPercent },
|
|
61
|
+
{ metric: "rss-bytes", value: memory.rss, threshold: thresholds.rssBytes },
|
|
62
|
+
{ metric: "heap-used-bytes", value: memory.heapUsed, threshold: thresholds.heapUsedBytes },
|
|
63
|
+
{
|
|
64
|
+
metric: "event-loop-delay-ms",
|
|
65
|
+
value: eventLoopDelayMs,
|
|
66
|
+
threshold: thresholds.eventLoopDelayMs,
|
|
67
|
+
},
|
|
68
|
+
{ metric: "pending-alerts", value: 0, threshold: thresholds.pendingAlerts },
|
|
69
|
+
{
|
|
70
|
+
metric: "notifier-failures",
|
|
71
|
+
value: notifierFailures,
|
|
72
|
+
threshold: thresholds.notifierFailures,
|
|
73
|
+
},
|
|
74
|
+
];
|
|
75
|
+
for (const reading of readings) {
|
|
76
|
+
const value = reading.metric === "pending-alerts" ? diagnostics.pendingAlerts : reading.value;
|
|
77
|
+
if (reading.threshold === undefined || value < reading.threshold) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
client.captureEvent({
|
|
81
|
+
level: "error",
|
|
82
|
+
kind: "runtime-monitor",
|
|
83
|
+
message: `Wotchi runtime pressure: ${reading.metric}`,
|
|
84
|
+
fingerprint: `wotchi.runtime.${reading.metric}`,
|
|
85
|
+
severity: "high",
|
|
86
|
+
alertThreshold,
|
|
87
|
+
context: {
|
|
88
|
+
metric: reading.metric,
|
|
89
|
+
value: Math.round(value * 100) / 100,
|
|
90
|
+
threshold: reading.threshold,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
const timer = setInterval(sample, intervalMs);
|
|
96
|
+
timer.unref?.();
|
|
97
|
+
return {
|
|
98
|
+
unregister: () => {
|
|
99
|
+
if (!registered) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
registered = false;
|
|
103
|
+
clearInterval(timer);
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
package/dist/esm/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { createWotchi } from "./core/client.js";
|
|
2
2
|
import { registerWotchiProcessMonitor } from "./core/process-monitor.js";
|
|
3
|
+
import { createWotchiPrometheusExporter, PROMETHEUS_CONTENT_TYPE } from "./core/prometheus.js";
|
|
4
|
+
import { registerWotchiRuntimeWatcher } from "./core/runtime-monitor.js";
|
|
3
5
|
import { consoleNotifier } from "./notifiers/console.js";
|
|
4
6
|
import { telegramNotifier } from "./notifiers/telegram.js";
|
|
5
7
|
import { webhookNotifier } from "./notifiers/webhook.js";
|
|
6
8
|
export { WotchiConfigurationError } from "./core/errors.js";
|
|
7
|
-
export { consoleNotifier, createWotchi, registerWotchiProcessMonitor, telegramNotifier, webhookNotifier, };
|
|
9
|
+
export { consoleNotifier, createWotchi, createWotchiPrometheusExporter, PROMETHEUS_CONTENT_TYPE, registerWotchiProcessMonitor, registerWotchiRuntimeWatcher, telegramNotifier, webhookNotifier, };
|
|
@@ -1,14 +1,32 @@
|
|
|
1
|
+
import { normalizeRequestContextOptions } from "../request-context.js";
|
|
1
2
|
import { getExpressRequestContext } from "./request-context.js";
|
|
2
3
|
import { markExpressErrorCaptured } from "./state.js";
|
|
3
4
|
export function createExpressErrorHandler(client, options) {
|
|
5
|
+
const normalizedOptions = normalizeRequestContextOptions(options);
|
|
4
6
|
return (error, request, response, next) => {
|
|
5
7
|
markExpressErrorCaptured(request);
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
8
|
+
let captured = false;
|
|
9
|
+
const captureAfterResponse = () => {
|
|
10
|
+
if (captured) {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
captured = true;
|
|
14
|
+
response.off("finish", captureAfterResponse);
|
|
15
|
+
response.off("close", captureAfterResponse);
|
|
16
|
+
try {
|
|
17
|
+
const requestContext = getExpressRequestContext(request, response, normalizedOptions);
|
|
18
|
+
client.captureException(error, undefined, requestContext === undefined ? undefined : { request: requestContext });
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// A capture failure must never change Express error handling.
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
if (response.writableEnded || response.finished) {
|
|
25
|
+
captureAfterResponse();
|
|
9
26
|
}
|
|
10
|
-
|
|
11
|
-
|
|
27
|
+
else {
|
|
28
|
+
response.once("finish", captureAfterResponse);
|
|
29
|
+
response.once("close", captureAfterResponse);
|
|
12
30
|
}
|
|
13
31
|
next(error);
|
|
14
32
|
};
|
|
@@ -1,26 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
const
|
|
1
|
+
const capturedRequests = new WeakSet();
|
|
2
|
+
const asRequestObject = (request) => typeof request === "object" && request !== null ? request : undefined;
|
|
3
3
|
export function markExpressErrorCaptured(request) {
|
|
4
|
-
const
|
|
5
|
-
if (
|
|
4
|
+
const requestObject = asRequestObject(request);
|
|
5
|
+
if (requestObject === undefined) {
|
|
6
6
|
return;
|
|
7
7
|
}
|
|
8
|
-
|
|
9
|
-
markedRequest[WOTCHI_ERROR_CAPTURED] = true;
|
|
10
|
-
}
|
|
11
|
-
catch {
|
|
12
|
-
// A request object can be frozen by host middleware; the observer remains best-effort.
|
|
13
|
-
}
|
|
8
|
+
capturedRequests.add(requestObject);
|
|
14
9
|
}
|
|
15
10
|
export function wasExpressErrorCaptured(request) {
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
return false;
|
|
19
|
-
}
|
|
20
|
-
try {
|
|
21
|
-
return markedRequest[WOTCHI_ERROR_CAPTURED] === true;
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
return false;
|
|
25
|
-
}
|
|
11
|
+
const requestObject = asRequestObject(request);
|
|
12
|
+
return requestObject === undefined ? false : capturedRequests.has(requestObject);
|
|
26
13
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getExpressRequestContext } from "./request-context.js";
|
|
2
|
+
import { normalizeRequestContextOptions } from "../request-context.js";
|
|
2
3
|
import { wasExpressErrorCaptured } from "./state.js";
|
|
3
4
|
const DEFAULT_STATUS_CODES = [401, 403, 429];
|
|
4
5
|
const DEFAULT_STATUS_CLASSES = ["5xx"];
|
|
@@ -35,16 +36,9 @@ const normalizeAlertThreshold = (value) => {
|
|
|
35
36
|
};
|
|
36
37
|
const normalizeOptions = (options) => {
|
|
37
38
|
const alertThreshold = normalizeAlertThreshold(options?.alertThreshold);
|
|
39
|
+
const requestContextOptions = normalizeRequestContextOptions(options);
|
|
38
40
|
return {
|
|
39
|
-
...
|
|
40
|
-
? {}
|
|
41
|
-
: { requestIdProperty: options.requestIdProperty }),
|
|
42
|
-
...(options?.correlationIdProperty === undefined
|
|
43
|
-
? {}
|
|
44
|
-
: { correlationIdProperty: options.correlationIdProperty }),
|
|
45
|
-
...(options?.traceContextProperty === undefined
|
|
46
|
-
? {}
|
|
47
|
-
: { traceContextProperty: options.traceContextProperty }),
|
|
41
|
+
...requestContextOptions,
|
|
48
42
|
statusCodes: normalizeStatuses(options?.statusCodes, DEFAULT_STATUS_CODES),
|
|
49
43
|
statusClasses: normalizeStatusClasses(options?.statusClasses),
|
|
50
44
|
ignoreStatusCodes: normalizeStatuses(options?.ignoreStatusCodes, []),
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { markExpressErrorCaptured } from "../express/state.js";
|
|
2
|
+
import { getNestRequestContext } from "./request-context.js";
|
|
3
|
+
export const captureWotchiNestException = (client, exception, host, options) => {
|
|
4
|
+
try {
|
|
5
|
+
markExpressErrorCaptured(host.switchToHttp().getRequest());
|
|
6
|
+
}
|
|
7
|
+
catch {
|
|
8
|
+
// Non-HTTP NestJS contexts do not expose an Express request to mark.
|
|
9
|
+
}
|
|
10
|
+
try {
|
|
11
|
+
const requestContext = getNestRequestContext(host, exception, options);
|
|
12
|
+
client.captureException(exception, undefined, requestContext === undefined ? undefined : { request: requestContext });
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
// A capture failure must never change NestJS exception handling.
|
|
16
|
+
}
|
|
17
|
+
};
|
|
@@ -7,35 +7,84 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
-
import { Catch } from "@nestjs/common";
|
|
10
|
+
import { Catch, HttpException } from "@nestjs/common";
|
|
11
11
|
import { BaseExceptionFilter } from "@nestjs/core";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { normalizeRequestContextOptions } from "../request-context.js";
|
|
13
|
+
import { captureWotchiNestException } from "./capture.js";
|
|
14
|
+
const FILTER_CATCH_EXCEPTIONS = "__filterCatchExceptions__";
|
|
15
|
+
const matchesException = (filter, exception) => {
|
|
16
|
+
const reflection = Reflect;
|
|
17
|
+
const exceptionTypes = reflection.getMetadata?.(FILTER_CATCH_EXCEPTIONS, filter.constructor);
|
|
18
|
+
if (!Array.isArray(exceptionTypes) || exceptionTypes.length === 0) {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
return exceptionTypes.some((exceptionType) => typeof exceptionType === "function" &&
|
|
22
|
+
exception instanceof exceptionType);
|
|
23
|
+
};
|
|
14
24
|
let WotchiNestExceptionFilter = class WotchiNestExceptionFilter extends BaseExceptionFilter {
|
|
15
|
-
constructor(client, applicationRef, options) {
|
|
25
|
+
constructor(client, applicationRef, options, previousGlobalFilters = []) {
|
|
16
26
|
super(applicationRef);
|
|
17
27
|
this.client = client;
|
|
18
|
-
this.
|
|
28
|
+
this.previousGlobalFilters = previousGlobalFilters;
|
|
29
|
+
this.options = normalizeRequestContextOptions(options);
|
|
19
30
|
}
|
|
20
31
|
catch(exception, host) {
|
|
21
|
-
|
|
22
|
-
|
|
32
|
+
captureWotchiNestException(this.client, exception, host, this.options);
|
|
33
|
+
const existingFilter = this.previousGlobalFilters.find((filter) => matchesException(filter, exception));
|
|
34
|
+
if (existingFilter !== undefined) {
|
|
35
|
+
existingFilter.catch(exception, host);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
this.respondWithoutLoggingRawUnknownException(exception, host);
|
|
39
|
+
}
|
|
40
|
+
respondWithoutLoggingRawUnknownException(exception, host) {
|
|
41
|
+
if (this.applicationRef === undefined) {
|
|
42
|
+
this.respondThroughExpressAdapter(exception, host);
|
|
43
|
+
return;
|
|
23
44
|
}
|
|
24
|
-
|
|
25
|
-
|
|
45
|
+
if (exception instanceof HttpException) {
|
|
46
|
+
super.catch(exception, host);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const response = host.getArgByIndex(1);
|
|
50
|
+
if (this.applicationRef.isHeadersSent(response)) {
|
|
51
|
+
this.applicationRef.end(response);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (this.isHttpError(exception)) {
|
|
55
|
+
this.applicationRef.reply(response, { statusCode: exception.statusCode, message: exception.message }, exception.statusCode);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
this.applicationRef.reply(response, { statusCode: 500, message: "Internal server error" }, 500);
|
|
59
|
+
}
|
|
60
|
+
respondThroughExpressAdapter(exception, host) {
|
|
61
|
+
const response = host.switchToHttp().getResponse();
|
|
62
|
+
if (response.headersSent) {
|
|
63
|
+
response.end();
|
|
64
|
+
return;
|
|
26
65
|
}
|
|
27
|
-
|
|
28
|
-
const
|
|
29
|
-
|
|
66
|
+
if (exception instanceof HttpException) {
|
|
67
|
+
const statusCode = exception.getStatus();
|
|
68
|
+
const exceptionResponse = exception.getResponse();
|
|
69
|
+
response
|
|
70
|
+
.status(statusCode)
|
|
71
|
+
.json(typeof exceptionResponse === "object" && exceptionResponse !== null
|
|
72
|
+
? exceptionResponse
|
|
73
|
+
: { statusCode, message: exceptionResponse });
|
|
74
|
+
return;
|
|
30
75
|
}
|
|
31
|
-
|
|
32
|
-
|
|
76
|
+
if (this.isHttpError(exception)) {
|
|
77
|
+
response.status(exception.statusCode).json({
|
|
78
|
+
statusCode: exception.statusCode,
|
|
79
|
+
message: exception.message,
|
|
80
|
+
});
|
|
81
|
+
return;
|
|
33
82
|
}
|
|
34
|
-
|
|
83
|
+
response.status(500).json({ statusCode: 500, message: "Internal server error" });
|
|
35
84
|
}
|
|
36
85
|
};
|
|
37
86
|
WotchiNestExceptionFilter = __decorate([
|
|
38
87
|
Catch(),
|
|
39
|
-
__metadata("design:paramtypes", [Object, Object, Object])
|
|
88
|
+
__metadata("design:paramtypes", [Object, Object, Object, Array])
|
|
40
89
|
], WotchiNestExceptionFilter);
|
|
41
90
|
export { WotchiNestExceptionFilter };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { normalizeRequestContextOptions } from "../request-context.js";
|
|
2
|
+
import { captureWotchiNestException } from "./capture.js";
|
|
3
|
+
export function withWotchiNestFilter(client, filter, options) {
|
|
4
|
+
const normalizedOptions = normalizeRequestContextOptions(options);
|
|
5
|
+
const wrapped = Object.create(filter);
|
|
6
|
+
Object.defineProperty(wrapped, "catch", {
|
|
7
|
+
configurable: false,
|
|
8
|
+
enumerable: false,
|
|
9
|
+
value: (exception, host) => {
|
|
10
|
+
captureWotchiNestException(client, exception, host, normalizedOptions);
|
|
11
|
+
filter.catch(exception, host);
|
|
12
|
+
},
|
|
13
|
+
writable: false,
|
|
14
|
+
});
|
|
15
|
+
return wrapped;
|
|
16
|
+
}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
export { consoleNotifier, createWotchi, telegramNotifier, webhookNotifier } from "../../index.js";
|
|
2
|
+
export { withWotchiNestFilter } from "./filter-wrapper.js";
|
|
3
|
+
export { WotchiModule, WOTCHI_CLIENT } from "./module.js";
|
|
2
4
|
export { registerWotchiNest, registerWotchiNestStatusObserver } from "./register.js";
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var WotchiModule_1;
|
|
8
|
+
import { Global, Module } from "@nestjs/common";
|
|
9
|
+
import { APP_FILTER } from "@nestjs/core";
|
|
10
|
+
import { createWotchi } from "../../core/client.js";
|
|
11
|
+
import { WotchiNestExceptionFilter } from "./exception-filter.js";
|
|
12
|
+
export const WOTCHI_CLIENT = Symbol("@futurewindai/wotchi/client");
|
|
13
|
+
let WotchiModule = WotchiModule_1 = class WotchiModule {
|
|
14
|
+
static forRoot(config, options) {
|
|
15
|
+
const registerGlobalFilter = options?.registerGlobalFilter ?? true;
|
|
16
|
+
const providers = [
|
|
17
|
+
{
|
|
18
|
+
provide: WOTCHI_CLIENT,
|
|
19
|
+
useFactory: () => createWotchi(config),
|
|
20
|
+
},
|
|
21
|
+
];
|
|
22
|
+
if (registerGlobalFilter) {
|
|
23
|
+
providers.push({
|
|
24
|
+
provide: APP_FILTER,
|
|
25
|
+
useFactory: (client) => new WotchiNestExceptionFilter(client),
|
|
26
|
+
inject: [WOTCHI_CLIENT],
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
module: WotchiModule_1,
|
|
31
|
+
global: true,
|
|
32
|
+
providers,
|
|
33
|
+
exports: [WOTCHI_CLIENT],
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
WotchiModule = WotchiModule_1 = __decorate([
|
|
38
|
+
Global(),
|
|
39
|
+
Module({})
|
|
40
|
+
], WotchiModule);
|
|
41
|
+
export { WotchiModule };
|
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
import { HttpAdapterHost } from "@nestjs/core";
|
|
2
|
+
import { normalizeRequestContextOptions } from "../request-context.js";
|
|
2
3
|
import { WotchiNestExceptionFilter } from "./exception-filter.js";
|
|
3
4
|
import { wotchiStatusObserver } from "../express/status-observer.js";
|
|
5
|
+
const isNestExceptionFilter = (value) => typeof value === "object" &&
|
|
6
|
+
value !== null &&
|
|
7
|
+
"catch" in value &&
|
|
8
|
+
typeof value.catch === "function";
|
|
9
|
+
const getPreviousGlobalFilters = (application) => {
|
|
10
|
+
const configuredFilters = application.config?.getGlobalFilters?.();
|
|
11
|
+
if (!Array.isArray(configuredFilters)) {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
return configuredFilters
|
|
15
|
+
.filter(isNestExceptionFilter)
|
|
16
|
+
.filter((filter) => !(filter instanceof WotchiNestExceptionFilter))
|
|
17
|
+
.reverse();
|
|
18
|
+
};
|
|
4
19
|
export function registerWotchiNest(app, client, options) {
|
|
20
|
+
const normalizedOptions = normalizeRequestContextOptions(options);
|
|
5
21
|
const application = app;
|
|
6
22
|
const directAdapter = application.getHttpAdapter?.();
|
|
7
23
|
const httpAdapter = directAdapter === undefined
|
|
8
24
|
? application.get(HttpAdapterHost).httpAdapter
|
|
9
25
|
: directAdapter;
|
|
10
|
-
application.useGlobalFilters(new WotchiNestExceptionFilter(client, httpAdapter,
|
|
26
|
+
application.useGlobalFilters(new WotchiNestExceptionFilter(client, httpAdapter, normalizedOptions, getPreviousGlobalFilters(application)));
|
|
11
27
|
}
|
|
12
28
|
export function registerWotchiNestStatusObserver(app, client, options) {
|
|
13
29
|
const application = app;
|
|
@@ -20,33 +20,25 @@ export function normalizeRoutePath(value) {
|
|
|
20
20
|
}
|
|
21
21
|
return path.split("/").map(dynamicSegment).join("/").slice(0, MAX_ROUTE_LENGTH);
|
|
22
22
|
}
|
|
23
|
-
const
|
|
23
|
+
const normalizePropertyName = (property, optionName) => {
|
|
24
24
|
if (property === undefined) {
|
|
25
25
|
return undefined;
|
|
26
26
|
}
|
|
27
27
|
if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(property)) {
|
|
28
|
-
throw new TypeError(
|
|
29
|
-
}
|
|
30
|
-
return property;
|
|
31
|
-
};
|
|
32
|
-
const normalizeTraceContextProperty = (property) => {
|
|
33
|
-
if (property === undefined) {
|
|
34
|
-
return undefined;
|
|
35
|
-
}
|
|
36
|
-
if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(property)) {
|
|
37
|
-
throw new TypeError("traceContextProperty must be a simple property name");
|
|
38
|
-
}
|
|
39
|
-
return property;
|
|
40
|
-
};
|
|
41
|
-
const normalizeCorrelationIdProperty = (property) => {
|
|
42
|
-
if (property === undefined) {
|
|
43
|
-
return undefined;
|
|
44
|
-
}
|
|
45
|
-
if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(property)) {
|
|
46
|
-
throw new TypeError("correlationIdProperty must be a simple property name");
|
|
28
|
+
throw new TypeError(`${optionName} must be a simple property name`);
|
|
47
29
|
}
|
|
48
30
|
return property;
|
|
49
31
|
};
|
|
32
|
+
export function normalizeRequestContextOptions(options) {
|
|
33
|
+
const requestIdProperty = normalizePropertyName(options?.requestIdProperty, "requestIdProperty");
|
|
34
|
+
const correlationIdProperty = normalizePropertyName(options?.correlationIdProperty, "correlationIdProperty");
|
|
35
|
+
const traceContextProperty = normalizePropertyName(options?.traceContextProperty, "traceContextProperty");
|
|
36
|
+
return Object.freeze({
|
|
37
|
+
...(requestIdProperty === undefined ? {} : { requestIdProperty }),
|
|
38
|
+
...(correlationIdProperty === undefined ? {} : { correlationIdProperty }),
|
|
39
|
+
...(traceContextProperty === undefined ? {} : { traceContextProperty }),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
50
42
|
const readRequestId = (request, property) => {
|
|
51
43
|
if (property === undefined || !isRecord(request)) {
|
|
52
44
|
return undefined;
|
|
@@ -104,9 +96,8 @@ const readCorrelationId = (request, property) => {
|
|
|
104
96
|
}
|
|
105
97
|
};
|
|
106
98
|
export function buildRequestContext(input) {
|
|
107
|
-
const
|
|
108
|
-
const correlationIdProperty =
|
|
109
|
-
const traceContextProperty = normalizeTraceContextProperty(input.options?.traceContextProperty);
|
|
99
|
+
const options = normalizeRequestContextOptions(input.options);
|
|
100
|
+
const { requestIdProperty, correlationIdProperty, traceContextProperty } = options;
|
|
110
101
|
let requestId;
|
|
111
102
|
try {
|
|
112
103
|
requestId = readRequestId(input.request, requestIdProperty);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface CaptureAdmissionOptions {
|
|
2
|
+
maxEventsPerSecond: number;
|
|
3
|
+
burst: number;
|
|
4
|
+
now?: () => number;
|
|
5
|
+
}
|
|
6
|
+
export interface CaptureAdmission {
|
|
7
|
+
tryAcquire(): boolean;
|
|
8
|
+
}
|
|
9
|
+
/** A small token bucket used before normalization to bound overload work. */
|
|
10
|
+
export declare function createCaptureAdmission(options: CaptureAdmissionOptions): CaptureAdmission;
|
|
@@ -22,6 +22,16 @@ export interface NormalizedWotchiConfig {
|
|
|
22
22
|
readonly queue: {
|
|
23
23
|
readonly maxPendingAlerts: number;
|
|
24
24
|
readonly concurrency: 1;
|
|
25
|
+
readonly notifierTimeoutMs: number;
|
|
26
|
+
readonly notifierCircuitBreaker: {
|
|
27
|
+
readonly failureThreshold: number;
|
|
28
|
+
readonly cooldownMs: number;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
readonly overload?: {
|
|
32
|
+
readonly maxEventsPerSecond: number;
|
|
33
|
+
readonly burst: number;
|
|
34
|
+
readonly alertCooldownMs: number;
|
|
25
35
|
};
|
|
26
36
|
readonly privacy: {
|
|
27
37
|
readonly redactKeys: readonly string[];
|
|
@@ -6,6 +6,8 @@ export interface DiagnosticsState {
|
|
|
6
6
|
filterFailures: number;
|
|
7
7
|
beforeSendFailures: number;
|
|
8
8
|
eventsSuppressed: number;
|
|
9
|
+
eventsDroppedOverload: number;
|
|
10
|
+
capturesAfterShutdown: number;
|
|
9
11
|
}
|
|
10
12
|
export declare function createDiagnosticsState(): DiagnosticsState;
|
|
11
|
-
export declare function snapshotDiagnostics(state: DiagnosticsState, values: Omit<WotchiDiagnostics, "capturedEvents" | "captureFailures" | "fingerprintCallbackFailures" | "filterFailures" | "beforeSendFailures" | "eventsSuppressed">): Readonly<WotchiDiagnostics>;
|
|
13
|
+
export declare function snapshotDiagnostics(state: DiagnosticsState, values: Omit<WotchiDiagnostics, "capturedEvents" | "captureFailures" | "fingerprintCallbackFailures" | "filterFailures" | "beforeSendFailures" | "eventsSuppressed" | "eventsDroppedOverload" | "capturesAfterShutdown">): Readonly<WotchiDiagnostics>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { IncidentGroup, IncidentSeverity } from "./types.js";
|
|
2
|
-
export type IncidentEventKind = "error" | "manual" | "process-monitor";
|
|
2
|
+
export type IncidentEventKind = "error" | "manual" | "process-monitor" | "runtime-monitor";
|
|
3
3
|
export interface IncidentPolicyInput {
|
|
4
4
|
group: IncidentGroup;
|
|
5
5
|
now: number;
|
|
@@ -4,6 +4,10 @@ export declare const MAX_PENDING_ALERTS = 10000;
|
|
|
4
4
|
export declare const MAX_WINDOW_MS: number;
|
|
5
5
|
export declare const MAX_ALERT_THRESHOLD = 1000000;
|
|
6
6
|
export declare const MAX_COOLDOWN_MS: number;
|
|
7
|
+
export declare const MAX_CAPTURE_RATE = 1000000;
|
|
8
|
+
export declare const MAX_NOTIFIER_TIMEOUT_MS = 60000;
|
|
9
|
+
export declare const MAX_CIRCUIT_BREAKER_COOLDOWN_MS: number;
|
|
10
|
+
export declare const MAX_CIRCUIT_BREAKER_FAILURES = 100;
|
|
7
11
|
export declare const MAX_NORMALIZATION_DEPTH = 20;
|
|
8
12
|
export declare const MAX_NORMALIZATION_KEYS = 10000;
|
|
9
13
|
export declare const MAX_NORMALIZATION_STRING_LENGTH = 32768;
|
|
@@ -2,6 +2,11 @@ import type { IncidentAlert, WotchiNotifier } from "./types.js";
|
|
|
2
2
|
export interface NotificationQueueOptions {
|
|
3
3
|
maxPendingAlerts: number;
|
|
4
4
|
concurrency: 1;
|
|
5
|
+
notifierTimeoutMs?: number;
|
|
6
|
+
notifierCircuitBreaker?: {
|
|
7
|
+
failureThreshold: number;
|
|
8
|
+
cooldownMs: number;
|
|
9
|
+
};
|
|
5
10
|
onNotifierError?: (error: unknown, notifier: WotchiNotifier) => void;
|
|
6
11
|
}
|
|
7
12
|
export interface NotificationJobResult {
|
|
@@ -11,10 +16,14 @@ export interface NotificationJobResult {
|
|
|
11
16
|
export interface NotificationQueue {
|
|
12
17
|
enqueue(alert: IncidentAlert, notifiers: readonly WotchiNotifier[], onComplete?: (result: NotificationJobResult) => void): boolean;
|
|
13
18
|
flush(timeoutMs?: number): Promise<void>;
|
|
19
|
+
close(timeoutMs?: number): Promise<void>;
|
|
20
|
+
isClosed(): boolean;
|
|
14
21
|
pending(): number;
|
|
15
22
|
alertsQueued(): number;
|
|
16
23
|
alertsDropped(): number;
|
|
17
24
|
alertsSent(): number;
|
|
18
25
|
notifierFailures(): number;
|
|
26
|
+
notifierTimeouts(): number;
|
|
27
|
+
notifierCircuitOpenSkips(): number;
|
|
19
28
|
}
|
|
20
29
|
export declare function createNotificationQueue(options: NotificationQueueOptions): NotificationQueue;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { WotchiClient } from "./types.js";
|
|
2
|
+
export declare const PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8";
|
|
3
|
+
export interface WotchiPrometheusExporter {
|
|
4
|
+
readonly contentType: typeof PROMETHEUS_CONTENT_TYPE;
|
|
5
|
+
render(): string;
|
|
6
|
+
}
|
|
7
|
+
export declare function createWotchiPrometheusExporter(client: Pick<WotchiClient, "getDiagnostics">): WotchiPrometheusExporter;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { WotchiClient } from "./types.js";
|
|
2
|
+
export interface WotchiRuntimeWatcherOptions {
|
|
3
|
+
intervalMs?: number;
|
|
4
|
+
cpuPercent?: number;
|
|
5
|
+
rssBytes?: number;
|
|
6
|
+
heapUsedBytes?: number;
|
|
7
|
+
eventLoopDelayMs?: number;
|
|
8
|
+
pendingAlerts?: number;
|
|
9
|
+
notifierFailures?: number;
|
|
10
|
+
alertThreshold?: number;
|
|
11
|
+
}
|
|
12
|
+
export interface WotchiRuntimeWatcherHandle {
|
|
13
|
+
unregister(): void;
|
|
14
|
+
}
|
|
15
|
+
export declare function registerWotchiRuntimeWatcher(client: Pick<WotchiClient, "captureEvent" | "getDiagnostics">, options?: WotchiRuntimeWatcherOptions): WotchiRuntimeWatcherHandle;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type IncidentSeverity = "low" | "medium" | "high" | "critical";
|
|
2
|
-
export type WotchiEventKind = "error" | "manual" | "process-monitor";
|
|
2
|
+
export type WotchiEventKind = "error" | "manual" | "process-monitor" | "runtime-monitor";
|
|
3
3
|
export interface WotchiTraceContext {
|
|
4
4
|
traceId?: string;
|
|
5
5
|
spanId?: string;
|
|
@@ -160,6 +160,16 @@ export interface WotchiConfig {
|
|
|
160
160
|
queue?: {
|
|
161
161
|
maxPendingAlerts?: number;
|
|
162
162
|
concurrency?: 1;
|
|
163
|
+
notifierTimeoutMs?: number;
|
|
164
|
+
notifierCircuitBreaker?: {
|
|
165
|
+
failureThreshold?: number;
|
|
166
|
+
cooldownMs?: number;
|
|
167
|
+
};
|
|
168
|
+
};
|
|
169
|
+
overload?: {
|
|
170
|
+
maxEventsPerSecond?: number;
|
|
171
|
+
burst?: number;
|
|
172
|
+
alertCooldownMs?: number;
|
|
163
173
|
};
|
|
164
174
|
privacy?: {
|
|
165
175
|
redactKeys?: string[];
|
|
@@ -182,8 +192,12 @@ export interface WotchiDiagnostics {
|
|
|
182
192
|
filterFailures: number;
|
|
183
193
|
beforeSendFailures: number;
|
|
184
194
|
eventsSuppressed: number;
|
|
195
|
+
eventsDroppedOverload: number;
|
|
196
|
+
capturesAfterShutdown: number;
|
|
185
197
|
activeGroups: number;
|
|
186
198
|
pendingAlerts: number;
|
|
199
|
+
notifierTimeouts: number;
|
|
200
|
+
notifierCircuitOpenSkips: number;
|
|
187
201
|
}
|
|
188
202
|
export type WotchiTestAlertStatus = "sent" | "queue-full" | "timeout" | "notifier-failed";
|
|
189
203
|
export interface WotchiTestAlertResult {
|
|
@@ -201,5 +215,6 @@ export interface WotchiClient {
|
|
|
201
215
|
captureEvent(event: WotchiEventInput): void;
|
|
202
216
|
testAlert(): Promise<WotchiTestAlertResult>;
|
|
203
217
|
flush(timeoutMs?: number): Promise<void>;
|
|
218
|
+
shutdown(timeoutMs?: number): Promise<void>;
|
|
204
219
|
getDiagnostics(): Readonly<WotchiDiagnostics>;
|
|
205
220
|
}
|