@futurewindai/wotchi 0.1.0-beta.5 → 0.1.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -24
- package/SECURITY.md +2 -2
- package/dist/cjs/core/admission.js +22 -0
- package/dist/cjs/core/client.js +46 -0
- package/dist/cjs/core/config.js +16 -0
- package/dist/cjs/core/diagnostics.js +4 -0
- package/dist/cjs/core/group-store.js +7 -8
- package/dist/cjs/core/limits.js +5 -1
- package/dist/cjs/core/normalize.js +6 -1
- package/dist/cjs/core/notification-queue.js +77 -8
- package/dist/cjs/core/prometheus.js +129 -0
- package/dist/cjs/core/redact.js +4 -1
- package/dist/cjs/core/runtime-monitor.js +109 -0
- package/dist/cjs/index.js +6 -1
- package/dist/cjs/integrations/express/error-handler.js +21 -5
- package/dist/cjs/integrations/nest/capture.js +21 -0
- package/dist/cjs/integrations/nest/exception-filter.js +62 -14
- package/dist/cjs/integrations/nest/filter-wrapper.js +17 -0
- package/dist/cjs/integrations/nest/index.js +6 -1
- package/dist/cjs/integrations/nest/module.js +44 -0
- package/dist/cjs/integrations/nest/register.js +15 -1
- package/dist/esm/core/admission.js +19 -0
- package/dist/esm/core/client.js +46 -0
- package/dist/esm/core/config.js +17 -1
- package/dist/esm/core/diagnostics.js +4 -0
- package/dist/esm/core/group-store.js +7 -8
- package/dist/esm/core/limits.js +4 -0
- package/dist/esm/core/normalize.js +6 -1
- package/dist/esm/core/notification-queue.js +78 -9
- package/dist/esm/core/prometheus.js +125 -0
- package/dist/esm/core/redact.js +4 -1
- package/dist/esm/core/runtime-monitor.js +106 -0
- package/dist/esm/index.js +3 -1
- package/dist/esm/integrations/express/error-handler.js +21 -5
- package/dist/esm/integrations/nest/capture.js +17 -0
- package/dist/esm/integrations/nest/exception-filter.js +63 -15
- package/dist/esm/integrations/nest/filter-wrapper.js +14 -0
- package/dist/esm/integrations/nest/index.js +2 -0
- package/dist/esm/integrations/nest/module.js +41 -0
- package/dist/esm/integrations/nest/register.js +15 -1
- package/dist/types/core/admission.d.ts +10 -0
- package/dist/types/core/config.d.ts +10 -0
- package/dist/types/core/diagnostics.d.ts +3 -1
- package/dist/types/core/incident-policy.d.ts +1 -1
- package/dist/types/core/limits.d.ts +4 -0
- package/dist/types/core/notification-queue.d.ts +9 -0
- package/dist/types/core/prometheus.d.ts +7 -0
- package/dist/types/core/runtime-monitor.d.ts +15 -0
- package/dist/types/core/types.d.ts +16 -1
- package/dist/types/index.d.ts +5 -1
- package/dist/types/integrations/nest/capture.d.ts +4 -0
- package/dist/types/integrations/nest/exception-filter.d.ts +6 -2
- package/dist/types/integrations/nest/filter-wrapper.d.ts +7 -0
- package/dist/types/integrations/nest/index.d.ts +4 -0
- package/dist/types/integrations/nest/module.d.ts +9 -0
- package/dist/types/integrations/nest/register.d.ts +3 -0
- package/dist/types-cjs/core/admission.d.cts +10 -0
- package/dist/types-cjs/core/config.d.cts +10 -0
- package/dist/types-cjs/core/diagnostics.d.cts +3 -1
- package/dist/types-cjs/core/incident-policy.d.cts +1 -1
- package/dist/types-cjs/core/limits.d.cts +4 -0
- package/dist/types-cjs/core/notification-queue.d.cts +9 -0
- package/dist/types-cjs/core/prometheus.d.cts +7 -0
- package/dist/types-cjs/core/runtime-monitor.d.cts +15 -0
- package/dist/types-cjs/core/types.d.cts +16 -1
- package/dist/types-cjs/index.d.cts +5 -1
- package/dist/types-cjs/integrations/nest/capture.d.cts +4 -0
- package/dist/types-cjs/integrations/nest/exception-filter.d.cts +6 -2
- package/dist/types-cjs/integrations/nest/filter-wrapper.d.cts +7 -0
- package/dist/types-cjs/integrations/nest/index.d.cts +4 -0
- package/dist/types-cjs/integrations/nest/module.d.cts +9 -0
- package/dist/types-cjs/integrations/nest/register.d.cts +3 -0
- package/package.json +12 -1
package/README.md
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
> Low-noise error alerts for Node.js services.
|
|
4
4
|
|
|
5
|
-
> **Status:** Public beta (`0.1.0-beta.
|
|
5
|
+
> **Status:** Public beta (`0.1.0-beta.6`). The API may evolve before the first stable release.
|
|
6
6
|
|
|
7
7
|
[](https://github.com/FutureWindAI/Wotchi/actions/workflows/ci.yml)
|
|
8
8
|
[](https://github.com/FutureWindAI/Wotchi/actions/workflows/codeql.yml)
|
|
9
9
|
[](https://www.npmjs.com/package/@futurewindai/wotchi)
|
|
10
|
-
[](LICENSE)
|
|
10
|
+
[](https://github.com/FutureWindAI/Wotchi/blob/v0.1.0-beta.6/LICENSE)
|
|
11
11
|
[](https://nodejs.org/en/about/previous-releases)
|
|
12
12
|
|
|
13
13
|
Wotchi captures application errors in-process, removes sensitive values, groups repeated failures, and delivers bounded console, Telegram, or HTTPS webhook alerts without changing framework response handling. It is a signal-conditioning layer, not a replacement for a full observability platform.
|
|
@@ -67,6 +67,45 @@ notifier.
|
|
|
67
67
|
| Generic HTTPS webhook notifier | Route bounded JSON alerts to an existing internal alert destination. |
|
|
68
68
|
| Actionable context and trace passthrough | Include route, release, request/correlation IDs, operation/job, safe tags, links, and existing trace/span IDs. |
|
|
69
69
|
| Optional status observation and JSON logs | Observe direct `401`/`403`/`429`/`5xx` responses and emit collector-friendly JSON. |
|
|
70
|
+
| Optional diagnostics exporter | Expose aggregate Wotchi health counters to an existing Prometheus-compatible `/metrics` endpoint. |
|
|
71
|
+
| Optional overload admission | Drop capture work before normalization at a measured rate and emit one sanitized overload signal. |
|
|
72
|
+
| Notifier isolation and shutdown | Bound slow destinations, open circuits after repeated failures, and drain safely during termination. |
|
|
73
|
+
| Optional runtime pressure watcher | Threshold CPU, RSS, heap, event-loop delay, or pending alerts with numeric-only local samples. |
|
|
74
|
+
|
|
75
|
+
## When to use Wotchi
|
|
76
|
+
|
|
77
|
+
Use Wotchi when an Express or NestJS service needs low-noise, privacy-conscious alerts without
|
|
78
|
+
adding a collector, database, or hosted account. Keep a full observability platform when you need
|
|
79
|
+
durable incident history, cross-replica deduplication, dashboards, traces, or paging workflows.
|
|
80
|
+
|
|
81
|
+
## Performance evidence
|
|
82
|
+
|
|
83
|
+
The package has zero direct runtime dependencies and keeps capture, grouping, and notification work
|
|
84
|
+
bounded. On a Node.js 22/macOS arm64 benchmark run of beta.6, capture p95/p99 was
|
|
85
|
+
`0.036 ms`/`0.036 ms`, duplicate-storm retained heap was `0.194 MiB`, and the packed artifact was
|
|
86
|
+
`60,534 bytes`. Prometheus rendering p95 was `0.0048 ms`. The queue benchmark admitted `101` alerts while a notifier was blocked, capped
|
|
87
|
+
pending work at `100`, and kept Express and NestJS responses completing. These are reproducible local
|
|
88
|
+
measurements, not production guarantees; repeat `npm run benchmark` and `npm run benchmark:queue`
|
|
89
|
+
on your runtime and workload. See [performance evidence](docs/PERFORMANCE.md).
|
|
90
|
+
|
|
91
|
+
The exporter does not open a server or send data. Attach it to a protected endpoint owned
|
|
92
|
+
by the host application:
|
|
93
|
+
|
|
94
|
+
```ts
|
|
95
|
+
import { createWotchiPrometheusExporter } from "@futurewindai/wotchi";
|
|
96
|
+
|
|
97
|
+
const metrics = createWotchiPrometheusExporter(wotchi);
|
|
98
|
+
app.get("/metrics", (_request, response) => {
|
|
99
|
+
response.setHeader("Content-Type", metrics.contentType);
|
|
100
|
+
response.send(metrics.render());
|
|
101
|
+
});
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Beta.6 also supports opt-in admission control, notifier timeouts/circuits, graceful shutdown, and a
|
|
105
|
+
numeric-only runtime watcher.
|
|
106
|
+
|
|
107
|
+
It emits only aggregate counters and queue/group gauges. Prometheus, Grafana, or another existing
|
|
108
|
+
collector remains responsible for scraping, authentication, retention, and dashboards.
|
|
70
109
|
|
|
71
110
|
## How it works
|
|
72
111
|
|
|
@@ -102,14 +141,29 @@ const wotchi = createWotchi({
|
|
|
102
141
|
app.use(wotchiErrorHandler(wotchi));
|
|
103
142
|
```
|
|
104
143
|
|
|
105
|
-
NestJS applications register
|
|
144
|
+
NestJS applications can register Wotchi from the root module, keeping `main.ts` focused on
|
|
145
|
+
application bootstrap:
|
|
106
146
|
|
|
107
147
|
```ts
|
|
108
|
-
import {
|
|
109
|
-
|
|
110
|
-
|
|
148
|
+
import { Module } from "@nestjs/common";
|
|
149
|
+
import { consoleNotifier } from "@futurewindai/wotchi";
|
|
150
|
+
import { WotchiModule } from "@futurewindai/wotchi/nest";
|
|
151
|
+
|
|
152
|
+
@Module({
|
|
153
|
+
imports: [
|
|
154
|
+
WotchiModule.forRoot({
|
|
155
|
+
service: "orders-api",
|
|
156
|
+
environment: "production",
|
|
157
|
+
notifiers: [consoleNotifier()],
|
|
158
|
+
}),
|
|
159
|
+
],
|
|
160
|
+
})
|
|
161
|
+
export class AppModule {}
|
|
111
162
|
```
|
|
112
163
|
|
|
164
|
+
For a custom dependency-injected `APP_FILTER`, keep its response and logging behavior, disable the
|
|
165
|
+
automatic Wotchi filter, and wrap it with `withWotchiNestFilter`. See the [NestJS API](docs/API.md#nestjs-entry-point).
|
|
166
|
+
|
|
113
167
|
The package keeps compatibility with Node.js `>=18.18.0` and is tested across Node.js 18–26. Node.js 22 or 24 LTS is recommended for production. Express and NestJS adapters are optional subpath integrations, so applications only load the framework adapter they use.
|
|
114
168
|
|
|
115
169
|
## Telegram alerts
|
|
@@ -140,7 +194,7 @@ function requiredEnv(name: string): string {
|
|
|
140
194
|
}
|
|
141
195
|
```
|
|
142
196
|
|
|
143
|
-
Wotchi does not ship a shared bot token. Delivery is queued outside the request path and sends only the sanitized incident alert. See [configuration](docs/CONFIGURATION.md) for notifier and security options.
|
|
197
|
+
Wotchi does not ship a shared bot token. Delivery is queued outside the request path and sends only the sanitized incident alert. See [configuration](https://github.com/FutureWindAI/Wotchi/blob/main/docs/CONFIGURATION.md) for notifier and security options.
|
|
144
198
|
|
|
145
199
|
## HTTPS webhook alerts
|
|
146
200
|
|
|
@@ -206,7 +260,7 @@ Wotchi keeps grouping and cooldown state in one process. Replicas have independe
|
|
|
206
260
|
reset groups, and a serverless instance can terminate before asynchronous delivery completes. An
|
|
207
261
|
in-process SDK cannot reliably detect an OOM kill, frozen event loop, host failure, or unavailable
|
|
208
262
|
network. Pair it with an external uptime monitor and keep graceful shutdown explicit; see the
|
|
209
|
-
[production recipe](examples/production-recipe/README.md).
|
|
263
|
+
[production recipe](https://github.com/FutureWindAI/Wotchi/blob/main/examples/production-recipe/README.md).
|
|
210
264
|
|
|
211
265
|
## Process monitoring
|
|
212
266
|
|
|
@@ -230,25 +284,25 @@ const monitor = registerWotchiProcessMonitor(wotchi);
|
|
|
230
284
|
|
|
231
285
|
## Documentation
|
|
232
286
|
|
|
233
|
-
- [Getting started](docs/GETTING_STARTED.md)
|
|
234
|
-
- [Examples](docs/EXAMPLES.md)
|
|
235
|
-
- [API reference](docs/API.md)
|
|
236
|
-
- [Configuration](docs/CONFIGURATION.md)
|
|
237
|
-
- [Compatibility](docs/COMPATIBILITY.md)
|
|
238
|
-
- [Performance](docs/PERFORMANCE.md)
|
|
239
|
-
- [Architecture](docs/ARCHITECTURE.md)
|
|
240
|
-
- [Roadmap](docs/ROADMAP.md)
|
|
241
|
-
- [Troubleshooting](docs/TROUBLESHOOTING.md)
|
|
242
|
-
- [FAQ](docs/FAQ.md)
|
|
243
|
-
- [Security and privacy](docs/SECURITY.md)
|
|
244
|
-
- [Threat model](docs/THREAT_MODEL.md)
|
|
245
|
-
- [Contributing](CONTRIBUTING.md)
|
|
246
|
-
- [Changelog](CHANGELOG.md)
|
|
287
|
+
- [Getting started](https://github.com/FutureWindAI/Wotchi/blob/main/docs/GETTING_STARTED.md)
|
|
288
|
+
- [Examples](https://github.com/FutureWindAI/Wotchi/blob/main/docs/EXAMPLES.md)
|
|
289
|
+
- [API reference](https://github.com/FutureWindAI/Wotchi/blob/main/docs/API.md)
|
|
290
|
+
- [Configuration](https://github.com/FutureWindAI/Wotchi/blob/main/docs/CONFIGURATION.md)
|
|
291
|
+
- [Compatibility](https://github.com/FutureWindAI/Wotchi/blob/main/docs/COMPATIBILITY.md)
|
|
292
|
+
- [Performance](https://github.com/FutureWindAI/Wotchi/blob/main/docs/PERFORMANCE.md)
|
|
293
|
+
- [Architecture](https://github.com/FutureWindAI/Wotchi/blob/main/docs/ARCHITECTURE.md)
|
|
294
|
+
- [Roadmap](https://github.com/FutureWindAI/Wotchi/blob/main/docs/ROADMAP.md)
|
|
295
|
+
- [Troubleshooting](https://github.com/FutureWindAI/Wotchi/blob/main/docs/TROUBLESHOOTING.md)
|
|
296
|
+
- [FAQ](https://github.com/FutureWindAI/Wotchi/blob/main/docs/FAQ.md)
|
|
297
|
+
- [Security and privacy](https://github.com/FutureWindAI/Wotchi/blob/main/docs/SECURITY.md)
|
|
298
|
+
- [Threat model](https://github.com/FutureWindAI/Wotchi/blob/main/docs/THREAT_MODEL.md)
|
|
299
|
+
- [Contributing](https://github.com/FutureWindAI/Wotchi/blob/main/CONTRIBUTING.md)
|
|
300
|
+
- [Changelog](https://github.com/FutureWindAI/Wotchi/blob/main/CHANGELOG.md)
|
|
247
301
|
- [GitHub releases](https://github.com/FutureWindAI/Wotchi/releases)
|
|
248
|
-
- [Apache License 2.0](LICENSE)
|
|
302
|
+
- [Apache License 2.0](https://github.com/FutureWindAI/Wotchi/blob/main/LICENSE)
|
|
249
303
|
|
|
250
304
|
## Security
|
|
251
305
|
|
|
252
|
-
Do not include real secrets or customer error data in issues, examples, or test fixtures. Report security vulnerabilities privately using [
|
|
306
|
+
Do not include real secrets or customer error data in issues, examples, or test fixtures. Report security vulnerabilities privately using the [security policy](https://github.com/FutureWindAI/Wotchi/blob/main/SECURITY.md).
|
|
253
307
|
|
|
254
308
|
Wotchi is open source and maintained by FutureWind AI.
|
package/SECURITY.md
CHANGED
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
|
|
5
5
|
Wotchi targets applications running Node.js `>=18.18.0` and is built and tested across the Node.js 18–26 matrix. Node.js versions below 18.18 are not supported.
|
|
6
6
|
|
|
7
|
-
The current package contains the framework-independent capture core, console and
|
|
7
|
+
The current package contains the framework-independent capture core, console, Telegram, and generic HTTPS webhook notifiers, Express 4/5 middleware, NestJS 10/11 exception-filter integration, optional HTTP status observation, and opt-in process monitoring. Supported runtime and framework combinations are documented after packed-tarball compatibility and release-security checks.
|
|
8
8
|
|
|
9
9
|
## Reporting a Vulnerability
|
|
10
10
|
|
|
11
11
|
Do not disclose a suspected vulnerability, exploit detail, credential, token, or real customer error in a public issue, discussion, pull request, or example.
|
|
12
12
|
|
|
13
|
-
Use [GitHub private vulnerability reporting](https://github.com/FutureWindAI/
|
|
13
|
+
Use [GitHub private vulnerability reporting](https://github.com/FutureWindAI/Wotchi/security/advisories/new). Do not open a public issue for a suspected vulnerability.
|
|
14
14
|
|
|
15
15
|
Include only the minimum sanitized information needed to reproduce the problem. Remove application secrets, personal data, proprietary source, production URLs, and raw customer payloads.
|
|
16
16
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createCaptureAdmission = createCaptureAdmission;
|
|
4
|
+
/** A small token bucket used before normalization to bound overload work. */
|
|
5
|
+
function createCaptureAdmission(options) {
|
|
6
|
+
const now = options.now ?? Date.now;
|
|
7
|
+
let tokens = options.burst;
|
|
8
|
+
let lastRefill = now();
|
|
9
|
+
return {
|
|
10
|
+
tryAcquire: () => {
|
|
11
|
+
const timestamp = now();
|
|
12
|
+
const elapsedMs = Math.max(0, timestamp - lastRefill);
|
|
13
|
+
tokens = Math.min(options.burst, tokens + (elapsedMs * options.maxEventsPerSecond) / 1_000);
|
|
14
|
+
lastRefill = timestamp;
|
|
15
|
+
if (tokens < 1) {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
tokens -= 1;
|
|
19
|
+
return true;
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
}
|
package/dist/cjs/core/client.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createWotchi = createWotchi;
|
|
4
4
|
const config_js_1 = require("./config.js");
|
|
5
|
+
const admission_js_1 = require("./admission.js");
|
|
5
6
|
const diagnostics_js_1 = require("./diagnostics.js");
|
|
6
7
|
const fingerprint_js_1 = require("./fingerprint.js");
|
|
7
8
|
const group_store_js_1 = require("./group-store.js");
|
|
@@ -282,9 +283,19 @@ function createWotchi(config) {
|
|
|
282
283
|
const queue = (0, notification_queue_js_1.createNotificationQueue)({
|
|
283
284
|
maxPendingAlerts: normalized.queue.maxPendingAlerts,
|
|
284
285
|
concurrency: normalized.queue.concurrency,
|
|
286
|
+
notifierTimeoutMs: normalized.queue.notifierTimeoutMs,
|
|
287
|
+
notifierCircuitBreaker: normalized.queue.notifierCircuitBreaker,
|
|
285
288
|
});
|
|
289
|
+
const admission = normalized.overload === undefined
|
|
290
|
+
? undefined
|
|
291
|
+
: (0, admission_js_1.createCaptureAdmission)({
|
|
292
|
+
maxEventsPerSecond: normalized.overload.maxEventsPerSecond,
|
|
293
|
+
burst: normalized.overload.burst,
|
|
294
|
+
now,
|
|
295
|
+
});
|
|
286
296
|
const privacy = normalized.privacy;
|
|
287
297
|
let eventSequence = 0;
|
|
298
|
+
let overloadLastAlertAt = 0;
|
|
288
299
|
const matchingRule = (event) => normalized.rules.find((rule) => (rule.environment === undefined || rule.environment === event.environment) &&
|
|
289
300
|
(rule.route === undefined || rule.route === event.request?.route));
|
|
290
301
|
const captureSafeEvent = (error, metadata, context, request, eventKind = "error", alertThreshold = normalized.grouping.alertThreshold, fingerprintOverride, severityOverride, trace, correlationId, operation, job, tags) => {
|
|
@@ -405,11 +416,28 @@ function createWotchi(config) {
|
|
|
405
416
|
}
|
|
406
417
|
diagnostics.capturedEvents += 1;
|
|
407
418
|
};
|
|
419
|
+
const emitOverloadSignal = (timestamp) => {
|
|
420
|
+
const cooldownMs = normalized.overload?.alertCooldownMs ?? 0;
|
|
421
|
+
if (cooldownMs > 0 && timestamp - overloadLastAlertAt < cooldownMs) {
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
overloadLastAlertAt = timestamp;
|
|
425
|
+
captureSafeEvent(new Error("Wotchi capture overload: admission limit reached"), undefined, { droppedEvents: diagnostics.eventsDroppedOverload }, undefined, "runtime-monitor", 1, "wotchi.capture.overload", "high", undefined, undefined, "wotchi.overload", undefined, undefined);
|
|
426
|
+
};
|
|
408
427
|
const captureException = (error, context, options) => {
|
|
409
428
|
if (!normalized.enabled) {
|
|
410
429
|
return;
|
|
411
430
|
}
|
|
431
|
+
if (queue.isClosed()) {
|
|
432
|
+
diagnostics.capturesAfterShutdown += 1;
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
412
435
|
try {
|
|
436
|
+
if (admission !== undefined && !admission.tryAcquire()) {
|
|
437
|
+
diagnostics.eventsDroppedOverload += 1;
|
|
438
|
+
emitOverloadSignal(now());
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
413
441
|
captureSafeEvent(error, undefined, context, options?.request, "error", normalizeEventAlertThreshold(options?.alertThreshold) ?? normalized.grouping.alertThreshold, options?.fingerprint, options?.severity, options?.trace, options?.correlationId, options?.operation, options?.job, options?.tags);
|
|
414
442
|
}
|
|
415
443
|
catch {
|
|
@@ -420,6 +448,10 @@ function createWotchi(config) {
|
|
|
420
448
|
if (!normalized.enabled) {
|
|
421
449
|
return;
|
|
422
450
|
}
|
|
451
|
+
if (queue.isClosed()) {
|
|
452
|
+
diagnostics.capturesAfterShutdown += 1;
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
423
455
|
try {
|
|
424
456
|
if (event === null || typeof event !== "object" || event.level !== "error") {
|
|
425
457
|
throw new TypeError("event.level must be error");
|
|
@@ -428,6 +460,14 @@ function createWotchi(config) {
|
|
|
428
460
|
throw new TypeError("event.message must be a string");
|
|
429
461
|
}
|
|
430
462
|
const alertThreshold = normalizeEventAlertThreshold(event.alertThreshold);
|
|
463
|
+
if (admission !== undefined &&
|
|
464
|
+
event.kind !== "process-monitor" &&
|
|
465
|
+
event.kind !== "runtime-monitor" &&
|
|
466
|
+
!admission.tryAcquire()) {
|
|
467
|
+
diagnostics.eventsDroppedOverload += 1;
|
|
468
|
+
emitOverloadSignal(now());
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
431
471
|
captureSafeEvent(event.error ?? safeMessage(event.message, privacy), event.metadata, event.context, event.request, event.kind ?? "error", alertThreshold ?? normalized.grouping.alertThreshold, event.fingerprint, event.severity, event.trace, event.correlationId, event.operation, event.job, event.tags);
|
|
432
472
|
}
|
|
433
473
|
catch {
|
|
@@ -440,6 +480,8 @@ function createWotchi(config) {
|
|
|
440
480
|
alertsDropped: queue.alertsDropped(),
|
|
441
481
|
alertsSent: queue.alertsSent(),
|
|
442
482
|
notifierFailures: queue.notifierFailures(),
|
|
483
|
+
notifierTimeouts: queue.notifierTimeouts(),
|
|
484
|
+
notifierCircuitOpenSkips: queue.notifierCircuitOpenSkips(),
|
|
443
485
|
activeGroups: groupStore.size(),
|
|
444
486
|
pendingAlerts: queue.pending(),
|
|
445
487
|
});
|
|
@@ -503,11 +545,15 @@ function createWotchi(config) {
|
|
|
503
545
|
...(status === "notifier-failed" ? { error: "One or more notifiers failed." } : {}),
|
|
504
546
|
};
|
|
505
547
|
};
|
|
548
|
+
const shutdown = async (timeoutMs) => {
|
|
549
|
+
await queue.close(timeoutMs);
|
|
550
|
+
};
|
|
506
551
|
return {
|
|
507
552
|
captureException,
|
|
508
553
|
captureEvent,
|
|
509
554
|
testAlert,
|
|
510
555
|
flush: (timeoutMs) => queue.flush(timeoutMs),
|
|
556
|
+
shutdown,
|
|
511
557
|
getDiagnostics: readDiagnostics,
|
|
512
558
|
};
|
|
513
559
|
}
|
package/dist/cjs/core/config.js
CHANGED
|
@@ -232,7 +232,9 @@ function validateConfig(config) {
|
|
|
232
232
|
}
|
|
233
233
|
const grouping = readOptionalRecord(config.grouping, "grouping");
|
|
234
234
|
const queue = readOptionalRecord(config.queue, "queue");
|
|
235
|
+
const notifierCircuitBreaker = readOptionalRecord(queue?.notifierCircuitBreaker, "queue.notifierCircuitBreaker");
|
|
235
236
|
const privacy = readOptionalRecord(config.privacy, "privacy");
|
|
237
|
+
const overload = readOptionalRecord(config.overload, "overload");
|
|
236
238
|
const links = normalizeLinks(config.links);
|
|
237
239
|
const concurrency = queue?.concurrency;
|
|
238
240
|
if (concurrency !== undefined && concurrency !== 1) {
|
|
@@ -259,7 +261,21 @@ function validateConfig(config) {
|
|
|
259
261
|
queue: Object.freeze({
|
|
260
262
|
maxPendingAlerts: readPositiveInteger(queue, "maxPendingAlerts", 100, limits_js_1.MAX_PENDING_ALERTS),
|
|
261
263
|
concurrency: 1,
|
|
264
|
+
notifierTimeoutMs: readPositiveInteger(queue, "notifierTimeoutMs", 5_000, limits_js_1.MAX_NOTIFIER_TIMEOUT_MS),
|
|
265
|
+
notifierCircuitBreaker: Object.freeze({
|
|
266
|
+
failureThreshold: readPositiveInteger(notifierCircuitBreaker, "failureThreshold", 3, limits_js_1.MAX_CIRCUIT_BREAKER_FAILURES),
|
|
267
|
+
cooldownMs: readPositiveInteger(notifierCircuitBreaker, "cooldownMs", 30_000, limits_js_1.MAX_CIRCUIT_BREAKER_COOLDOWN_MS),
|
|
268
|
+
}),
|
|
262
269
|
}),
|
|
270
|
+
...(overload === undefined
|
|
271
|
+
? {}
|
|
272
|
+
: {
|
|
273
|
+
overload: Object.freeze({
|
|
274
|
+
maxEventsPerSecond: readPositiveInteger(overload, "maxEventsPerSecond", 1_000, limits_js_1.MAX_CAPTURE_RATE),
|
|
275
|
+
burst: readPositiveInteger(overload, "burst", readPositiveInteger(overload, "maxEventsPerSecond", 1_000, limits_js_1.MAX_CAPTURE_RATE), limits_js_1.MAX_CAPTURE_RATE),
|
|
276
|
+
alertCooldownMs: readPositiveInteger(overload, "alertCooldownMs", 60_000, limits_js_1.MAX_COOLDOWN_MS),
|
|
277
|
+
}),
|
|
278
|
+
}),
|
|
263
279
|
privacy: Object.freeze({
|
|
264
280
|
redactKeys: normalizeRedactKeys(privacy),
|
|
265
281
|
maxDepth: readPositiveInteger(privacy, "maxDepth", 5, limits_js_1.MAX_NORMALIZATION_DEPTH),
|
|
@@ -10,6 +10,8 @@ function createDiagnosticsState() {
|
|
|
10
10
|
filterFailures: 0,
|
|
11
11
|
beforeSendFailures: 0,
|
|
12
12
|
eventsSuppressed: 0,
|
|
13
|
+
eventsDroppedOverload: 0,
|
|
14
|
+
capturesAfterShutdown: 0,
|
|
13
15
|
};
|
|
14
16
|
}
|
|
15
17
|
function snapshotDiagnostics(state, values) {
|
|
@@ -20,6 +22,8 @@ function snapshotDiagnostics(state, values) {
|
|
|
20
22
|
filterFailures: state.filterFailures,
|
|
21
23
|
beforeSendFailures: state.beforeSendFailures,
|
|
22
24
|
eventsSuppressed: state.eventsSuppressed,
|
|
25
|
+
eventsDroppedOverload: state.eventsDroppedOverload,
|
|
26
|
+
capturesAfterShutdown: state.capturesAfterShutdown,
|
|
23
27
|
...values,
|
|
24
28
|
});
|
|
25
29
|
}
|
|
@@ -50,14 +50,9 @@ function createGroupStore(options) {
|
|
|
50
50
|
severity: group.severity,
|
|
51
51
|
});
|
|
52
52
|
const evictLeastRecent = () => {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
candidate = group;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
if (candidate !== undefined) {
|
|
60
|
-
groups.delete(candidate.fingerprint);
|
|
53
|
+
const oldestFingerprint = groups.keys().next().value;
|
|
54
|
+
if (typeof oldestFingerprint === "string") {
|
|
55
|
+
groups.delete(oldestFingerprint);
|
|
61
56
|
evicted += 1;
|
|
62
57
|
}
|
|
63
58
|
};
|
|
@@ -83,6 +78,10 @@ function createGroupStore(options) {
|
|
|
83
78
|
};
|
|
84
79
|
groups.set(fingerprint, group);
|
|
85
80
|
}
|
|
81
|
+
else {
|
|
82
|
+
groups.delete(fingerprint);
|
|
83
|
+
groups.set(fingerprint, group);
|
|
84
|
+
}
|
|
86
85
|
group.lastSeenMs = timestamp;
|
|
87
86
|
group.totalCount += 1;
|
|
88
87
|
group.severity = severity;
|
package/dist/cjs/core/limits.js
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.MAX_NORMALIZATION_STACK_LENGTH = exports.MAX_NORMALIZATION_STRING_LENGTH = exports.MAX_NORMALIZATION_KEYS = exports.MAX_NORMALIZATION_DEPTH = exports.MAX_COOLDOWN_MS = exports.MAX_ALERT_THRESHOLD = exports.MAX_WINDOW_MS = exports.MAX_PENDING_ALERTS = exports.MAX_EVENTS_PER_WINDOW = exports.MAX_GROUPS = void 0;
|
|
3
|
+
exports.MAX_NORMALIZATION_STACK_LENGTH = exports.MAX_NORMALIZATION_STRING_LENGTH = exports.MAX_NORMALIZATION_KEYS = exports.MAX_NORMALIZATION_DEPTH = exports.MAX_CIRCUIT_BREAKER_FAILURES = exports.MAX_CIRCUIT_BREAKER_COOLDOWN_MS = exports.MAX_NOTIFIER_TIMEOUT_MS = exports.MAX_CAPTURE_RATE = exports.MAX_COOLDOWN_MS = exports.MAX_ALERT_THRESHOLD = exports.MAX_WINDOW_MS = exports.MAX_PENDING_ALERTS = exports.MAX_EVENTS_PER_WINDOW = exports.MAX_GROUPS = void 0;
|
|
4
4
|
exports.MAX_GROUPS = 10_000;
|
|
5
5
|
exports.MAX_EVENTS_PER_WINDOW = 10_000;
|
|
6
6
|
exports.MAX_PENDING_ALERTS = 10_000;
|
|
7
7
|
exports.MAX_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
8
8
|
exports.MAX_ALERT_THRESHOLD = 1_000_000;
|
|
9
9
|
exports.MAX_COOLDOWN_MS = 30 * 24 * 60 * 60 * 1_000;
|
|
10
|
+
exports.MAX_CAPTURE_RATE = 1_000_000;
|
|
11
|
+
exports.MAX_NOTIFIER_TIMEOUT_MS = 60_000;
|
|
12
|
+
exports.MAX_CIRCUIT_BREAKER_COOLDOWN_MS = 24 * 60 * 60 * 1_000;
|
|
13
|
+
exports.MAX_CIRCUIT_BREAKER_FAILURES = 100;
|
|
10
14
|
exports.MAX_NORMALIZATION_DEPTH = 20;
|
|
11
15
|
exports.MAX_NORMALIZATION_KEYS = 10_000;
|
|
12
16
|
exports.MAX_NORMALIZATION_STRING_LENGTH = 32_768;
|
|
@@ -98,7 +98,12 @@ const normalizeValue = (value, depth, state, limits) => {
|
|
|
98
98
|
break;
|
|
99
99
|
}
|
|
100
100
|
state.keysVisited += 1;
|
|
101
|
-
|
|
101
|
+
const child = readProperty(value, key);
|
|
102
|
+
const normalizedKey = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
103
|
+
result[truncate(key, limits.maxStringLength)] =
|
|
104
|
+
normalizedKey === "stack" && typeof child === "string"
|
|
105
|
+
? truncate(child, limits.maxStackLength)
|
|
106
|
+
: normalizeValue(child, depth + 1, state, limits);
|
|
102
107
|
}
|
|
103
108
|
return result;
|
|
104
109
|
};
|
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.createNotificationQueue = createNotificationQueue;
|
|
4
4
|
const limits_js_1 = require("./limits.js");
|
|
5
5
|
const DEFAULT_FLUSH_TIMEOUT_MS = 5_000;
|
|
6
|
+
const DEFAULT_NOTIFIER_TIMEOUT_MS = 5_000;
|
|
7
|
+
const DEFAULT_CIRCUIT_FAILURE_THRESHOLD = 3;
|
|
8
|
+
const DEFAULT_CIRCUIT_COOLDOWN_MS = 30_000;
|
|
6
9
|
function createNotificationQueue(options) {
|
|
7
10
|
if (!Number.isSafeInteger(options.maxPendingAlerts) ||
|
|
8
11
|
options.maxPendingAlerts <= 0 ||
|
|
@@ -12,6 +15,23 @@ function createNotificationQueue(options) {
|
|
|
12
15
|
if (options.concurrency !== 1) {
|
|
13
16
|
throw new RangeError("concurrency must be 1");
|
|
14
17
|
}
|
|
18
|
+
if (options.notifierTimeoutMs !== undefined &&
|
|
19
|
+
(!Number.isSafeInteger(options.notifierTimeoutMs) ||
|
|
20
|
+
options.notifierTimeoutMs <= 0 ||
|
|
21
|
+
options.notifierTimeoutMs > limits_js_1.MAX_NOTIFIER_TIMEOUT_MS)) {
|
|
22
|
+
throw new RangeError("notifierTimeoutMs must be a positive integer");
|
|
23
|
+
}
|
|
24
|
+
if (options.notifierCircuitBreaker !== undefined) {
|
|
25
|
+
const { failureThreshold, cooldownMs } = options.notifierCircuitBreaker;
|
|
26
|
+
if (!Number.isSafeInteger(failureThreshold) ||
|
|
27
|
+
failureThreshold <= 0 ||
|
|
28
|
+
failureThreshold > limits_js_1.MAX_CIRCUIT_BREAKER_FAILURES ||
|
|
29
|
+
!Number.isSafeInteger(cooldownMs) ||
|
|
30
|
+
cooldownMs <= 0 ||
|
|
31
|
+
cooldownMs > limits_js_1.MAX_CIRCUIT_BREAKER_COOLDOWN_MS) {
|
|
32
|
+
throw new RangeError("notifierCircuitBreaker limits are invalid");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
15
35
|
const pendingJobs = [];
|
|
16
36
|
let drainPromise;
|
|
17
37
|
let resolveDrain;
|
|
@@ -20,6 +40,15 @@ function createNotificationQueue(options) {
|
|
|
20
40
|
let dropped = 0;
|
|
21
41
|
let sent = 0;
|
|
22
42
|
let failures = 0;
|
|
43
|
+
let timeouts = 0;
|
|
44
|
+
let circuitOpenSkips = 0;
|
|
45
|
+
let closed = false;
|
|
46
|
+
const notifierStates = new Map();
|
|
47
|
+
const notifierTimeoutMs = options.notifierTimeoutMs ?? DEFAULT_NOTIFIER_TIMEOUT_MS;
|
|
48
|
+
const circuit = options.notifierCircuitBreaker ?? {
|
|
49
|
+
failureThreshold: DEFAULT_CIRCUIT_FAILURE_THRESHOLD,
|
|
50
|
+
cooldownMs: DEFAULT_CIRCUIT_COOLDOWN_MS,
|
|
51
|
+
};
|
|
23
52
|
const notifyFailure = (error, notifier) => {
|
|
24
53
|
failures += 1;
|
|
25
54
|
try {
|
|
@@ -38,19 +67,51 @@ function createNotificationQueue(options) {
|
|
|
38
67
|
resolveDrain = undefined;
|
|
39
68
|
};
|
|
40
69
|
const processJob = async (job) => {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
70
|
+
const results = await Promise.all(job.notifiers.map(async (notifier) => {
|
|
71
|
+
const state = notifierStates.get(notifier) ?? { failures: 0, openUntil: 0 };
|
|
72
|
+
const timestamp = Date.now();
|
|
73
|
+
if (state.openUntil > timestamp) {
|
|
74
|
+
circuitOpenSkips += 1;
|
|
75
|
+
notifyFailure(new Error(`Notifier circuit is open: ${notifier.name}`), notifier);
|
|
76
|
+
return { sent: 0, failed: 1 };
|
|
77
|
+
}
|
|
78
|
+
if (state.openUntil !== 0) {
|
|
79
|
+
state.openUntil = 0;
|
|
80
|
+
state.failures = 0;
|
|
81
|
+
}
|
|
82
|
+
let timer;
|
|
44
83
|
try {
|
|
45
|
-
|
|
84
|
+
const timeout = new Promise((_, reject) => {
|
|
85
|
+
timer = setTimeout(() => {
|
|
86
|
+
timeouts += 1;
|
|
87
|
+
reject(new Error(`Notifier timed out after ${notifierTimeoutMs}ms: ${notifier.name}`));
|
|
88
|
+
}, notifierTimeoutMs);
|
|
89
|
+
timer.unref?.();
|
|
90
|
+
});
|
|
91
|
+
await Promise.race([notifier.send(job.alert), timeout]);
|
|
92
|
+
if (timer !== undefined) {
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
}
|
|
95
|
+
state.failures = 0;
|
|
96
|
+
notifierStates.set(notifier, state);
|
|
46
97
|
sent += 1;
|
|
47
|
-
|
|
98
|
+
return { sent: 1, failed: 0 };
|
|
48
99
|
}
|
|
49
100
|
catch (error) {
|
|
50
|
-
|
|
101
|
+
if (timer !== undefined) {
|
|
102
|
+
clearTimeout(timer);
|
|
103
|
+
}
|
|
104
|
+
state.failures += 1;
|
|
105
|
+
if (state.failures >= circuit.failureThreshold) {
|
|
106
|
+
state.openUntil = Date.now() + circuit.cooldownMs;
|
|
107
|
+
}
|
|
108
|
+
notifierStates.set(notifier, state);
|
|
51
109
|
notifyFailure(error, notifier);
|
|
110
|
+
return { sent: 0, failed: 1 };
|
|
52
111
|
}
|
|
53
|
-
}
|
|
112
|
+
}));
|
|
113
|
+
const jobSent = results.reduce((total, result) => total + result.sent, 0);
|
|
114
|
+
const jobFailures = results.reduce((total, result) => total + result.failed, 0);
|
|
54
115
|
try {
|
|
55
116
|
job.onComplete?.({ notifierFailures: jobFailures, sent: jobSent });
|
|
56
117
|
}
|
|
@@ -73,7 +134,7 @@ function createNotificationQueue(options) {
|
|
|
73
134
|
}
|
|
74
135
|
};
|
|
75
136
|
const enqueue = (alert, notifiers, onComplete) => {
|
|
76
|
-
if (pendingJobs.length >= options.maxPendingAlerts) {
|
|
137
|
+
if (closed || pendingJobs.length >= options.maxPendingAlerts) {
|
|
77
138
|
dropped += 1;
|
|
78
139
|
return false;
|
|
79
140
|
}
|
|
@@ -86,6 +147,10 @@ function createNotificationQueue(options) {
|
|
|
86
147
|
pump();
|
|
87
148
|
return true;
|
|
88
149
|
};
|
|
150
|
+
const close = (timeoutMs = DEFAULT_FLUSH_TIMEOUT_MS) => {
|
|
151
|
+
closed = true;
|
|
152
|
+
return flush(timeoutMs);
|
|
153
|
+
};
|
|
89
154
|
const flush = (timeoutMs = DEFAULT_FLUSH_TIMEOUT_MS) => {
|
|
90
155
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
|
|
91
156
|
return Promise.reject(new RangeError("timeoutMs must be a positive integer"));
|
|
@@ -111,10 +176,14 @@ function createNotificationQueue(options) {
|
|
|
111
176
|
return {
|
|
112
177
|
enqueue,
|
|
113
178
|
flush,
|
|
179
|
+
close,
|
|
180
|
+
isClosed: () => closed,
|
|
114
181
|
pending: () => pendingJobs.length,
|
|
115
182
|
alertsQueued: () => queued,
|
|
116
183
|
alertsDropped: () => dropped,
|
|
117
184
|
alertsSent: () => sent,
|
|
118
185
|
notifierFailures: () => failures,
|
|
186
|
+
notifierTimeouts: () => timeouts,
|
|
187
|
+
notifierCircuitOpenSkips: () => circuitOpenSkips,
|
|
119
188
|
};
|
|
120
189
|
}
|