@zerotal/monitor 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/LICENSE +21 -0
- package/README.md +174 -0
- package/package.json +74 -0
- package/src/MonitorStore.ts +1471 -0
- package/src/alerting.ts +173 -0
- package/src/config.ts +171 -0
- package/src/facades/Monitor.ts +75 -0
- package/src/index.ts +62 -0
- package/src/instance.ts +16 -0
- package/src/middleware/MonitorAuthMiddleware.ts +20 -0
- package/src/middleware/MonitorPayloadMiddleware.ts +96 -0
- package/src/panel.ts +159 -0
- package/src/prometheus.ts +128 -0
- package/src/provider/MonitorProvider.ts +318 -0
- package/src/recorder/MonitorEventBridge.ts +157 -0
- package/src/recorder/ctxBuffers.ts +91 -0
- package/src/sources/live.ts +249 -0
- package/src/sources/realtime.ts +24 -0
- package/src/sources/system.ts +151 -0
- package/src/store/MonitorDb.ts +415 -0
- package/src/store/RingBuffer.ts +89 -0
- package/src/store/types.ts +580 -0
- package/src/support/time.ts +25 -0
- package/src/ui/MonitorLayout.tsx +42 -0
- package/src/ui/MonitorPage.tsx +3722 -0
- package/src/ui/charts.tsx +55 -0
- package/src/ui/icons.ts +32 -0
- package/src/ui/tones.ts +78 -0
package/src/alerting.ts
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Threshold-based alerting over the live snapshot. The provider evaluates these
|
|
3
|
+
* on a short interval; a newly-firing alert is logged, recorded (so it shows in
|
|
4
|
+
* the panel), and dispatched to any handlers registered via {@link onAlert} —
|
|
5
|
+
* wire those to `@zerotal/notifications`, Slack, PagerDuty, etc.
|
|
6
|
+
*
|
|
7
|
+
* Alerts are edge-triggered: each fires once when it crosses its threshold and
|
|
8
|
+
* resets when it recovers, so handlers aren't spammed every tick.
|
|
9
|
+
*/
|
|
10
|
+
import type { MonitorSnapshot } from "./store/types.ts";
|
|
11
|
+
|
|
12
|
+
export interface AlertThresholds {
|
|
13
|
+
/** Fire when the 5xx rate exceeds this %. Default: 5. */
|
|
14
|
+
errorRatePct?: number;
|
|
15
|
+
/** Fire when pending jobs across queues exceed this. Default: 500. */
|
|
16
|
+
queuePending?: number;
|
|
17
|
+
/** Fire when p95 latency exceeds this many ms. Default: 2000. */
|
|
18
|
+
p95Ms?: number;
|
|
19
|
+
/** Fire when transaction rollbacks in the window exceed this. Default: 10. */
|
|
20
|
+
rolledBackInWindow?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface AlertNotice {
|
|
24
|
+
id: string;
|
|
25
|
+
level: "warning" | "critical";
|
|
26
|
+
title: string;
|
|
27
|
+
detail: string;
|
|
28
|
+
/** Human label for the breaching metric, e.g. "p95 latency". */
|
|
29
|
+
metric: string;
|
|
30
|
+
/** Observed value at firing time. */
|
|
31
|
+
value: number;
|
|
32
|
+
/** Configured threshold it crossed. */
|
|
33
|
+
threshold: number;
|
|
34
|
+
/** Unit for value/threshold: "ms" | "%" | "". */
|
|
35
|
+
unit: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const DEFAULTS: Required<AlertThresholds> = {
|
|
39
|
+
errorRatePct: 5,
|
|
40
|
+
queuePending: 500,
|
|
41
|
+
p95Ms: 2000,
|
|
42
|
+
rolledBackInWindow: 10,
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** Evaluate the snapshot against thresholds; returns the currently-breaching alerts. */
|
|
46
|
+
export function evaluateAlerts(
|
|
47
|
+
snap: MonitorSnapshot,
|
|
48
|
+
thresholds: AlertThresholds = {},
|
|
49
|
+
): AlertNotice[] {
|
|
50
|
+
const t = { ...DEFAULTS, ...thresholds };
|
|
51
|
+
const out: AlertNotice[] = [];
|
|
52
|
+
|
|
53
|
+
const errCard = snap.statCards.find((c) => c.label === "Error rate");
|
|
54
|
+
const errRate = errCard ? parseFloat(errCard.value) || 0 : 0;
|
|
55
|
+
if (errRate > t.errorRatePct) {
|
|
56
|
+
out.push({
|
|
57
|
+
id: "error-rate",
|
|
58
|
+
level: errRate > t.errorRatePct * 2 ? "critical" : "warning",
|
|
59
|
+
title: "Elevated error rate",
|
|
60
|
+
detail: `${errRate.toFixed(1)}% of requests are 5xx (threshold ${t.errorRatePct}%).`,
|
|
61
|
+
metric: "error rate",
|
|
62
|
+
value: +errRate.toFixed(1),
|
|
63
|
+
threshold: t.errorRatePct,
|
|
64
|
+
unit: "%",
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const p95 = snap.percentiles.find((p) => p.label === "p95")?.value ?? 0;
|
|
69
|
+
if (p95 > t.p95Ms) {
|
|
70
|
+
out.push({
|
|
71
|
+
id: "p95-latency",
|
|
72
|
+
level: p95 > t.p95Ms * 2 ? "critical" : "warning",
|
|
73
|
+
title: "Slow responses",
|
|
74
|
+
detail: `p95 latency is ${p95}ms (threshold ${t.p95Ms}ms).`,
|
|
75
|
+
metric: "p95 latency",
|
|
76
|
+
value: p95,
|
|
77
|
+
threshold: t.p95Ms,
|
|
78
|
+
unit: "ms",
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const pending = snap.queues.reduce((a, q) => a + q.pending, 0);
|
|
83
|
+
if (pending > t.queuePending) {
|
|
84
|
+
out.push({
|
|
85
|
+
id: "queue-backlog",
|
|
86
|
+
level: "warning",
|
|
87
|
+
title: "Queue backlog",
|
|
88
|
+
detail: `${pending} jobs pending across ${snap.queues.length} queues (threshold ${t.queuePending}).`,
|
|
89
|
+
metric: "pending jobs",
|
|
90
|
+
value: pending,
|
|
91
|
+
threshold: t.queuePending,
|
|
92
|
+
unit: "",
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (snap.transactions.rolledBack > t.rolledBackInWindow) {
|
|
97
|
+
out.push({
|
|
98
|
+
id: "tx-rollbacks",
|
|
99
|
+
level: "warning",
|
|
100
|
+
title: "Transaction rollbacks",
|
|
101
|
+
detail: `${snap.transactions.rolledBack} rollbacks in the window (threshold ${t.rolledBackInWindow}).`,
|
|
102
|
+
metric: "rollbacks",
|
|
103
|
+
value: snap.transactions.rolledBack,
|
|
104
|
+
threshold: t.rolledBackInWindow,
|
|
105
|
+
unit: "",
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return out;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Decide which currently-breaching alerts should actually fire now.
|
|
114
|
+
*
|
|
115
|
+
* Edge-triggered + cooldown: an alert fires when it first crosses its threshold,
|
|
116
|
+
* and won't fire again within `cooldownMs` even if it recovers and re-breaches.
|
|
117
|
+
* This stops an oscillating metric (e.g. p95 hovering around the limit) from
|
|
118
|
+
* paging an operator — or flooding the feed — every time it dips back over.
|
|
119
|
+
*
|
|
120
|
+
* Mutates `firing` (the in-flight episode set) and `lastFired` (id → timestamp)
|
|
121
|
+
* in place, and returns the subset of `active` to record/dispatch. Pass `now` so
|
|
122
|
+
* the decision is deterministic and testable; `cooldownMs <= 0` disables cooldown
|
|
123
|
+
* (re-fire on every fresh breach).
|
|
124
|
+
*/
|
|
125
|
+
export function alertsToFire(
|
|
126
|
+
active: AlertNotice[],
|
|
127
|
+
firing: Set<string>,
|
|
128
|
+
lastFired: Map<string, number>,
|
|
129
|
+
now: number,
|
|
130
|
+
cooldownMs: number,
|
|
131
|
+
): AlertNotice[] {
|
|
132
|
+
const toFire: AlertNotice[] = [];
|
|
133
|
+
for (const a of active) {
|
|
134
|
+
if (firing.has(a.id)) continue; // already firing this episode
|
|
135
|
+
firing.add(a.id);
|
|
136
|
+
const last = lastFired.get(a.id);
|
|
137
|
+
// Suppress only when it HAS fired before and is still within the cooldown.
|
|
138
|
+
if (last !== undefined && cooldownMs > 0 && now - last < cooldownMs) continue;
|
|
139
|
+
lastFired.set(a.id, now);
|
|
140
|
+
toFire.push(a);
|
|
141
|
+
}
|
|
142
|
+
return toFire;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// ── Handler registry ────────────────────────────────────────────────────────────
|
|
146
|
+
|
|
147
|
+
type AlertHandler = (alert: AlertNotice) => void | Promise<void>;
|
|
148
|
+
const _handlers: AlertHandler[] = [];
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Register a handler called once whenever a monitor alert fires. Returns an
|
|
152
|
+
* unsubscribe function. Wire it to notifications:
|
|
153
|
+
*
|
|
154
|
+
* onAlert((a) => Notification.route("slack", SLACK_URL).notify(new MonitorAlert(a)));
|
|
155
|
+
*/
|
|
156
|
+
export function onAlert(fn: AlertHandler): () => void {
|
|
157
|
+
_handlers.push(fn);
|
|
158
|
+
return () => {
|
|
159
|
+
const i = _handlers.indexOf(fn);
|
|
160
|
+
if (i >= 0) _handlers.splice(i, 1);
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** @internal — fan an alert out to registered handlers (errors swallowed). */
|
|
165
|
+
export function _dispatchAlert(alert: AlertNotice): void {
|
|
166
|
+
for (const h of _handlers) {
|
|
167
|
+
try {
|
|
168
|
+
void h(alert);
|
|
169
|
+
} catch {
|
|
170
|
+
/* a handler must never break alerting */
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for `@zerotal/monitor`, authored in `config/monitor.ts`:
|
|
3
|
+
*
|
|
4
|
+
* import { MonitorConfig } from "@zerotal/monitor";
|
|
5
|
+
* export default MonitorConfig({
|
|
6
|
+
* path: "/monitor",
|
|
7
|
+
* auth: (user) => user?.role === "admin",
|
|
8
|
+
* });
|
|
9
|
+
*/
|
|
10
|
+
import { deepMerge, isDevSurfaceAllowed } from "@zerotal/core";
|
|
11
|
+
import type { RetentionMode } from "./store/MonitorDb.ts";
|
|
12
|
+
import type { AlertThresholds } from "./alerting.ts";
|
|
13
|
+
|
|
14
|
+
export interface MonitorConfigShape {
|
|
15
|
+
/** URL prefix the panel mounts at. Default: `/monitor`. */
|
|
16
|
+
path?: string;
|
|
17
|
+
/** Browser tab title / sidebar heading. Default: `Super Panel`. */
|
|
18
|
+
title?: string;
|
|
19
|
+
/** Sub-label under the title. Default: `Zerotal Ops`. */
|
|
20
|
+
subtitle?: string;
|
|
21
|
+
/**
|
|
22
|
+
* Authorization gate. Receives the authenticated user (or undefined) and
|
|
23
|
+
* must return true to allow access. Default: allow only outside production.
|
|
24
|
+
*/
|
|
25
|
+
auth?: (user: unknown) => boolean | Promise<boolean>;
|
|
26
|
+
/** Install the recorder middleware to capture live request data. Default: true. */
|
|
27
|
+
record?: boolean;
|
|
28
|
+
/** Auto-refresh cadence in milliseconds while "Live" is on. Default: 3000. */
|
|
29
|
+
refreshMs?: number;
|
|
30
|
+
/** Apdex satisfaction threshold (T) in ms. Default: 100. */
|
|
31
|
+
apdexTargetMs?: number;
|
|
32
|
+
/** Queries at/above this many ms count as "slow". Default: 100. */
|
|
33
|
+
slowQueryMs?: number;
|
|
34
|
+
/** Requests at/above this many ms count as "slow" (Slow Requests widget). Default: 1000. */
|
|
35
|
+
slowRequestMs?: number;
|
|
36
|
+
/**
|
|
37
|
+
* Capture request/response headers and bodies on each request (Telescope-style),
|
|
38
|
+
* shown in the request trace. Off by default — it buffers bodies and is
|
|
39
|
+
* privacy-sensitive. Sensitive headers (authorization/cookie) and body keys
|
|
40
|
+
* (password/token/secret/…) are redacted automatically.
|
|
41
|
+
*/
|
|
42
|
+
capturePayloads?: boolean;
|
|
43
|
+
/** Truncate captured request/response bodies to this many bytes. Default: 65536. */
|
|
44
|
+
payloadMaxBytes?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Cache built snapshots for this many ms, keyed by range, so overlapping reads
|
|
47
|
+
* (panel poll + Prometheus scrape + alert loop) share one build. Mutating actions
|
|
48
|
+
* invalidate it. Default: 1000. Set `0` to always build fresh.
|
|
49
|
+
*/
|
|
50
|
+
snapshotCacheMs?: number;
|
|
51
|
+
/**
|
|
52
|
+
* SQLite file the panel persists to, so history survives restarts and the
|
|
53
|
+
* 1h/24h/7d ranges trace real data. Default: `storage/monitor.sqlite`. Use
|
|
54
|
+
* `:memory:` for an ephemeral, in-process store.
|
|
55
|
+
*/
|
|
56
|
+
storage?: string;
|
|
57
|
+
/** Days of history to keep before pruning. Default: 7. */
|
|
58
|
+
retentionDays?: number;
|
|
59
|
+
/** What to do with data past retention: `delete` it or `archive` it. Default: `delete`. */
|
|
60
|
+
retentionMode?: RetentionMode;
|
|
61
|
+
/**
|
|
62
|
+
* Expose a Prometheus text-exposition endpoint. Default: `false` (opt-in).
|
|
63
|
+
* The endpoint is unauthenticated (a scraper can't satisfy user-auth), so it
|
|
64
|
+
* ships off; enable it only when you protect `metricsPath` at the network
|
|
65
|
+
* layer (firewall/ingress) or scope it to a private interface.
|
|
66
|
+
*/
|
|
67
|
+
metrics?: boolean;
|
|
68
|
+
/** Path for the Prometheus endpoint. Default: `/metrics`. Protect it at the network layer. */
|
|
69
|
+
metricsPath?: string;
|
|
70
|
+
/** Evaluate threshold alerts on a short interval. Default: `true`. */
|
|
71
|
+
alerts?: boolean;
|
|
72
|
+
/** Alert thresholds (error rate, queue backlog, p95, rollbacks). */
|
|
73
|
+
alertThresholds?: AlertThresholds;
|
|
74
|
+
/**
|
|
75
|
+
* Minimum gap (ms) before the same alert can fire again after it recovers and
|
|
76
|
+
* re-breaches. Stops an oscillating metric from re-paging / re-flooding the feed
|
|
77
|
+
* for one ongoing issue. Default: 30 min. Set `0` to fire on every fresh breach.
|
|
78
|
+
*/
|
|
79
|
+
alertCooldownMs?: number;
|
|
80
|
+
/**
|
|
81
|
+
* Slack-compatible webhook URL. When set, every newly-firing alert is POSTed to
|
|
82
|
+
* it as JSON (`{ text, level, title, detail }`) so alerts page someone instead of
|
|
83
|
+
* only lighting up the panel. Dependency-free (uses `fetch`); delivery is
|
|
84
|
+
* best-effort. For richer routing, register a handler with `onAlert()` instead.
|
|
85
|
+
*/
|
|
86
|
+
alertWebhook?: string;
|
|
87
|
+
/** Accent colour (hex) for the panel chrome. Default: Zerotal orange. */
|
|
88
|
+
accent?: string;
|
|
89
|
+
/** Version string shown in the System footer. */
|
|
90
|
+
zerotalVersion?: string;
|
|
91
|
+
/** Region label shown in the System footer. */
|
|
92
|
+
region?: string;
|
|
93
|
+
/** Deploy SHA shown in the System footer. */
|
|
94
|
+
deploy?: string;
|
|
95
|
+
/**
|
|
96
|
+
* Switch contributed sections off by id — `{ scheduler: false }` keeps the
|
|
97
|
+
* scheduler installed but drops its section from the panel. Anything absent
|
|
98
|
+
* here is on.
|
|
99
|
+
*/
|
|
100
|
+
sections?: Record<string, boolean>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface ResolvedMonitorConfig extends MonitorConfigShape {
|
|
104
|
+
path: string;
|
|
105
|
+
title: string;
|
|
106
|
+
subtitle: string;
|
|
107
|
+
record: boolean;
|
|
108
|
+
refreshMs: number;
|
|
109
|
+
apdexTargetMs: number;
|
|
110
|
+
slowQueryMs: number;
|
|
111
|
+
slowRequestMs: number;
|
|
112
|
+
snapshotCacheMs: number;
|
|
113
|
+
storage: string;
|
|
114
|
+
retentionDays: number;
|
|
115
|
+
retentionMode: RetentionMode;
|
|
116
|
+
metrics: boolean;
|
|
117
|
+
metricsPath: string;
|
|
118
|
+
alerts: boolean;
|
|
119
|
+
alertCooldownMs: number;
|
|
120
|
+
capturePayloads: boolean;
|
|
121
|
+
payloadMaxBytes: number;
|
|
122
|
+
accent: string;
|
|
123
|
+
/** Always set after MonitorConfig() applies defaults. */
|
|
124
|
+
auth: (user: unknown) => boolean | Promise<boolean>;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// The default gate keys off the framework's own APP_ENV (via the shared
|
|
128
|
+
// fail-closed predicate), NOT NODE_ENV — which Bun leaves unset, so a
|
|
129
|
+
// documented `APP_ENV=production` deploy would otherwise expose the panel.
|
|
130
|
+
// Only explicitly non-prod envs get open-by-default access; everything else
|
|
131
|
+
// (unset, staging, production) must supply an explicit `auth` predicate.
|
|
132
|
+
const defaultAuthOpen = (): boolean => isDevSurfaceAllowed(Bun.env["APP_ENV"] ?? "");
|
|
133
|
+
|
|
134
|
+
const defaults: ResolvedMonitorConfig = {
|
|
135
|
+
path: "/monitor",
|
|
136
|
+
title: "Super Panel",
|
|
137
|
+
subtitle: "Zerotal Ops",
|
|
138
|
+
record: true,
|
|
139
|
+
refreshMs: 3000,
|
|
140
|
+
apdexTargetMs: 100,
|
|
141
|
+
slowQueryMs: 100,
|
|
142
|
+
slowRequestMs: 1000,
|
|
143
|
+
snapshotCacheMs: 1000,
|
|
144
|
+
storage: "storage/monitor.sqlite",
|
|
145
|
+
retentionDays: 7,
|
|
146
|
+
retentionMode: "delete",
|
|
147
|
+
metrics: false,
|
|
148
|
+
metricsPath: "/metrics",
|
|
149
|
+
alerts: true,
|
|
150
|
+
alertCooldownMs: 30 * 60 * 1000,
|
|
151
|
+
capturePayloads: false,
|
|
152
|
+
payloadMaxBytes: 65536,
|
|
153
|
+
accent: "#f97316",
|
|
154
|
+
// deepMerge can't carry a function through, so MonitorConfig() reattaches the
|
|
155
|
+
// real predicate; this default keeps the type honest and is the prod-safe gate.
|
|
156
|
+
auth: () => defaultAuthOpen(),
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
export function MonitorConfig(options: Partial<MonitorConfigShape> = {}): ResolvedMonitorConfig {
|
|
160
|
+
const merged = deepMerge(defaults, options) as ResolvedMonitorConfig;
|
|
161
|
+
// deepMerge can't carry a function through; reattach the auth callback.
|
|
162
|
+
if (options.auth) merged.auth = options.auth;
|
|
163
|
+
else if (!merged.auth) merged.auth = () => defaultAuthOpen();
|
|
164
|
+
return merged;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
declare module "@zerotal/core" {
|
|
168
|
+
interface ConfigRegistry {
|
|
169
|
+
monitor: MonitorConfigShape;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Monitor` facade — a thin, always-available entry point for feeding the
|
|
3
|
+
* monitoring store from application code. Use it to record data the recorder
|
|
4
|
+
* middleware can't see automatically (SQL queries, outgoing HTTP calls, cache
|
|
5
|
+
* hits, sent mail, deploys):
|
|
6
|
+
*
|
|
7
|
+
* import { Monitor } from "@zerotal/monitor";
|
|
8
|
+
* Monitor.recordQuery({ sql, ms, location: "PostController@index" });
|
|
9
|
+
* Monitor.recordHttp({ host: "api.stripe.com", ms: 412 });
|
|
10
|
+
* Monitor.recordCache(hit, key);
|
|
11
|
+
* Monitor.recordMail({ subject, to, mailer, status: "sent", ms, body });
|
|
12
|
+
*
|
|
13
|
+
* Every method is a no-op if the monitor provider hasn't booted, so it's safe
|
|
14
|
+
* to call unconditionally.
|
|
15
|
+
*/
|
|
16
|
+
import { RequestContext } from "@zerotal/core";
|
|
17
|
+
import { _getStore } from "../instance.ts";
|
|
18
|
+
import { addContext } from "../recorder/ctxBuffers.ts";
|
|
19
|
+
import type { MonitorStore } from "../MonitorStore.ts";
|
|
20
|
+
|
|
21
|
+
type StoreMethods = Pick<
|
|
22
|
+
MonitorStore,
|
|
23
|
+
| "recordRequest"
|
|
24
|
+
| "recordException"
|
|
25
|
+
| "recordQuery"
|
|
26
|
+
| "recordHttp"
|
|
27
|
+
| "recordCache"
|
|
28
|
+
| "recordMail"
|
|
29
|
+
| "recordDeploy"
|
|
30
|
+
| "recordJob"
|
|
31
|
+
| "snapshot"
|
|
32
|
+
>;
|
|
33
|
+
|
|
34
|
+
function call<K extends keyof StoreMethods>(
|
|
35
|
+
method: K,
|
|
36
|
+
...args: Parameters<StoreMethods[K]>
|
|
37
|
+
): ReturnType<StoreMethods[K]> | undefined {
|
|
38
|
+
const store = _getStore();
|
|
39
|
+
if (!store) return undefined;
|
|
40
|
+
|
|
41
|
+
return (store[method] as (...a: unknown[]) => unknown)(...args) as ReturnType<StoreMethods[K]>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const Monitor = {
|
|
45
|
+
recordRequest: (...a: Parameters<MonitorStore["recordRequest"]>) => call("recordRequest", ...a),
|
|
46
|
+
recordException: (...a: Parameters<MonitorStore["recordException"]>) =>
|
|
47
|
+
call("recordException", ...a),
|
|
48
|
+
recordQuery: (...a: Parameters<MonitorStore["recordQuery"]>) => call("recordQuery", ...a),
|
|
49
|
+
recordHttp: (...a: Parameters<MonitorStore["recordHttp"]>) => call("recordHttp", ...a),
|
|
50
|
+
recordCache: (...a: Parameters<MonitorStore["recordCache"]>) => call("recordCache", ...a),
|
|
51
|
+
recordMail: (...a: Parameters<MonitorStore["recordMail"]>) => call("recordMail", ...a),
|
|
52
|
+
recordDeploy: (...a: Parameters<MonitorStore["recordDeploy"]>) => call("recordDeploy", ...a),
|
|
53
|
+
recordJob: (...a: Parameters<MonitorStore["recordJob"]>) => call("recordJob", ...a),
|
|
54
|
+
/**
|
|
55
|
+
* Attach custom metadata to the current request or Flow action so it shows on
|
|
56
|
+
* the trace — tenant, plan, feature flags, anything useful for debugging:
|
|
57
|
+
*
|
|
58
|
+
* Monitor.context({ tenant: tenant.id, plan: user.plan });
|
|
59
|
+
*
|
|
60
|
+
* No-op outside a request/action scope.
|
|
61
|
+
*/
|
|
62
|
+
context(data: Record<string, unknown>): void {
|
|
63
|
+
const ctx = RequestContext.tryGet();
|
|
64
|
+
if (ctx) addContext(ctx as object, data);
|
|
65
|
+
},
|
|
66
|
+
/** Read the current aggregated snapshot (used by the page; rarely needed directly). */
|
|
67
|
+
snapshot: (...a: Parameters<MonitorStore["snapshot"]>) => {
|
|
68
|
+
const store = _getStore();
|
|
69
|
+
return store ? store.snapshot(...a) : undefined;
|
|
70
|
+
},
|
|
71
|
+
/** The live store instance, if booted. */
|
|
72
|
+
get store(): MonitorStore | undefined {
|
|
73
|
+
return _getStore();
|
|
74
|
+
},
|
|
75
|
+
} as const;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@zerotal/monitor` — a production monitoring & queue dashboard for Zerotal.
|
|
3
|
+
*
|
|
4
|
+
* A server-driven Flow panel ("Super Panel") with eight tabs: Overview,
|
|
5
|
+
* Requests, Exceptions, Queues, Mail, Database, Cache and System. It records
|
|
6
|
+
* live request/exception data via middleware, reads queues/scheduler/health
|
|
7
|
+
* from the running app, and exposes a `Monitor` facade for everything else.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* // bootstrap/providers.ts
|
|
11
|
+
* import { FlowProvider } from "@zerotal/flow";
|
|
12
|
+
* import { MonitorProvider } from "@zerotal/monitor";
|
|
13
|
+
* export default [FlowProvider, MonitorProvider];
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* // config/monitor.ts
|
|
17
|
+
* import { MonitorConfig } from "@zerotal/monitor";
|
|
18
|
+
* export default MonitorConfig({ path: "/monitor", auth: (u) => u?.role === "admin" });
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// Provider + config
|
|
22
|
+
export { MonitorProvider } from "./provider/MonitorProvider.ts";
|
|
23
|
+
export { MonitorConfig } from "./config.ts";
|
|
24
|
+
export type { MonitorConfigShape, ResolvedMonitorConfig } from "./config.ts";
|
|
25
|
+
|
|
26
|
+
// Facade + store
|
|
27
|
+
export { Monitor } from "./facades/Monitor.ts";
|
|
28
|
+
export { MonitorStore } from "./MonitorStore.ts";
|
|
29
|
+
export type { MonitorStoreOptions } from "./MonitorStore.ts";
|
|
30
|
+
|
|
31
|
+
// Prometheus exporter
|
|
32
|
+
export { renderPrometheus } from "./prometheus.ts";
|
|
33
|
+
|
|
34
|
+
// Alerting
|
|
35
|
+
export { evaluateAlerts, onAlert } from "./alerting.ts";
|
|
36
|
+
export type { AlertThresholds, AlertNotice } from "./alerting.ts";
|
|
37
|
+
|
|
38
|
+
// Recorder + guard
|
|
39
|
+
export { installMonitorEventBridge } from "./recorder/MonitorEventBridge.ts";
|
|
40
|
+
export { MonitorAuthMiddleware } from "./middleware/MonitorAuthMiddleware.ts";
|
|
41
|
+
export { MonitorPayloadMiddleware } from "./middleware/MonitorPayloadMiddleware.ts";
|
|
42
|
+
|
|
43
|
+
// UI
|
|
44
|
+
// The contribution surface. Packages push into the `monitor.panel` binding
|
|
45
|
+
// rather than importing these types; they're exported for app-authored sections.
|
|
46
|
+
export { MonitorPanel } from "./panel.ts";
|
|
47
|
+
export type {
|
|
48
|
+
MonitorPanelHost,
|
|
49
|
+
MonitorSection,
|
|
50
|
+
MonitorSectionData,
|
|
51
|
+
MonitorStat,
|
|
52
|
+
MonitorTable,
|
|
53
|
+
MonitorTableColumn,
|
|
54
|
+
MonitorTone,
|
|
55
|
+
MonitorRow,
|
|
56
|
+
} from "./panel.ts";
|
|
57
|
+
|
|
58
|
+
export { MonitorPage } from "./ui/MonitorPage.tsx";
|
|
59
|
+
export { MonitorLayout } from "./ui/MonitorLayout.tsx";
|
|
60
|
+
|
|
61
|
+
// Data shapes
|
|
62
|
+
export type * from "./store/types.ts";
|
package/src/instance.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-wide handle to the active {@link MonitorStore}. The provider sets it
|
|
3
|
+
* during boot; the recorder middleware and the {@link ./facades/Monitor.ts}
|
|
4
|
+
* facade read it without needing the IoC container in hand.
|
|
5
|
+
*/
|
|
6
|
+
import type { MonitorStore } from "./MonitorStore.ts";
|
|
7
|
+
|
|
8
|
+
let _store: MonitorStore | undefined;
|
|
9
|
+
|
|
10
|
+
export function _setStore(store: MonitorStore | undefined): void {
|
|
11
|
+
_store = store;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function _getStore(): MonitorStore | undefined {
|
|
15
|
+
return _store;
|
|
16
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Guards the monitoring panel. Resolves the panel config from the container and
|
|
3
|
+
* runs its `auth(user)` predicate; denies access otherwise. Mirrors the admin
|
|
4
|
+
* package's guard so behaviour is consistent across Zerotal's ops surfaces.
|
|
5
|
+
*/
|
|
6
|
+
import type { Pipe, NextFn, HttpContext } from "@zerotal/core";
|
|
7
|
+
import { currentApp } from "@zerotal/core";
|
|
8
|
+
import type { ResolvedMonitorConfig } from "../config.ts";
|
|
9
|
+
|
|
10
|
+
export class MonitorAuthMiddleware implements Pipe<HttpContext> {
|
|
11
|
+
async handle(http: HttpContext, next: NextFn): Promise<Response | void> {
|
|
12
|
+
const config = currentApp().container.makeSync("monitor") as ResolvedMonitorConfig;
|
|
13
|
+
const user = (http as { user?: unknown }).user ?? null;
|
|
14
|
+
const allowed = await config.auth(user);
|
|
15
|
+
if (!allowed) {
|
|
16
|
+
return Response.redirect("/login", 302);
|
|
17
|
+
}
|
|
18
|
+
return next();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Captures request/response headers and bodies for the request trace (Telescope's
|
|
3
|
+
* payload watcher). Opt-in via `capturePayloads` — the provider registers it
|
|
4
|
+
* globally only when enabled. Bodies are read from clones so the handler still gets
|
|
5
|
+
* the originals, and sensitive headers/keys are redacted before anything is stored.
|
|
6
|
+
*/
|
|
7
|
+
import type { Pipe, NextFn, HttpContext } from "@zerotal/core";
|
|
8
|
+
import { currentApp } from "@zerotal/core";
|
|
9
|
+
import { addPayload } from "../recorder/ctxBuffers.ts";
|
|
10
|
+
import type { ResolvedMonitorConfig } from "../config.ts";
|
|
11
|
+
|
|
12
|
+
const REDACT_HEADERS = new Set(["authorization", "cookie", "set-cookie", "proxy-authorization"]);
|
|
13
|
+
const REDACT_KEY = /pass(word)?|token|secret|api[-_]?key|authorization|credit[-_]?card|cvv/i;
|
|
14
|
+
const REDACTED = "[redacted]";
|
|
15
|
+
|
|
16
|
+
function redactHeaders(headers: Headers): Record<string, string> {
|
|
17
|
+
const out: Record<string, string> = {};
|
|
18
|
+
headers.forEach((value, key) => {
|
|
19
|
+
out[key] = REDACT_HEADERS.has(key.toLowerCase()) ? REDACTED : value;
|
|
20
|
+
});
|
|
21
|
+
return out;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Recursively mask values of sensitive keys when the body parses as JSON. */
|
|
25
|
+
function redactJson(value: unknown): unknown {
|
|
26
|
+
if (Array.isArray(value)) return value.map(redactJson);
|
|
27
|
+
if (value && typeof value === "object") {
|
|
28
|
+
const out: Record<string, unknown> = {};
|
|
29
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
30
|
+
out[k] = REDACT_KEY.test(k) ? REDACTED : redactJson(v);
|
|
31
|
+
}
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function clip(s: string, maxBytes: number): string {
|
|
38
|
+
return s.length > maxBytes ? `${s.slice(0, maxBytes)}…(truncated)` : s;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function redactBody(raw: string, maxBytes: number): string {
|
|
42
|
+
if (!raw) return "";
|
|
43
|
+
try {
|
|
44
|
+
return clip(JSON.stringify(redactJson(JSON.parse(raw))), maxBytes);
|
|
45
|
+
} catch {
|
|
46
|
+
return clip(raw, maxBytes); // not JSON — keep raw, just truncated
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export class MonitorPayloadMiddleware implements Pipe<HttpContext> {
|
|
51
|
+
async handle(http: HttpContext, next: NextFn): Promise<Response | void> {
|
|
52
|
+
let maxBytes = 65536;
|
|
53
|
+
let monitorPath = "/monitor";
|
|
54
|
+
try {
|
|
55
|
+
const cfg = currentApp().container.makeSync("monitor") as ResolvedMonitorConfig;
|
|
56
|
+
maxBytes = cfg.payloadMaxBytes;
|
|
57
|
+
monitorPath = cfg.path;
|
|
58
|
+
} catch {
|
|
59
|
+
/* config not ready — use defaults */
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Don't capture the panel's own traffic or framework internals.
|
|
63
|
+
const path = http.url.pathname;
|
|
64
|
+
if (path.startsWith(monitorPath) || path.startsWith("/metrics") || path.startsWith("/__")) {
|
|
65
|
+
return next();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Request body from a clone so the handler can still read the original. Skip huge
|
|
69
|
+
// bodies (best-effort via content-length) to avoid buffering large uploads.
|
|
70
|
+
let reqBody = "";
|
|
71
|
+
try {
|
|
72
|
+
const len = Number(http.request.headers.get("content-length") ?? 0);
|
|
73
|
+
if (http.request.body && len <= maxBytes * 16) reqBody = await http.request.clone().text();
|
|
74
|
+
} catch {
|
|
75
|
+
/* unreadable / streamed body — skip */
|
|
76
|
+
}
|
|
77
|
+
const reqHeaders = redactHeaders(http.request.headers);
|
|
78
|
+
|
|
79
|
+
const result = await next();
|
|
80
|
+
|
|
81
|
+
let resBody = "";
|
|
82
|
+
try {
|
|
83
|
+
if (http.response) resBody = await http.response.clone().text();
|
|
84
|
+
} catch {
|
|
85
|
+
/* streamed / binary response — skip */
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
addPayload(http, {
|
|
89
|
+
reqHeaders,
|
|
90
|
+
reqBody: redactBody(reqBody, maxBytes),
|
|
91
|
+
resBody: clip(resBody, maxBytes),
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
return result;
|
|
95
|
+
}
|
|
96
|
+
}
|