4runr-os 2.10.39 → 2.10.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/gateway/dist/apps/gateway/src/index.js +14 -4
- package/apps/gateway/dist/apps/gateway/src/index.js.map +1 -1
- package/apps/gateway/dist/apps/gateway/src/metrics/monitoring-detail.d.ts +18 -0
- package/apps/gateway/dist/apps/gateway/src/metrics/monitoring-detail.d.ts.map +1 -0
- package/apps/gateway/dist/apps/gateway/src/metrics/monitoring-detail.js +117 -0
- package/apps/gateway/dist/apps/gateway/src/metrics/monitoring-detail.js.map +1 -0
- package/apps/gateway/dist/apps/gateway/src/middleware/log-capture.d.ts +2 -0
- package/apps/gateway/dist/apps/gateway/src/middleware/log-capture.d.ts.map +1 -0
- package/apps/gateway/dist/apps/gateway/src/middleware/log-capture.js +54 -0
- package/apps/gateway/dist/apps/gateway/src/middleware/log-capture.js.map +1 -0
- package/apps/gateway/dist/apps/gateway/src/routes/monitoring.d.ts +15 -0
- package/apps/gateway/dist/apps/gateway/src/routes/monitoring.d.ts.map +1 -0
- package/apps/gateway/dist/apps/gateway/src/routes/monitoring.js +164 -0
- package/apps/gateway/dist/apps/gateway/src/routes/monitoring.js.map +1 -0
- package/apps/gateway/package-lock.json +204 -353
- package/apps/gateway/src/index.ts +27 -8
- package/apps/gateway/src/metrics/monitoring-detail.ts +162 -0
- package/apps/gateway/src/middleware/log-capture.ts +70 -0
- package/apps/gateway/src/routes/monitoring.ts +298 -0
- package/dist/gateway-client.d.ts +2 -0
- package/dist/gateway-client.d.ts.map +1 -1
- package/dist/gateway-client.js +22 -0
- package/dist/gateway-client.js.map +1 -1
- package/dist/tui-handlers.js +498 -0
- package/dist/tui-handlers.js.map +1 -1
- package/mk3-tui/src/app/render_scheduler.rs +111 -112
- package/mk3-tui/src/app.rs +1078 -295
- package/mk3-tui/src/debug_log.rs +131 -124
- package/mk3-tui/src/io/mod.rs +63 -66
- package/mk3-tui/src/io/protocol.rs +14 -15
- package/mk3-tui/src/io/stdio.rs +31 -32
- package/mk3-tui/src/io/ws.rs +25 -32
- package/mk3-tui/src/main.rs +774 -212
- package/mk3-tui/src/monitoring/mod.rs +428 -0
- package/mk3-tui/src/screens/mod.rs +53 -39
- package/mk3-tui/src/storage/cache.rs +221 -224
- package/mk3-tui/src/storage/mod.rs +5 -6
- package/mk3-tui/src/ui/agent_builder.rs +1148 -922
- package/mk3-tui/src/ui/agent_list.rs +344 -295
- package/mk3-tui/src/ui/boot.rs +145 -148
- package/mk3-tui/src/ui/connection_portal.rs +121 -98
- package/mk3-tui/src/ui/help.rs +340 -284
- package/mk3-tui/src/ui/layout.rs +966 -803
- package/mk3-tui/src/ui/mod.rs +1 -1
- package/mk3-tui/src/ui/portal_monitoring.rs +1027 -147
- package/mk3-tui/src/ui/run_manager.rs +784 -764
- package/mk3-tui/src/ui/safe_viewport.rs +236 -235
- package/mk3-tui/src/ui/settings.rs +414 -362
- package/mk3-tui/src/ui/setup_portal.rs +158 -101
- package/mk3-tui/src/websocket.rs +315 -308
- package/package.json +2 -2
|
@@ -1,10 +1,23 @@
|
|
|
1
|
-
//! Portal Monitoring —
|
|
1
|
+
//! Portal Monitoring — Phase 1: section-based layout from the CLI observability snapshot
|
|
2
|
+
//! (same data as before: `gateway.observability` → healthLines + Prometheus stats).
|
|
3
|
+
//!
|
|
4
|
+
//! ## Snapshot layout contract (fragile)
|
|
5
|
+
//! `parse_snapshot` splits on **substring markers** that must stay aligned across:
|
|
6
|
+
//! - `packages/os-cli/src/tui-handlers.ts` (`observabilityHealthLines`, `summarizeGatewayPrometheusMetrics` output)
|
|
7
|
+
//! - `main.rs` (where `content_lines` are assembled for `PortalMonitoringState`)
|
|
8
|
+
//!
|
|
9
|
+
//! If any layer renames or drops these strings, `unwrap_or(lines.len())` paths can put most of the
|
|
10
|
+
//! text into `overview_prefix` and leave `health` / `dependencies` empty. When changing those call
|
|
11
|
+
//! sites, run `cargo test -p mk3-tui portal_snapshot` and update the fixture in this module.
|
|
12
|
+
//!
|
|
13
|
+
//! Marker constants in this file (`SNAPSHOT_MARKER_*`) are the single source of truth for those strings.
|
|
2
14
|
|
|
3
15
|
use ratatui::layout::{Alignment, Constraint, Direction, Layout};
|
|
4
16
|
use ratatui::prelude::*;
|
|
5
17
|
use ratatui::widgets::{Block, Borders, Clear, Paragraph, Wrap};
|
|
6
18
|
|
|
7
19
|
use crate::app::AppState;
|
|
20
|
+
use crate::monitoring::{MetricsDrillPanel, MonitoringSection, SectionStatus};
|
|
8
21
|
|
|
9
22
|
const BRAND_PURPLE: Color = Color::Rgb(138, 43, 226);
|
|
10
23
|
const CYBER_CYAN: Color = Color::Rgb(0, 255, 255);
|
|
@@ -15,6 +28,17 @@ const TEXT_WARN: Color = Color::Rgb(255, 180, 80);
|
|
|
15
28
|
const TEXT_HEADER: Color = Color::Rgb(0, 200, 255);
|
|
16
29
|
const BG_PANEL: Color = Color::Rgb(18, 18, 25);
|
|
17
30
|
const SEPARATOR_COLOR: Color = Color::Rgb(60, 60, 80);
|
|
31
|
+
const SEL_BG: Color = Color::Rgb(35, 40, 55);
|
|
32
|
+
|
|
33
|
+
// Substrings searched with `contains()` — keep in sync with tui-handlers + main.rs assembly.
|
|
34
|
+
/// First line of `observabilityHealthLines` in `tui-handlers.ts` must still include this.
|
|
35
|
+
pub(crate) const SNAPSHOT_MARKER_LIVE: &str = "Live link check";
|
|
36
|
+
/// `Dependency checks (from /ready):` block header from CLI.
|
|
37
|
+
pub(crate) const SNAPSHOT_MARKER_DEPS: &str = "Dependency checks (from /ready):";
|
|
38
|
+
/// Title line before Prometheus stats block (see `main.rs` portal observability handler).
|
|
39
|
+
pub(crate) const SNAPSHOT_MARKER_PROM: &str = "Prometheus /metrics";
|
|
40
|
+
/// Lines after stats; route table header from `main.rs`.
|
|
41
|
+
pub(crate) const SNAPSHOT_MARKER_TOP_ROUTES: &str = "Top HTTP routes";
|
|
18
42
|
|
|
19
43
|
fn gateway_display_url(state: &AppState) -> String {
|
|
20
44
|
state
|
|
@@ -25,12 +49,772 @@ fn gateway_display_url(state: &AppState) -> String {
|
|
|
25
49
|
.to_string()
|
|
26
50
|
}
|
|
27
51
|
|
|
52
|
+
/// Partition `content_lines` from `gateway.observability` into sections.
|
|
53
|
+
#[derive(Debug, Clone, Default)]
|
|
54
|
+
struct ParsedSnapshot {
|
|
55
|
+
overview_prefix: Vec<String>,
|
|
56
|
+
health: Vec<String>,
|
|
57
|
+
dependencies: Vec<String>,
|
|
58
|
+
metrics: Vec<String>,
|
|
59
|
+
queue: Vec<String>,
|
|
60
|
+
routes: Vec<String>,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
fn trim_strings(lines: &[String]) -> Vec<String> {
|
|
64
|
+
lines.iter().map(|s| s.trim_end().to_string()).collect()
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
fn parse_snapshot(lines: &[String]) -> ParsedSnapshot {
|
|
68
|
+
let mut out = ParsedSnapshot::default();
|
|
69
|
+
if lines.is_empty() {
|
|
70
|
+
return out;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
let live_idx = lines.iter().position(|l| l.contains(SNAPSHOT_MARKER_LIVE));
|
|
74
|
+
let deps_idx = lines.iter().position(|l| l.contains(SNAPSHOT_MARKER_DEPS));
|
|
75
|
+
let prom_idx = lines.iter().position(|l| l.contains(SNAPSHOT_MARKER_PROM));
|
|
76
|
+
let top_idx = lines
|
|
77
|
+
.iter()
|
|
78
|
+
.position(|l| l.contains(SNAPSHOT_MARKER_TOP_ROUTES));
|
|
79
|
+
|
|
80
|
+
let live = live_idx.unwrap_or(lines.len());
|
|
81
|
+
out.overview_prefix = trim_strings(&lines[..live]);
|
|
82
|
+
|
|
83
|
+
let after_live = if live < lines.len() {
|
|
84
|
+
live
|
|
85
|
+
} else {
|
|
86
|
+
lines.len()
|
|
87
|
+
};
|
|
88
|
+
let deps_start = deps_idx.unwrap_or(lines.len());
|
|
89
|
+
let prom_start = prom_idx.unwrap_or(lines.len());
|
|
90
|
+
|
|
91
|
+
if deps_start < lines.len() && deps_start > after_live {
|
|
92
|
+
out.health = trim_strings(&lines[after_live..deps_start]);
|
|
93
|
+
} else if prom_start > after_live && deps_idx.is_none() {
|
|
94
|
+
// No dependency block — treat remainder before Prometheus as health only
|
|
95
|
+
let end_h = prom_start.min(lines.len());
|
|
96
|
+
out.health = trim_strings(&lines[after_live..end_h]);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if deps_start < lines.len() && prom_start > deps_start {
|
|
100
|
+
out.dependencies = trim_strings(&lines[deps_start..prom_start]);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
let routes_start = top_idx.unwrap_or(lines.len());
|
|
104
|
+
if prom_start < lines.len() && routes_start >= prom_start {
|
|
105
|
+
// Skip Prometheus title line(s) and blank → stats
|
|
106
|
+
let mut i = prom_start;
|
|
107
|
+
while i < lines.len() && i < routes_start {
|
|
108
|
+
let l = lines[i].trim();
|
|
109
|
+
if l.is_empty() {
|
|
110
|
+
i += 1;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if l.contains(SNAPSHOT_MARKER_PROM) {
|
|
114
|
+
i += 1;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
let stats_end = routes_start.min(lines.len());
|
|
120
|
+
if i < stats_end {
|
|
121
|
+
let stats_block = trim_strings(&lines[i..stats_end]);
|
|
122
|
+
for row in stats_block {
|
|
123
|
+
if row.contains("Queue jobs") {
|
|
124
|
+
out.queue.push(row);
|
|
125
|
+
} else if !row.is_empty() {
|
|
126
|
+
out.metrics.push(row);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if top_idx.is_some() && routes_start < lines.len() {
|
|
133
|
+
// Include header line "Top HTTP routes..." and column header + rows
|
|
134
|
+
out.routes = trim_strings(&lines[routes_start..]);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
out
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
fn status_from_text(block: &[String]) -> SectionStatus {
|
|
141
|
+
let joined = block.join("\n");
|
|
142
|
+
let not_ready_bad =
|
|
143
|
+
joined.contains("not ready") && !joined.contains("✓ /ready: all critical dependencies OK");
|
|
144
|
+
if joined.contains("✗")
|
|
145
|
+
|| joined.contains(": down")
|
|
146
|
+
|| joined.contains("Verify failed")
|
|
147
|
+
|| not_ready_bad
|
|
148
|
+
{
|
|
149
|
+
return SectionStatus::Unhealthy;
|
|
150
|
+
}
|
|
151
|
+
if joined.contains('⚠') || joined.contains("degraded") || joined.contains("not OK") {
|
|
152
|
+
return SectionStatus::Degraded;
|
|
153
|
+
}
|
|
154
|
+
if joined.contains('✓') || joined.contains(": up") {
|
|
155
|
+
return SectionStatus::Healthy;
|
|
156
|
+
}
|
|
157
|
+
SectionStatus::Unknown
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
fn overall_status(parsed: &ParsedSnapshot) -> SectionStatus {
|
|
161
|
+
let mut s = status_from_text(&parsed.health);
|
|
162
|
+
let ds = status_from_text(&parsed.dependencies);
|
|
163
|
+
if matches!(ds, SectionStatus::Unhealthy) {
|
|
164
|
+
s = SectionStatus::Unhealthy;
|
|
165
|
+
} else if matches!(s, SectionStatus::Healthy) && matches!(ds, SectionStatus::Degraded) {
|
|
166
|
+
s = SectionStatus::Degraded;
|
|
167
|
+
} else if matches!(s, SectionStatus::Healthy) && matches!(ds, SectionStatus::Unhealthy) {
|
|
168
|
+
s = SectionStatus::Unhealthy;
|
|
169
|
+
}
|
|
170
|
+
s
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
fn summarize_overview(parsed: &ParsedSnapshot, url: &str) -> String {
|
|
174
|
+
let st = overall_status(parsed);
|
|
175
|
+
let tag = match st {
|
|
176
|
+
SectionStatus::Healthy => "healthy",
|
|
177
|
+
SectionStatus::Degraded => "degraded",
|
|
178
|
+
SectionStatus::Unhealthy => "unhealthy",
|
|
179
|
+
_ => "unknown",
|
|
180
|
+
};
|
|
181
|
+
format!("{} · {}", url, tag)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
fn summarize_health(health: &[String]) -> String {
|
|
185
|
+
for row in health {
|
|
186
|
+
if row.contains("Uptime:") {
|
|
187
|
+
return row.trim().to_string();
|
|
188
|
+
}
|
|
189
|
+
if row.contains("/health OK") {
|
|
190
|
+
return row.trim().chars().take(64).collect();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
if health.is_empty() {
|
|
194
|
+
"—".into()
|
|
195
|
+
} else {
|
|
196
|
+
health[0].trim().chars().take(56).collect()
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
fn summarize_dependencies(deps: &[String]) -> String {
|
|
201
|
+
let mut down = 0;
|
|
202
|
+
let mut warn = 0;
|
|
203
|
+
for row in deps {
|
|
204
|
+
if row.contains(": down") || row.contains('✗') {
|
|
205
|
+
down += 1;
|
|
206
|
+
}
|
|
207
|
+
if row.contains('⚠') || row.contains("degraded") {
|
|
208
|
+
warn += 1;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if down > 0 {
|
|
212
|
+
format!("{} dependency down", down)
|
|
213
|
+
} else if warn > 0 {
|
|
214
|
+
format!("{} warning(s)", warn)
|
|
215
|
+
} else if deps.is_empty() {
|
|
216
|
+
"—".into()
|
|
217
|
+
} else {
|
|
218
|
+
"all checks OK".into()
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
fn summarize_metrics(metrics: &[String]) -> String {
|
|
223
|
+
let mut req = None;
|
|
224
|
+
let mut err = None;
|
|
225
|
+
for row in metrics {
|
|
226
|
+
let t = row.trim();
|
|
227
|
+
if t.starts_with("HTTP requests total") {
|
|
228
|
+
if let Some(n) = t.split_whitespace().last() {
|
|
229
|
+
req = Some(n.to_string());
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if t.starts_with("HTTP request errors") {
|
|
233
|
+
if let Some(rest) = t.strip_prefix("HTTP request errors") {
|
|
234
|
+
let parts: Vec<&str> = rest.split_whitespace().collect();
|
|
235
|
+
if let Some(first) = parts.first() {
|
|
236
|
+
err = Some(first.to_string());
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
match (req, err) {
|
|
242
|
+
(Some(r), Some(e)) => format!("→ {} req, {} err", r, e),
|
|
243
|
+
(Some(r), None) => format!("→ {} req", r),
|
|
244
|
+
_ => "—".into(),
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
fn summarize_queue(queue: &[String]) -> String {
|
|
249
|
+
if queue.is_empty() {
|
|
250
|
+
"—".into()
|
|
251
|
+
} else {
|
|
252
|
+
queue
|
|
253
|
+
.iter()
|
|
254
|
+
.map(|s| s.trim().chars().take(48).collect::<String>())
|
|
255
|
+
.collect::<Vec<_>>()
|
|
256
|
+
.join(" · ")
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/// First unsigned integer token after stripping a line prefix (handles `packages/os-cli` stats formatting).
|
|
261
|
+
fn first_u64_after_line_prefix(line: &str, prefix: &str) -> Option<u64> {
|
|
262
|
+
let rest = line.trim().strip_prefix(prefix)?.trim();
|
|
263
|
+
let tok = rest.split_whitespace().next()?;
|
|
264
|
+
tok.replace(",", "").parse().ok()
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/// Metrics row tint: non-zero **HTTP request errors** total ⇒ degraded (Prometheus text from CLI).
|
|
268
|
+
fn metrics_section_status(metrics: &[String]) -> SectionStatus {
|
|
269
|
+
for row in metrics {
|
|
270
|
+
let t = row.trim();
|
|
271
|
+
if t.starts_with("HTTP request errors") {
|
|
272
|
+
return match first_u64_after_line_prefix(t, "HTTP request errors") {
|
|
273
|
+
Some(n) if n > 0 => SectionStatus::Degraded,
|
|
274
|
+
_ => SectionStatus::Healthy,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
// No matching row (unusual) — avoid implying “all green”.
|
|
279
|
+
SectionStatus::Unknown
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/// Queue stats tint: non-zero **Queue jobs failed** gauge ⇒ degraded (`gateway-observability.ts` labels).
|
|
283
|
+
fn queue_section_status(queue: &[String]) -> SectionStatus {
|
|
284
|
+
for row in queue {
|
|
285
|
+
let t = row.trim();
|
|
286
|
+
if t.starts_with("Queue jobs failed") {
|
|
287
|
+
return match first_u64_after_line_prefix(t, "Queue jobs failed") {
|
|
288
|
+
Some(n) if n > 0 => SectionStatus::Degraded,
|
|
289
|
+
_ => SectionStatus::Healthy,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
SectionStatus::Healthy
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/// Log lines from CLI use `timestamp [level] message`; derive status from worst `[level]` seen.
|
|
297
|
+
fn logs_section_status(lines: &[String]) -> SectionStatus {
|
|
298
|
+
if lines.is_empty() {
|
|
299
|
+
return SectionStatus::Unknown;
|
|
300
|
+
}
|
|
301
|
+
let mut worst: u8 = 0; // 0 ok, 1 warn, 2 error
|
|
302
|
+
for row in lines {
|
|
303
|
+
let lower = row.to_ascii_lowercase();
|
|
304
|
+
if lower.contains("[fatal]") || lower.contains("[error]") || lower.contains("[critical]") {
|
|
305
|
+
worst = worst.max(2);
|
|
306
|
+
} else if lower.contains("[warn]") || lower.contains("[warning]") {
|
|
307
|
+
worst = worst.max(1);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
match worst {
|
|
311
|
+
2 => SectionStatus::Unhealthy,
|
|
312
|
+
1 => SectionStatus::Degraded,
|
|
313
|
+
_ => SectionStatus::Healthy,
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/// Phase 3: tiny ASCII sparkline from monotonic `http_requests_total` samples (full snapshot polling).
|
|
318
|
+
fn ascii_sparkline(samples: &[u64]) -> String {
|
|
319
|
+
if samples.len() < 2 {
|
|
320
|
+
return String::new();
|
|
321
|
+
}
|
|
322
|
+
const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
|
|
323
|
+
let mn = *samples.iter().min().unwrap();
|
|
324
|
+
let mx = *samples.iter().max().unwrap();
|
|
325
|
+
let span = (mx - mn).max(1);
|
|
326
|
+
samples
|
|
327
|
+
.iter()
|
|
328
|
+
.map(|&v| {
|
|
329
|
+
let i = (((v - mn) as f64 / span as f64) * 7.0).round() as usize;
|
|
330
|
+
BLOCKS[i.min(7)]
|
|
331
|
+
})
|
|
332
|
+
.collect()
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
fn tui_runtime_lines(state: &AppState) -> Vec<String> {
|
|
336
|
+
let count = state.render_durations.len() as u64;
|
|
337
|
+
let avg_render = if count > 0 {
|
|
338
|
+
state.render_durations.iter().copied().sum::<u64>() as f64 / count as f64
|
|
339
|
+
} else {
|
|
340
|
+
0.0
|
|
341
|
+
};
|
|
342
|
+
vec![
|
|
343
|
+
String::new(),
|
|
344
|
+
"MK3 TUI runtime".into(),
|
|
345
|
+
format!(
|
|
346
|
+
" uptime={}s tick={} renders={} scheduled={} logs={}",
|
|
347
|
+
state.uptime_secs,
|
|
348
|
+
state.tick,
|
|
349
|
+
state.render_count,
|
|
350
|
+
state.render_scheduled_count,
|
|
351
|
+
state.log_write_count
|
|
352
|
+
),
|
|
353
|
+
format!(
|
|
354
|
+
" last_render={}ms avg_render={:.1}ms viewport_lines={}",
|
|
355
|
+
state.last_render_time.elapsed().as_millis(),
|
|
356
|
+
avg_render,
|
|
357
|
+
state.portal_monitoring.viewport_lines
|
|
358
|
+
),
|
|
359
|
+
format!(
|
|
360
|
+
" screen_scroll={} auto_refresh={}",
|
|
361
|
+
state.portal_monitoring.scroll_offset,
|
|
362
|
+
if state.portal_monitoring.auto_refresh_enabled {
|
|
363
|
+
"on"
|
|
364
|
+
} else {
|
|
365
|
+
"off"
|
|
366
|
+
}
|
|
367
|
+
),
|
|
368
|
+
]
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
fn metric_history_sparkline<F>(state: &AppState, f: F) -> String
|
|
372
|
+
where
|
|
373
|
+
F: Fn(&crate::app::MetricHistoryEntry) -> u64,
|
|
374
|
+
{
|
|
375
|
+
let samples: Vec<u64> = state
|
|
376
|
+
.portal_monitoring
|
|
377
|
+
.metric_history
|
|
378
|
+
.iter()
|
|
379
|
+
.map(f)
|
|
380
|
+
.collect();
|
|
381
|
+
ascii_sparkline(&samples)
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
fn metric_history_lines(state: &AppState) -> Vec<String> {
|
|
385
|
+
let hist = &state.portal_monitoring.metric_history;
|
|
386
|
+
if hist.is_empty() {
|
|
387
|
+
return vec!["History: no samples yet (wait for a full snapshot refresh).".into()];
|
|
388
|
+
}
|
|
389
|
+
let first = hist.front().map(|x| x.timestamp.as_str()).unwrap_or("?");
|
|
390
|
+
let last = hist.back().map(|x| x.timestamp.as_str()).unwrap_or("?");
|
|
391
|
+
vec![
|
|
392
|
+
format!(
|
|
393
|
+
"History: {} / 720 samples in memory (first {} last {}; wall-clock span depends on refresh cadence)",
|
|
394
|
+
hist.len(),
|
|
395
|
+
first,
|
|
396
|
+
last
|
|
397
|
+
),
|
|
398
|
+
format!(
|
|
399
|
+
" HTTP requests {}",
|
|
400
|
+
metric_history_sparkline(state, |x| x.http_requests)
|
|
401
|
+
),
|
|
402
|
+
format!(
|
|
403
|
+
" HTTP errors {}",
|
|
404
|
+
metric_history_sparkline(state, |x| x.http_errors)
|
|
405
|
+
),
|
|
406
|
+
format!(
|
|
407
|
+
" Queue waiting {}",
|
|
408
|
+
metric_history_sparkline(state, |x| x.queue_waiting)
|
|
409
|
+
),
|
|
410
|
+
format!(
|
|
411
|
+
" Queue failed {}",
|
|
412
|
+
metric_history_sparkline(state, |x| x.queue_failed)
|
|
413
|
+
),
|
|
414
|
+
format!(
|
|
415
|
+
" Runs active {}",
|
|
416
|
+
metric_history_sparkline(state, |x| x.runs_active)
|
|
417
|
+
),
|
|
418
|
+
format!(
|
|
419
|
+
" SSE active {}",
|
|
420
|
+
metric_history_sparkline(state, |x| x.sse_active)
|
|
421
|
+
),
|
|
422
|
+
]
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
fn section_row_style(status: SectionStatus) -> Style {
|
|
426
|
+
match status {
|
|
427
|
+
SectionStatus::Healthy => Style::default().fg(NEON_GREEN),
|
|
428
|
+
SectionStatus::Degraded => Style::default().fg(TEXT_WARN),
|
|
429
|
+
SectionStatus::Unhealthy => Style::default().fg(Color::Red),
|
|
430
|
+
SectionStatus::Loading => Style::default().fg(CYBER_CYAN),
|
|
431
|
+
SectionStatus::Unknown => Style::default().fg(TEXT_DIM),
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const FIXED_TOP_LINES: usize = 3;
|
|
436
|
+
|
|
437
|
+
fn centered_rect(percent_x: u16, percent_y: u16, area: Rect) -> Rect {
|
|
438
|
+
let vertical = Layout::default()
|
|
439
|
+
.direction(Direction::Vertical)
|
|
440
|
+
.constraints([
|
|
441
|
+
Constraint::Percentage((100 - percent_y) / 2),
|
|
442
|
+
Constraint::Percentage(percent_y),
|
|
443
|
+
Constraint::Percentage((100 - percent_y) / 2),
|
|
444
|
+
])
|
|
445
|
+
.split(area);
|
|
446
|
+
Layout::default()
|
|
447
|
+
.direction(Direction::Horizontal)
|
|
448
|
+
.constraints([
|
|
449
|
+
Constraint::Percentage((100 - percent_x) / 2),
|
|
450
|
+
Constraint::Percentage(percent_x),
|
|
451
|
+
Constraint::Percentage((100 - percent_x) / 2),
|
|
452
|
+
])
|
|
453
|
+
.split(vertical[1])[1]
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
fn summary_max_chars(area_width: u16) -> usize {
|
|
457
|
+
(area_width as usize).saturating_sub(26).clamp(24, 120)
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/// Max `scroll_offset` for section list (excludes fixed gateway header inside body panel).
|
|
461
|
+
pub fn portal_monitoring_scroll_max(
|
|
462
|
+
state: &AppState,
|
|
463
|
+
summary_width: usize,
|
|
464
|
+
body_inner_h: usize,
|
|
465
|
+
) -> usize {
|
|
466
|
+
let total = build_body_lines(state, summary_width).len();
|
|
467
|
+
let content_h = body_inner_h.saturating_sub(FIXED_TOP_LINES).max(1);
|
|
468
|
+
total.saturating_sub(content_h.min(total.max(1)))
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/// Build scrollable section lines. **Phase 1:** collapsed summaries and ●/⚠/✖ colors are derived only from
|
|
472
|
+
/// `parse_snapshot` of `content_lines`; `MonitoringState.sections` controls **expand** + **selection** only
|
|
473
|
+
/// (not `SectionState.status` / `summary`).
|
|
474
|
+
fn build_body_lines(state: &AppState, summary_width: usize) -> Vec<Line<'static>> {
|
|
475
|
+
let mut out: Vec<Line<'static>> = Vec::new();
|
|
476
|
+
|
|
477
|
+
if state.portal_monitoring.loading && state.portal_monitoring.content_lines.is_empty() {
|
|
478
|
+
out.push(Line::from(Span::styled(
|
|
479
|
+
"⏳ Fetching Gateway observability snapshot…",
|
|
480
|
+
Style::default().fg(TEXT_WARN),
|
|
481
|
+
)));
|
|
482
|
+
return out;
|
|
483
|
+
}
|
|
484
|
+
if let Some(ref err) = state.portal_monitoring.error {
|
|
485
|
+
out.push(Line::from(vec![
|
|
486
|
+
Span::styled("✗ ", Style::default().fg(Color::Red).bold()),
|
|
487
|
+
Span::styled(err.clone(), Style::default().fg(TEXT_PRIMARY)),
|
|
488
|
+
]));
|
|
489
|
+
let hint = if state.portal_monitoring.content_lines.is_empty() {
|
|
490
|
+
"Press R to retry"
|
|
491
|
+
} else {
|
|
492
|
+
"Showing last known snapshot below. Press R to retry."
|
|
493
|
+
};
|
|
494
|
+
out.push(Line::from(Span::styled(
|
|
495
|
+
hint,
|
|
496
|
+
Style::default().fg(TEXT_DIM),
|
|
497
|
+
)));
|
|
498
|
+
if state.portal_monitoring.content_lines.is_empty() {
|
|
499
|
+
return out;
|
|
500
|
+
}
|
|
501
|
+
out.push(Line::from(""));
|
|
502
|
+
}
|
|
503
|
+
if state.portal_monitoring.content_lines.is_empty() {
|
|
504
|
+
out.push(Line::from(Span::styled(
|
|
505
|
+
"No snapshot yet. Select Overview and press R for a full snapshot.",
|
|
506
|
+
Style::default().fg(TEXT_DIM),
|
|
507
|
+
)));
|
|
508
|
+
return out;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
let parsed = parse_snapshot(&state.portal_monitoring.content_lines);
|
|
512
|
+
let url = gateway_display_url(state);
|
|
513
|
+
let sections_order = MonitoringSection::all();
|
|
514
|
+
let ms = &state.advanced_monitoring.monitoring_state;
|
|
515
|
+
let sel = ms.selected_index;
|
|
516
|
+
let overrides = &state.portal_monitoring.section_overrides;
|
|
517
|
+
|
|
518
|
+
if let Some(alert) = state.portal_monitoring.dependency_alerts.back() {
|
|
519
|
+
if alert.current != "healthy" {
|
|
520
|
+
out.push(Line::from(vec![
|
|
521
|
+
Span::styled("⚠ ", Style::default().fg(TEXT_WARN).bold()),
|
|
522
|
+
Span::styled(
|
|
523
|
+
format!(
|
|
524
|
+
"Dependency status changed: {} → {} ({})",
|
|
525
|
+
alert.previous, alert.current, alert.timestamp
|
|
526
|
+
),
|
|
527
|
+
Style::default().fg(TEXT_WARN),
|
|
528
|
+
),
|
|
529
|
+
]));
|
|
530
|
+
out.push(Line::from(""));
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
for (idx, section) in sections_order.iter().enumerate() {
|
|
535
|
+
let sec_state = ms.sections.get(section);
|
|
536
|
+
let expanded = sec_state.map(|s| s.expanded).unwrap_or(false);
|
|
537
|
+
let selected = idx == sel;
|
|
538
|
+
let ov = overrides.get(section);
|
|
539
|
+
|
|
540
|
+
let (status, summary) = match section {
|
|
541
|
+
MonitoringSection::Overview => {
|
|
542
|
+
let st = overall_status(&parsed);
|
|
543
|
+
(st, summarize_overview(&parsed, &url))
|
|
544
|
+
}
|
|
545
|
+
MonitoringSection::Health => {
|
|
546
|
+
if let Some(lines) = ov {
|
|
547
|
+
(status_from_text(lines), summarize_health(lines))
|
|
548
|
+
} else {
|
|
549
|
+
let st = status_from_text(&parsed.health);
|
|
550
|
+
(st, summarize_health(&parsed.health))
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
MonitoringSection::Dependencies => {
|
|
554
|
+
if let Some(lines) = ov {
|
|
555
|
+
(status_from_text(lines), summarize_dependencies(lines))
|
|
556
|
+
} else {
|
|
557
|
+
let st = status_from_text(&parsed.dependencies);
|
|
558
|
+
(st, summarize_dependencies(&parsed.dependencies))
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
MonitoringSection::Metrics => {
|
|
562
|
+
if let Some(lines) = ov {
|
|
563
|
+
(metrics_section_status(lines), summarize_metrics(lines))
|
|
564
|
+
} else {
|
|
565
|
+
let st = metrics_section_status(&parsed.metrics);
|
|
566
|
+
(st, summarize_metrics(&parsed.metrics))
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
MonitoringSection::Queue => {
|
|
570
|
+
if let Some(lines) = ov {
|
|
571
|
+
(queue_section_status(lines), summarize_queue(lines))
|
|
572
|
+
} else {
|
|
573
|
+
let st = queue_section_status(&parsed.queue);
|
|
574
|
+
(st, summarize_queue(&parsed.queue))
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
MonitoringSection::Logs => {
|
|
578
|
+
if let Some(lines) = ov {
|
|
579
|
+
let summ = if lines.is_empty() {
|
|
580
|
+
"no entries".into()
|
|
581
|
+
} else {
|
|
582
|
+
format!("{} entries", lines.len())
|
|
583
|
+
};
|
|
584
|
+
(logs_section_status(lines), summ)
|
|
585
|
+
} else if state.portal_monitoring.logs_fetch_loading {
|
|
586
|
+
(SectionStatus::Loading, "loading…".into())
|
|
587
|
+
} else {
|
|
588
|
+
(SectionStatus::Unknown, "Press L to fetch logs".to_string())
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
MonitoringSection::System => {
|
|
592
|
+
if let Some(lines) = ov {
|
|
593
|
+
let summ = lines
|
|
594
|
+
.first()
|
|
595
|
+
.map(|s| s.trim().chars().take(56).collect::<String>())
|
|
596
|
+
.unwrap_or_else(|| "—".into());
|
|
597
|
+
(SectionStatus::Healthy, summ)
|
|
598
|
+
} else {
|
|
599
|
+
(
|
|
600
|
+
SectionStatus::Unknown,
|
|
601
|
+
"Press R for CLI/host stats".to_string(),
|
|
602
|
+
)
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
};
|
|
606
|
+
|
|
607
|
+
let sym = status.symbol();
|
|
608
|
+
let arrow = if expanded { "▼" } else { "▶" };
|
|
609
|
+
let prefix = if selected { "› " } else { " " };
|
|
610
|
+
|
|
611
|
+
let header_style = section_row_style(status).patch(
|
|
612
|
+
Style::default()
|
|
613
|
+
.bg(if selected { SEL_BG } else { BG_PANEL })
|
|
614
|
+
.bold(),
|
|
615
|
+
);
|
|
616
|
+
|
|
617
|
+
let spans = vec![
|
|
618
|
+
Span::styled(
|
|
619
|
+
prefix,
|
|
620
|
+
Style::default().fg(if selected { CYBER_CYAN } else { TEXT_DIM }),
|
|
621
|
+
),
|
|
622
|
+
Span::styled(format!("{} ", arrow), Style::default().fg(TEXT_HEADER)),
|
|
623
|
+
Span::styled(
|
|
624
|
+
format!("{:<14}", section.display_name()),
|
|
625
|
+
Style::default().fg(TEXT_PRIMARY),
|
|
626
|
+
),
|
|
627
|
+
Span::styled(format!("{} ", sym), header_style),
|
|
628
|
+
Span::styled(
|
|
629
|
+
summary.chars().take(summary_width).collect::<String>(),
|
|
630
|
+
Style::default().fg(TEXT_PRIMARY),
|
|
631
|
+
),
|
|
632
|
+
];
|
|
633
|
+
out.push(Line::from(spans));
|
|
634
|
+
|
|
635
|
+
if expanded {
|
|
636
|
+
let loading_detail = state.portal_monitoring.section_refresh_loading == Some(*section)
|
|
637
|
+
|| (matches!(section, MonitoringSection::Logs)
|
|
638
|
+
&& state.portal_monitoring.logs_fetch_loading)
|
|
639
|
+
|| (matches!(section, MonitoringSection::Metrics)
|
|
640
|
+
&& state.portal_monitoring.metrics_drill_loading);
|
|
641
|
+
let using_override = ov.is_some();
|
|
642
|
+
|
|
643
|
+
let mut detail: Vec<String> = if loading_detail {
|
|
644
|
+
vec!["⏳ Loading…".into()]
|
|
645
|
+
} else if matches!(section, MonitoringSection::Metrics)
|
|
646
|
+
&& state.portal_monitoring.metrics_drill.is_some()
|
|
647
|
+
{
|
|
648
|
+
state.portal_monitoring.metrics_drill_lines.clone()
|
|
649
|
+
} else if let Some(lines) = ov {
|
|
650
|
+
if matches!(section, MonitoringSection::Logs) && lines.is_empty() {
|
|
651
|
+
vec![
|
|
652
|
+
"No lines returned — the Gateway monitoring log buffer is empty.".into(),
|
|
653
|
+
"Entries require the Gateway to record events (e.g. addMonitoringLog). Traffic alone may not appear here.".into(),
|
|
654
|
+
]
|
|
655
|
+
} else if matches!(section, MonitoringSection::System) {
|
|
656
|
+
let mut combined = lines.clone();
|
|
657
|
+
combined.extend(tui_runtime_lines(state));
|
|
658
|
+
combined
|
|
659
|
+
} else {
|
|
660
|
+
lines.clone()
|
|
661
|
+
}
|
|
662
|
+
} else {
|
|
663
|
+
match section {
|
|
664
|
+
MonitoringSection::Overview => parsed.overview_prefix.clone(),
|
|
665
|
+
MonitoringSection::Health => parsed.health.clone(),
|
|
666
|
+
MonitoringSection::Dependencies => parsed.dependencies.clone(),
|
|
667
|
+
MonitoringSection::Metrics => {
|
|
668
|
+
let mut m = parsed.metrics.clone();
|
|
669
|
+
if state.portal_monitoring.trend_view {
|
|
670
|
+
let mut v = metric_history_lines(state);
|
|
671
|
+
v.push(String::new());
|
|
672
|
+
v.extend(m);
|
|
673
|
+
m = v;
|
|
674
|
+
} else if state.portal_monitoring.metrics_drill.is_none()
|
|
675
|
+
&& !state.portal_monitoring.http_total_sparkline.is_empty()
|
|
676
|
+
{
|
|
677
|
+
let samples: Vec<u64> = state
|
|
678
|
+
.portal_monitoring
|
|
679
|
+
.http_total_sparkline
|
|
680
|
+
.iter()
|
|
681
|
+
.copied()
|
|
682
|
+
.collect();
|
|
683
|
+
let spark = ascii_sparkline(&samples);
|
|
684
|
+
if !spark.is_empty() {
|
|
685
|
+
let mut v = vec![format!(
|
|
686
|
+
"HTTP total (from recent full snapshots) {}",
|
|
687
|
+
spark
|
|
688
|
+
)];
|
|
689
|
+
v.push(String::new());
|
|
690
|
+
v.extend(m);
|
|
691
|
+
m = v;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
m
|
|
695
|
+
}
|
|
696
|
+
MonitoringSection::Queue => parsed.queue.clone(),
|
|
697
|
+
MonitoringSection::Logs => vec![
|
|
698
|
+
"Press L to fetch Gateway logs (monitoring.logs → GET /api/monitoring/logs).".into(),
|
|
699
|
+
"If results stay empty, nothing may be writing to the Gateway log ring buffer yet.".into(),
|
|
700
|
+
],
|
|
701
|
+
MonitoringSection::System => vec![
|
|
702
|
+
"Press R to fetch CLI process + host OS stats.".into(),
|
|
703
|
+
"Press S to run diagnostics (Gateway verify + TUI WebSocket details).".into(),
|
|
704
|
+
String::new(),
|
|
705
|
+
"MK3 TUI runtime".into(),
|
|
706
|
+
format!(
|
|
707
|
+
" uptime={}s renders={} scheduled={}",
|
|
708
|
+
state.uptime_secs,
|
|
709
|
+
state.render_count,
|
|
710
|
+
state.render_scheduled_count
|
|
711
|
+
),
|
|
712
|
+
],
|
|
713
|
+
}
|
|
714
|
+
};
|
|
715
|
+
if matches!(section, MonitoringSection::Dependencies)
|
|
716
|
+
&& !state.portal_monitoring.dependency_alerts.is_empty()
|
|
717
|
+
{
|
|
718
|
+
detail.push(String::new());
|
|
719
|
+
detail.push(
|
|
720
|
+
"Recent dependency status changes (text-derived from health lines)".into(),
|
|
721
|
+
);
|
|
722
|
+
for alert in state
|
|
723
|
+
.portal_monitoring
|
|
724
|
+
.dependency_alerts
|
|
725
|
+
.iter()
|
|
726
|
+
.rev()
|
|
727
|
+
.take(10)
|
|
728
|
+
{
|
|
729
|
+
detail.push(format!(
|
|
730
|
+
" {} {} → {}",
|
|
731
|
+
alert.timestamp, alert.previous, alert.current
|
|
732
|
+
));
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if matches!(section, MonitoringSection::Overview) {
|
|
736
|
+
if let Some(path) = &state.portal_monitoring.last_export_path {
|
|
737
|
+
detail.push(String::new());
|
|
738
|
+
detail.push(format!("Last export: {}", path));
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
if matches!(section, MonitoringSection::Metrics)
|
|
742
|
+
&& !using_override
|
|
743
|
+
&& state.portal_monitoring.metrics_drill.is_none()
|
|
744
|
+
&& !parsed.routes.is_empty()
|
|
745
|
+
{
|
|
746
|
+
let mut m = detail;
|
|
747
|
+
m.push(String::new());
|
|
748
|
+
m.extend(parsed.routes.clone());
|
|
749
|
+
for row in m {
|
|
750
|
+
out.push(Line::from(vec![Span::styled(
|
|
751
|
+
format!(" {}", row),
|
|
752
|
+
Style::default().fg(TEXT_DIM),
|
|
753
|
+
)]));
|
|
754
|
+
}
|
|
755
|
+
} else {
|
|
756
|
+
for row in detail {
|
|
757
|
+
out.push(Line::from(vec![Span::styled(
|
|
758
|
+
format!(" {}", row),
|
|
759
|
+
Style::default().fg(TEXT_DIM),
|
|
760
|
+
)]));
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
if matches!(section, MonitoringSection::Overview) {
|
|
764
|
+
out.push(Line::from(vec![Span::styled(
|
|
765
|
+
" Tip: Overview+R = full snapshot · other rows+R = section · E exports JSON to current working directory · H trends · L logs · A auto.",
|
|
766
|
+
Style::default().fg(TEXT_DIM),
|
|
767
|
+
)]));
|
|
768
|
+
}
|
|
769
|
+
if matches!(section, MonitoringSection::System) && expanded {
|
|
770
|
+
out.push(Line::from(vec![Span::styled(
|
|
771
|
+
" Phase 4: R = stats, S = diagnostics. Disk is intentionally marked unavailable until a native probe exists.",
|
|
772
|
+
Style::default().fg(TEXT_DIM),
|
|
773
|
+
)]));
|
|
774
|
+
}
|
|
775
|
+
if matches!(section, MonitoringSection::Metrics) && expanded {
|
|
776
|
+
let sub = state
|
|
777
|
+
.portal_monitoring
|
|
778
|
+
.metrics_drill
|
|
779
|
+
.map(|p: MetricsDrillPanel| p.label())
|
|
780
|
+
.unwrap_or("(snapshot)");
|
|
781
|
+
out.push(Line::from(vec![Span::styled(
|
|
782
|
+
format!(
|
|
783
|
+
" Phase 3/5: {} │ → next panel │ ← leave drill │ H trends {}",
|
|
784
|
+
sub,
|
|
785
|
+
if state.portal_monitoring.trend_view {
|
|
786
|
+
"on"
|
|
787
|
+
} else {
|
|
788
|
+
"off"
|
|
789
|
+
}
|
|
790
|
+
),
|
|
791
|
+
Style::default().fg(TEXT_DIM),
|
|
792
|
+
)]));
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
out
|
|
798
|
+
}
|
|
799
|
+
|
|
28
800
|
/// Full-screen Portal Monitoring (standalone base screen).
|
|
29
801
|
pub fn render(f: &mut Frame, state: &mut AppState) {
|
|
30
802
|
let area = f.size();
|
|
31
803
|
if area.width == 0 || area.height == 0 {
|
|
32
804
|
return;
|
|
33
805
|
}
|
|
806
|
+
let summary_width = summary_max_chars(area.width);
|
|
807
|
+
state.portal_monitoring.summary_clip_width = summary_width;
|
|
808
|
+
|
|
809
|
+
// Auto-refresh flag + one mirror of `AppState.gateway_url` for code that reads `MonitoringState` only.
|
|
810
|
+
// (URL shown in the panel always comes from `gateway_display_url` → `AppState`.)
|
|
811
|
+
state
|
|
812
|
+
.advanced_monitoring
|
|
813
|
+
.monitoring_state
|
|
814
|
+
.gateway_url
|
|
815
|
+
.clone_from(&state.gateway_url);
|
|
816
|
+
state.advanced_monitoring.monitoring_state.live_mode =
|
|
817
|
+
state.portal_monitoring.auto_refresh_enabled;
|
|
34
818
|
|
|
35
819
|
let chunks = Layout::default()
|
|
36
820
|
.direction(Direction::Vertical)
|
|
@@ -47,7 +831,6 @@ pub fn render(f: &mut Frame, state: &mut AppState) {
|
|
|
47
831
|
f.render_widget(Clear, area);
|
|
48
832
|
f.render_widget(Block::default().style(Style::default().bg(BG_PANEL)), area);
|
|
49
833
|
|
|
50
|
-
// Header
|
|
51
834
|
let title = Block::default()
|
|
52
835
|
.title(" Portal Monitoring ")
|
|
53
836
|
.title_style(Style::default().fg(BRAND_PURPLE).bold())
|
|
@@ -57,141 +840,53 @@ pub fn render(f: &mut Frame, state: &mut AppState) {
|
|
|
57
840
|
f.render_widget(title, chunks[0]);
|
|
58
841
|
|
|
59
842
|
let url = gateway_display_url(state);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
let mut body_lines: Vec<Line> = vec![];
|
|
63
|
-
|
|
64
|
-
// Gateway URL header
|
|
65
|
-
body_lines.push(Line::from(vec![
|
|
66
|
-
Span::styled("Linked Gateway: ", Style::default().fg(TEXT_DIM)),
|
|
843
|
+
let header_line = Line::from(vec![
|
|
844
|
+
Span::styled("Gateway: ", Style::default().fg(TEXT_DIM)),
|
|
67
845
|
Span::styled(url, Style::default().fg(CYBER_CYAN).bold()),
|
|
68
|
-
])
|
|
69
|
-
body_lines.push(Line::from(""));
|
|
846
|
+
]);
|
|
70
847
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
Style::default().fg(TEXT_WARN),
|
|
75
|
-
)));
|
|
76
|
-
} else if let Some(ref err) = state.portal_monitoring.error {
|
|
77
|
-
body_lines.push(Line::from(vec![
|
|
78
|
-
Span::styled("✗ Error: ", Style::default().fg(Color::Red).bold()),
|
|
79
|
-
Span::styled(err, Style::default().fg(TEXT_PRIMARY)),
|
|
80
|
-
]));
|
|
81
|
-
body_lines.push(Line::from(""));
|
|
82
|
-
body_lines.push(Line::from(Span::styled(
|
|
83
|
-
"Press R to retry",
|
|
84
|
-
Style::default().fg(TEXT_DIM),
|
|
85
|
-
)));
|
|
86
|
-
} else if state.portal_monitoring.content_lines.is_empty() {
|
|
87
|
-
body_lines.push(Line::from(Span::styled(
|
|
88
|
-
"No snapshot yet. Press R to fetch /metrics.",
|
|
89
|
-
Style::default().fg(TEXT_DIM),
|
|
90
|
-
)));
|
|
848
|
+
let auto_iv = state.portal_monitoring.auto_refresh_interval.as_secs();
|
|
849
|
+
let live_txt = if state.portal_monitoring.auto_refresh_enabled {
|
|
850
|
+
format!("Live ✓ {}s", auto_iv)
|
|
91
851
|
} else {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|| line_text.starts_with("Top HTTP routes") {
|
|
104
|
-
in_section = true;
|
|
105
|
-
section_name = line_text;
|
|
106
|
-
body_lines.push(Line::from(Span::styled(
|
|
107
|
-
format!("━━ {} ━━", line_text),
|
|
108
|
-
Style::default().fg(TEXT_HEADER).bold(),
|
|
109
|
-
)));
|
|
110
|
-
continue;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
// Empty lines
|
|
114
|
-
if line_text.is_empty() {
|
|
115
|
-
body_lines.push(Line::from(""));
|
|
116
|
-
in_section = false;
|
|
117
|
-
continue;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// Special formatting for different line types
|
|
121
|
-
if line_text.starts_with("LIVE:") {
|
|
122
|
-
body_lines.push(Line::from(Span::styled(
|
|
123
|
-
line_text,
|
|
124
|
-
Style::default().fg(NEON_GREEN).bold(),
|
|
125
|
-
)));
|
|
126
|
-
} else if line_text.starts_with("✓") || line_text.starts_with("/health OK") {
|
|
127
|
-
body_lines.push(Line::from(Span::styled(
|
|
128
|
-
line_text,
|
|
129
|
-
Style::default().fg(NEON_GREEN),
|
|
130
|
-
)));
|
|
131
|
-
} else if line_text.starts_with("⚠") || line_text.contains("degraded") || line_text.contains("not ready") {
|
|
132
|
-
body_lines.push(Line::from(Span::styled(
|
|
133
|
-
line_text,
|
|
134
|
-
Style::default().fg(TEXT_WARN),
|
|
135
|
-
)));
|
|
136
|
-
} else if line_text.starts_with(" •") {
|
|
137
|
-
// Dependency check lines - color based on status
|
|
138
|
-
let color = if line_text.contains(": up") {
|
|
139
|
-
NEON_GREEN
|
|
140
|
-
} else if line_text.contains(": down") {
|
|
141
|
-
Color::Red
|
|
142
|
-
} else {
|
|
143
|
-
TEXT_WARN
|
|
144
|
-
};
|
|
145
|
-
body_lines.push(Line::from(Span::styled(
|
|
146
|
-
line_text,
|
|
147
|
-
Style::default().fg(color),
|
|
148
|
-
)));
|
|
149
|
-
} else if line_text.starts_with(" ") {
|
|
150
|
-
// Indented content (route metrics, etc)
|
|
151
|
-
body_lines.push(Line::from(Span::styled(
|
|
152
|
-
line_text,
|
|
153
|
-
Style::default().fg(TEXT_PRIMARY),
|
|
154
|
-
)));
|
|
155
|
-
} else if line_text.contains("total") || line_text.contains("latency") || line_text.contains("created") {
|
|
156
|
-
// Metrics lines - highlight numbers
|
|
157
|
-
let parts: Vec<&str> = line_text.split_whitespace().collect();
|
|
158
|
-
if parts.len() >= 2 {
|
|
159
|
-
let (label, value) = line_text.split_at(line_text.rfind(char::is_whitespace).unwrap_or(line_text.len()));
|
|
160
|
-
body_lines.push(Line::from(vec![
|
|
161
|
-
Span::styled(format!("{} ", label.trim()), Style::default().fg(TEXT_DIM)),
|
|
162
|
-
Span::styled(value.trim(), Style::default().fg(CYBER_CYAN).bold()),
|
|
163
|
-
]));
|
|
164
|
-
} else {
|
|
165
|
-
body_lines.push(Line::from(Span::styled(
|
|
166
|
-
line_text,
|
|
167
|
-
Style::default().fg(TEXT_PRIMARY),
|
|
168
|
-
)));
|
|
169
|
-
}
|
|
170
|
-
} else {
|
|
171
|
-
// Default text
|
|
172
|
-
body_lines.push(Line::from(Span::styled(
|
|
173
|
-
line_text,
|
|
174
|
-
Style::default().fg(TEXT_PRIMARY),
|
|
175
|
-
)));
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
852
|
+
"Live ✗".into()
|
|
853
|
+
};
|
|
854
|
+
let meta = Line::from(vec![
|
|
855
|
+
Span::styled(" ", Style::default()),
|
|
856
|
+
Span::styled(live_txt, Style::default().fg(NEON_GREEN)),
|
|
857
|
+
]);
|
|
858
|
+
|
|
859
|
+
let keys_hint = Line::from(vec![Span::styled(
|
|
860
|
+
"R sect/full · H trends · E export · Metrics ▼ + →/← drill · Dep ▼ + T detail · Sys ▼ + S diagnostics · L logs · A auto",
|
|
861
|
+
Style::default().fg(TEXT_DIM),
|
|
862
|
+
)]);
|
|
179
863
|
|
|
180
|
-
|
|
864
|
+
let body_lines = build_body_lines(state, summary_width);
|
|
181
865
|
let total = body_lines.len();
|
|
182
|
-
let
|
|
866
|
+
let content_h = body_inner_h.saturating_sub(FIXED_TOP_LINES).max(1);
|
|
867
|
+
let max_scroll = total.saturating_sub(content_h.min(total.max(1)));
|
|
183
868
|
let start = state.portal_monitoring.scroll_offset.min(max_scroll);
|
|
184
|
-
let end = (start +
|
|
185
|
-
let window: Vec<Line> =
|
|
869
|
+
let end = (start + content_h).min(total);
|
|
870
|
+
let window: Vec<Line> = if start < total {
|
|
871
|
+
body_lines[start..end].to_vec()
|
|
872
|
+
} else {
|
|
873
|
+
vec![]
|
|
874
|
+
};
|
|
186
875
|
|
|
187
|
-
let
|
|
876
|
+
let header_lines = vec![header_line, meta, keys_hint];
|
|
877
|
+
let combined: Vec<Line> = header_lines.into_iter().chain(window.into_iter()).collect();
|
|
878
|
+
|
|
879
|
+
let body = Paragraph::new(combined)
|
|
188
880
|
.wrap(Wrap { trim: true })
|
|
189
881
|
.style(Style::default().bg(BG_PANEL))
|
|
190
882
|
.block(
|
|
191
883
|
Block::default()
|
|
192
884
|
.title(vec![
|
|
193
885
|
Span::styled(" ", Style::default()),
|
|
194
|
-
Span::styled(
|
|
886
|
+
Span::styled(
|
|
887
|
+
"Sections (same Gateway data as before)",
|
|
888
|
+
Style::default().fg(NEON_GREEN).bold(),
|
|
889
|
+
),
|
|
195
890
|
Span::styled(" ", Style::default()),
|
|
196
891
|
])
|
|
197
892
|
.borders(Borders::ALL)
|
|
@@ -200,53 +895,103 @@ pub fn render(f: &mut Frame, state: &mut AppState) {
|
|
|
200
895
|
);
|
|
201
896
|
f.render_widget(body, chunks[1]);
|
|
202
897
|
|
|
203
|
-
// Footer with
|
|
898
|
+
// Footer (Phase 1 keys) — align with body when the last observability pull failed
|
|
204
899
|
let status = if state.portal_monitoring.loading {
|
|
205
900
|
vec![
|
|
206
901
|
Span::styled("⏳ ", Style::default().fg(TEXT_WARN)),
|
|
207
902
|
Span::styled("loading", Style::default().fg(TEXT_WARN)),
|
|
208
903
|
]
|
|
904
|
+
} else if state.portal_monitoring.section_refresh_loading.is_some()
|
|
905
|
+
|| state.portal_monitoring.logs_fetch_loading
|
|
906
|
+
|| state.portal_monitoring.metrics_drill_loading
|
|
907
|
+
{
|
|
908
|
+
vec![
|
|
909
|
+
Span::styled("◉ ", Style::default().fg(CYBER_CYAN)),
|
|
910
|
+
Span::styled("section…", Style::default().fg(CYBER_CYAN)),
|
|
911
|
+
]
|
|
912
|
+
} else if state.portal_monitoring.error.is_some() {
|
|
913
|
+
vec![
|
|
914
|
+
Span::styled("⚠ ", Style::default().fg(TEXT_WARN)),
|
|
915
|
+
Span::styled("error", Style::default().fg(TEXT_WARN)),
|
|
916
|
+
]
|
|
209
917
|
} else {
|
|
210
918
|
vec![
|
|
211
919
|
Span::styled("● ", Style::default().fg(NEON_GREEN)),
|
|
212
|
-
Span::styled("
|
|
920
|
+
Span::styled("ok", Style::default().fg(NEON_GREEN)),
|
|
213
921
|
]
|
|
214
922
|
};
|
|
215
|
-
|
|
216
|
-
// Auto-refresh status
|
|
923
|
+
|
|
217
924
|
let auto_status = if state.portal_monitoring.auto_refresh_enabled {
|
|
218
925
|
Span::styled("auto ✓", Style::default().fg(NEON_GREEN))
|
|
219
926
|
} else {
|
|
220
927
|
Span::styled("auto ✗", Style::default().fg(TEXT_DIM))
|
|
221
928
|
};
|
|
222
|
-
|
|
223
|
-
// Last refresh timer
|
|
929
|
+
|
|
224
930
|
let refresh_info = if let Some(last) = state.portal_monitoring.last_refresh {
|
|
225
931
|
let elapsed = last.elapsed().as_secs();
|
|
226
932
|
format!(" ~{}s ago", elapsed)
|
|
227
933
|
} else {
|
|
228
934
|
" never".to_string()
|
|
229
935
|
};
|
|
230
|
-
|
|
231
|
-
let mut footer_spans =
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
936
|
+
|
|
937
|
+
let mut footer_spans = if area.width < 100 {
|
|
938
|
+
vec![
|
|
939
|
+
Span::styled("? ", Style::default().fg(CYBER_CYAN).bold()),
|
|
940
|
+
Span::styled("help ", Style::default().fg(TEXT_DIM)),
|
|
941
|
+
Span::styled("ESC ", Style::default().fg(BRAND_PURPLE).bold()),
|
|
942
|
+
Span::styled("main ", Style::default().fg(TEXT_DIM)),
|
|
943
|
+
Span::styled("↑↓ ", Style::default().fg(CYBER_CYAN).bold()),
|
|
944
|
+
Span::styled("sel ", Style::default().fg(TEXT_DIM)),
|
|
945
|
+
Span::styled("Enter ", Style::default().fg(CYBER_CYAN).bold()),
|
|
946
|
+
Span::styled("open ", Style::default().fg(TEXT_DIM)),
|
|
947
|
+
Span::styled("R ", Style::default().fg(CYBER_CYAN).bold()),
|
|
948
|
+
Span::styled("refresh ", Style::default().fg(TEXT_DIM)),
|
|
949
|
+
]
|
|
950
|
+
} else {
|
|
951
|
+
vec![
|
|
952
|
+
Span::styled("? ", Style::default().fg(CYBER_CYAN).bold()),
|
|
953
|
+
Span::styled("help ", Style::default().fg(TEXT_DIM)),
|
|
954
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
955
|
+
Span::styled("ESC ", Style::default().fg(BRAND_PURPLE).bold()),
|
|
956
|
+
Span::styled("Main ", Style::default().fg(TEXT_DIM)),
|
|
957
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
958
|
+
Span::styled("↑↓ ", Style::default().fg(CYBER_CYAN).bold()),
|
|
959
|
+
Span::styled("section ", Style::default().fg(TEXT_DIM)),
|
|
960
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
961
|
+
Span::styled("Enter ", Style::default().fg(CYBER_CYAN).bold()),
|
|
962
|
+
Span::styled("expand ", Style::default().fg(TEXT_DIM)),
|
|
963
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
964
|
+
Span::styled("R ", Style::default().fg(CYBER_CYAN).bold()),
|
|
965
|
+
Span::styled("full/sect ", Style::default().fg(TEXT_DIM)),
|
|
966
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
967
|
+
Span::styled("L ", Style::default().fg(CYBER_CYAN).bold()),
|
|
968
|
+
Span::styled("logs ", Style::default().fg(TEXT_DIM)),
|
|
969
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
970
|
+
Span::styled("H ", Style::default().fg(CYBER_CYAN).bold()),
|
|
971
|
+
Span::styled("trend ", Style::default().fg(TEXT_DIM)),
|
|
972
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
973
|
+
Span::styled("E ", Style::default().fg(CYBER_CYAN).bold()),
|
|
974
|
+
Span::styled("export ", Style::default().fg(TEXT_DIM)),
|
|
975
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
976
|
+
Span::styled("S ", Style::default().fg(CYBER_CYAN).bold()),
|
|
977
|
+
Span::styled("diag ", Style::default().fg(TEXT_DIM)),
|
|
978
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
979
|
+
Span::styled("A ", Style::default().fg(CYBER_CYAN).bold()),
|
|
980
|
+
Span::styled("auto ", Style::default().fg(TEXT_DIM)),
|
|
981
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
982
|
+
Span::styled("Pg ", Style::default().fg(CYBER_CYAN).bold()),
|
|
983
|
+
Span::styled("scroll ", Style::default().fg(TEXT_DIM)),
|
|
984
|
+
Span::styled("│ ", Style::default().fg(SEPARATOR_COLOR)),
|
|
985
|
+
]
|
|
986
|
+
};
|
|
245
987
|
footer_spans.extend(status);
|
|
246
|
-
footer_spans.push(Span::styled(
|
|
988
|
+
footer_spans.push(Span::styled(
|
|
989
|
+
refresh_info.as_str(),
|
|
990
|
+
Style::default().fg(TEXT_DIM),
|
|
991
|
+
));
|
|
247
992
|
footer_spans.push(Span::styled(" ", Style::default()));
|
|
248
993
|
footer_spans.push(auto_status);
|
|
249
|
-
|
|
994
|
+
|
|
250
995
|
let footer = Paragraph::new(Line::from(footer_spans))
|
|
251
996
|
.alignment(Alignment::Center)
|
|
252
997
|
.block(
|
|
@@ -256,4 +1001,139 @@ pub fn render(f: &mut Frame, state: &mut AppState) {
|
|
|
256
1001
|
.style(Style::default().bg(BG_PANEL)),
|
|
257
1002
|
);
|
|
258
1003
|
f.render_widget(footer, chunks[2]);
|
|
1004
|
+
|
|
1005
|
+
if state.portal_monitoring.help_visible {
|
|
1006
|
+
let help_area = centered_rect(72, 62, area);
|
|
1007
|
+
let help_lines = vec![
|
|
1008
|
+
Line::from(vec![Span::styled(
|
|
1009
|
+
"Portal Monitoring Help",
|
|
1010
|
+
Style::default().fg(CYBER_CYAN).bold(),
|
|
1011
|
+
)]),
|
|
1012
|
+
Line::from(""),
|
|
1013
|
+
Line::from("Navigation"),
|
|
1014
|
+
Line::from(" ↑/↓ select section Enter/Space expand/collapse PgUp/PgDn scroll"),
|
|
1015
|
+
Line::from(" ESC close help / return to Main"),
|
|
1016
|
+
Line::from(""),
|
|
1017
|
+
Line::from("Actions"),
|
|
1018
|
+
Line::from(" R refresh current row (Overview = full snapshot)"),
|
|
1019
|
+
Line::from(
|
|
1020
|
+
" L fetch Gateway logs H toggle trends E export JSON to current directory",
|
|
1021
|
+
),
|
|
1022
|
+
Line::from(" Metrics expanded: → cycle HTTP/Runs/Queue/SSE drill, ← leave drill"),
|
|
1023
|
+
Line::from(" Dependencies expanded: T detail System expanded: S diagnostics"),
|
|
1024
|
+
Line::from(""),
|
|
1025
|
+
Line::from("Recovery"),
|
|
1026
|
+
Line::from(
|
|
1027
|
+
" If a refresh fails, the last known snapshot remains visible when available.",
|
|
1028
|
+
),
|
|
1029
|
+
Line::from(" Dependency alert banners are text-derived from Gateway health lines."),
|
|
1030
|
+
Line::from(""),
|
|
1031
|
+
Line::from("Press ? or ESC to close."),
|
|
1032
|
+
];
|
|
1033
|
+
let help = Paragraph::new(help_lines)
|
|
1034
|
+
.wrap(Wrap { trim: true })
|
|
1035
|
+
.style(Style::default().bg(BG_PANEL).fg(TEXT_PRIMARY))
|
|
1036
|
+
.block(
|
|
1037
|
+
Block::default()
|
|
1038
|
+
.title(" Help ")
|
|
1039
|
+
.title_style(Style::default().fg(BRAND_PURPLE).bold())
|
|
1040
|
+
.borders(Borders::ALL)
|
|
1041
|
+
.border_style(Style::default().fg(CYBER_CYAN))
|
|
1042
|
+
.style(Style::default().bg(BG_PANEL)),
|
|
1043
|
+
);
|
|
1044
|
+
f.render_widget(Clear, help_area);
|
|
1045
|
+
f.render_widget(help, help_area);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
#[cfg(test)]
|
|
1050
|
+
mod tests {
|
|
1051
|
+
use super::*;
|
|
1052
|
+
|
|
1053
|
+
/// Ordering mirrors `main.rs` (observability handler) + strings from `tui-handlers` healthLines.
|
|
1054
|
+
fn fixture_representative() -> Vec<String> {
|
|
1055
|
+
vec![
|
|
1056
|
+
"Portal snapshot (health + metrics on each refresh / auto poll).".into(),
|
|
1057
|
+
"Fetched: 2026-01-01T00:00:00.000Z".into(),
|
|
1058
|
+
String::new(),
|
|
1059
|
+
"LIVE: +0 HTTP since last screen refresh (~0/min) --".into(),
|
|
1060
|
+
String::new(),
|
|
1061
|
+
"Live link check (/health + /ready)".into(),
|
|
1062
|
+
"✓ /health OK — Gateway alive (5 ms)".into(),
|
|
1063
|
+
String::new(),
|
|
1064
|
+
"Dependency checks (from /ready):".into(),
|
|
1065
|
+
" • ✓ PostgreSQL (4Runr DB): up".into(),
|
|
1066
|
+
String::new(),
|
|
1067
|
+
"Prometheus /metrics (HTTP total includes this screen’s /health, /ready, and /metrics polls; add API/agent traffic for runs/queue/SSE)."
|
|
1068
|
+
.into(),
|
|
1069
|
+
String::new(),
|
|
1070
|
+
"HTTP requests total 10".into(),
|
|
1071
|
+
"HTTP request errors 0 (0.00%)".into(),
|
|
1072
|
+
"Queue jobs waiting 0".into(),
|
|
1073
|
+
"Queue jobs failed 0".into(),
|
|
1074
|
+
String::new(),
|
|
1075
|
+
"Top HTTP routes (from http_requests_total labels)".into(),
|
|
1076
|
+
" GET /health 3".into(),
|
|
1077
|
+
]
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
#[test]
|
|
1081
|
+
fn portal_snapshot_parse_partitions_sections() {
|
|
1082
|
+
let p = parse_snapshot(&fixture_representative());
|
|
1083
|
+
assert!(
|
|
1084
|
+
!p.overview_prefix.is_empty(),
|
|
1085
|
+
"overview should hold preamble + LIVE block"
|
|
1086
|
+
);
|
|
1087
|
+
assert!(
|
|
1088
|
+
p.health.iter().any(|l| l.contains(SNAPSHOT_MARKER_LIVE)),
|
|
1089
|
+
"health should contain live marker"
|
|
1090
|
+
);
|
|
1091
|
+
assert!(
|
|
1092
|
+
p.dependencies
|
|
1093
|
+
.iter()
|
|
1094
|
+
.any(|l| l.contains(SNAPSHOT_MARKER_DEPS)),
|
|
1095
|
+
"dependencies block present"
|
|
1096
|
+
);
|
|
1097
|
+
assert!(
|
|
1098
|
+
p.metrics.iter().any(|l| l.contains("HTTP requests total")),
|
|
1099
|
+
"metrics stats extracted"
|
|
1100
|
+
);
|
|
1101
|
+
assert!(
|
|
1102
|
+
p.queue.iter().any(|l| l.contains("Queue jobs")),
|
|
1103
|
+
"queue stats extracted"
|
|
1104
|
+
);
|
|
1105
|
+
assert!(
|
|
1106
|
+
p.routes
|
|
1107
|
+
.iter()
|
|
1108
|
+
.any(|l| l.contains(SNAPSHOT_MARKER_TOP_ROUTES)),
|
|
1109
|
+
"routes tail present"
|
|
1110
|
+
);
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
#[test]
|
|
1114
|
+
fn metrics_section_status_errors_nonzero_is_degraded() {
|
|
1115
|
+
let m = vec!["HTTP request errors 3 (2.00%)".into()];
|
|
1116
|
+
assert_eq!(metrics_section_status(&m), SectionStatus::Degraded);
|
|
1117
|
+
let ok = vec!["HTTP request errors 0 (0.00%)".into()];
|
|
1118
|
+
assert_eq!(metrics_section_status(&ok), SectionStatus::Healthy);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
#[test]
|
|
1122
|
+
fn queue_section_status_failed_nonzero_is_degraded() {
|
|
1123
|
+
let q = vec!["Queue jobs failed 2".into()];
|
|
1124
|
+
assert_eq!(queue_section_status(&q), SectionStatus::Degraded);
|
|
1125
|
+
let ok = vec!["Queue jobs failed 0".into()];
|
|
1126
|
+
assert_eq!(queue_section_status(&ok), SectionStatus::Healthy);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
#[test]
|
|
1130
|
+
fn logs_section_status_picks_worst_level() {
|
|
1131
|
+
assert_eq!(logs_section_status(&[]), SectionStatus::Unknown);
|
|
1132
|
+
let ok = vec!["2026-01-01T00:00:00.000Z [info] up".into()];
|
|
1133
|
+
assert_eq!(logs_section_status(&ok), SectionStatus::Healthy);
|
|
1134
|
+
let warn = vec!["t [warn] slow".into(), "t [info] fine".into()];
|
|
1135
|
+
assert_eq!(logs_section_status(&warn), SectionStatus::Degraded);
|
|
1136
|
+
let err = vec!["t [warn] x".into(), "t [error] boom".into()];
|
|
1137
|
+
assert_eq!(logs_section_status(&err), SectionStatus::Unhealthy);
|
|
1138
|
+
}
|
|
259
1139
|
}
|