@zerotal/monitor 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1471 @@
1
+ /**
2
+ * The monitoring panel's aggregation store, backed by SQLite
3
+ * ({@link ./store/MonitorDb.ts}).
4
+ *
5
+ * Raw events are appended to the database with timestamps via the `record*`
6
+ * methods. `snapshot(range)` reads the rows within the selected window and
7
+ * derives a {@link MonitorSnapshot} — percentiles, throughput, slow routes,
8
+ * exception groups, cache and outgoing-HTTP stats. Because everything is
9
+ * persisted, the live / 1h / 24h / 7d ranges all trace real history and survive
10
+ * restarts. There is no sample data: a quiet window reads as honest zeros.
11
+ *
12
+ * A retention policy (`prune`) deletes or archives data past the configured
13
+ * window, and `wipe` clears everything on demand from the panel.
14
+ */
15
+ import { percentile } from "./store/RingBuffer.ts";
16
+ import { MonitorDb } from "./store/MonitorDb.ts";
17
+ import {
18
+ bufferQuery as _bufferQuery,
19
+ markNPlus as _markNPlus,
20
+ collectRequestState as _collectRequestState,
21
+ } from "./recorder/ctxBuffers.ts";
22
+ import type { RequestState } from "./recorder/ctxBuffers.ts";
23
+ import type {
24
+ RetentionMode,
25
+ StorageInfo,
26
+ RequestRow,
27
+ QueryRow,
28
+ ExceptionRow,
29
+ HttpRow,
30
+ CacheRow,
31
+ MailRow,
32
+ DeployRow,
33
+ JobRow,
34
+ EventRow,
35
+ } from "./store/MonitorDb.ts";
36
+ import type {
37
+ Alert,
38
+ AlertContext,
39
+ AlertEntry,
40
+ CacheStats,
41
+ CommandEntry,
42
+ ConnectedClient,
43
+ DbStat,
44
+ Deploy,
45
+ ExceptionGroup,
46
+ FeedEvent,
47
+ HealthEntry,
48
+ JobEntry,
49
+ MailEntry,
50
+ MemoryRoute,
51
+ MigrationEntry,
52
+ ModelEvent,
53
+ ModelStat,
54
+ MonitorRange,
55
+ MonitorSnapshot,
56
+ NotificationEntry,
57
+ NPlusOne,
58
+ OutgoingHttp,
59
+ Percentile,
60
+ PulseStats,
61
+ QueueMetric,
62
+ QueueRow,
63
+ RealtimeStats,
64
+ RequestEntry,
65
+ RequestPayload,
66
+ RequestQuery,
67
+ RequestSpan,
68
+ RouteDetail,
69
+ RouteStat,
70
+ SlowQuery,
71
+ StatCard,
72
+ StatusClassCount,
73
+ TxStats,
74
+ UptimeCheck,
75
+ UserUsage,
76
+ Worker,
77
+ } from "./store/types.ts";
78
+ import { activeConnections, connectedClients } from "./sources/realtime.ts";
79
+ import { ago, commas, delta } from "./support/time.ts";
80
+ import {
81
+ liveCheckins,
82
+ liveDeadLetter,
83
+ liveFailedJobs,
84
+ liveHealth,
85
+ liveQueues,
86
+ liveQueueStats,
87
+ liveScheduled,
88
+ liveWorkers,
89
+ type AppShape,
90
+ } from "./sources/live.ts";
91
+ import { systemGauges, systemMeta, httpPulse } from "./sources/system.ts";
92
+
93
+ interface ReqSample {
94
+ t: number;
95
+ method: string;
96
+ path: string;
97
+ status: number;
98
+ ms: number;
99
+ queries: RequestQuery[];
100
+ nplus: boolean;
101
+ user: string | null;
102
+ ip: string | null;
103
+ mem: number;
104
+ context: Record<string, unknown>;
105
+ payload: RequestPayload | null;
106
+ error: string | null;
107
+ }
108
+
109
+ export interface MonitorStoreOptions {
110
+ apdexTargetMs?: number;
111
+ slowQueryMs?: number;
112
+ zerotalVersion?: string;
113
+ region?: string;
114
+ deploy?: string;
115
+ /** SQLite path. `:memory:` (default) for tests; a file path for persistence. */
116
+ storage?: string;
117
+ /** Days of history to keep before pruning. Default: 7. */
118
+ retentionDays?: number;
119
+ /** What to do with data past retention: delete it or move it to the archive. Default: `delete`. */
120
+ retentionMode?: RetentionMode;
121
+ /** Requests at/above this many ms count as "slow". Default: 1000. */
122
+ slowRequestMs?: number;
123
+ /**
124
+ * Cache built snapshots for this many ms, keyed by range. De-dupes overlapping
125
+ * builds (panel poll + Prometheus scrape + alert loop all hit "live"). Mutating
126
+ * actions invalidate it. Default: 0 (off) — the provider enables it in production.
127
+ */
128
+ snapshotCacheMs?: number;
129
+ }
130
+
131
+ const DAY_MS = 24 * 60 * 60 * 1000;
132
+
133
+ export class MonitorStore {
134
+ private readonly _db: MonitorDb;
135
+ private _app?: AppShape;
136
+ private readonly _pausedQueues = new Set<string>();
137
+ private readonly _removedFailed = new Set<number>();
138
+ private readonly _removedDead = new Set<number>();
139
+ private readonly _snapTtlMs: number;
140
+ private readonly _snapCache = new Map<MonitorRange, { at: number; snap: MonitorSnapshot }>();
141
+
142
+ private readonly _opts: Required<
143
+ Pick<
144
+ MonitorStoreOptions,
145
+ "apdexTargetMs" | "slowQueryMs" | "slowRequestMs" | "retentionDays" | "retentionMode"
146
+ >
147
+ > &
148
+ MonitorStoreOptions;
149
+
150
+ constructor(opts: MonitorStoreOptions = {}) {
151
+ this._opts = {
152
+ apdexTargetMs: opts.apdexTargetMs ?? 100,
153
+ slowQueryMs: opts.slowQueryMs ?? 100,
154
+ slowRequestMs: opts.slowRequestMs ?? 1000,
155
+ retentionDays: opts.retentionDays ?? 7,
156
+ retentionMode: opts.retentionMode ?? "delete",
157
+ ...opts,
158
+ };
159
+ this._snapTtlMs = Math.max(0, opts.snapshotCacheMs ?? 0);
160
+ this._db = new MonitorDb(opts.storage ?? ":memory:");
161
+ }
162
+
163
+ /** Bind the IoC app so live sources (queue/scheduler/health) can be read. */
164
+ bindApp(app: AppShape): void {
165
+ this._app = app;
166
+ }
167
+
168
+ // ── Ingestion ──────────────────────────────────────────────────────────────
169
+
170
+ recordRequest(r: {
171
+ method: string;
172
+ path: string;
173
+ status: number;
174
+ ms: number;
175
+ queries?: RequestQuery[];
176
+ logs?: unknown;
177
+ nplus?: boolean;
178
+ spans?: unknown;
179
+ user?: string | null;
180
+ ip?: string | null;
181
+ memKb?: number;
182
+ context?: Record<string, unknown>;
183
+ payload?: RequestPayload | null;
184
+ error?: string | null;
185
+ }): void {
186
+ const queries = r.queries ?? [];
187
+ const ms = Math.max(0, Math.round(r.ms));
188
+ this._db.recordRequest({
189
+ t: Date.now(),
190
+ method: r.method.toUpperCase(),
191
+ path: r.path,
192
+ status: r.status,
193
+ ms,
194
+ nplus: (r.nplus ?? queries.length > 25) ? 1 : 0,
195
+ queries: JSON.stringify(queries),
196
+ user: r.user ?? null,
197
+ ip: r.ip ?? null,
198
+ mem: Math.max(0, Math.round(r.memKb ?? 0)),
199
+ context: JSON.stringify(r.context ?? {}),
200
+ payload: r.payload ? JSON.stringify(r.payload) : null,
201
+ error: r.error ?? null,
202
+ });
203
+ }
204
+
205
+ recordException(
206
+ error: Error | { type: string; message: string },
207
+ location?: string,
208
+ user?: string | null,
209
+ ): void {
210
+ const type =
211
+ "type" in error && typeof error.type === "string"
212
+ ? error.type
213
+ : (error as Error).name || "Error";
214
+ const message = error.message ?? "";
215
+ const frames = this._stackFrames((error as Error).stack);
216
+ const loc = location ?? frames[0] ?? "unknown";
217
+ this._db.recordException({
218
+ t: Date.now(),
219
+ type,
220
+ message,
221
+ location: loc,
222
+ frames: JSON.stringify(frames),
223
+ user: user ?? null,
224
+ });
225
+ }
226
+
227
+ recordQuery(q: { sql: string; ms: number; location?: string }): void {
228
+ this._db.recordQuery({
229
+ t: Date.now(),
230
+ sql: q.sql,
231
+ ms: Math.round(q.ms),
232
+ location: q.location ?? "—",
233
+ });
234
+ }
235
+
236
+ recordHttp(h: { host: string; ms: number; error?: boolean }): void {
237
+ this._db.recordHttp({
238
+ t: Date.now(),
239
+ host: h.host,
240
+ ms: Math.round(h.ms),
241
+ error: h.error ? 1 : 0,
242
+ });
243
+ }
244
+
245
+ recordCache(hit: boolean, key?: string): void {
246
+ this._db.recordCache({ t: Date.now(), hit: hit ? 1 : 0, key: key ?? "" });
247
+ }
248
+
249
+ recordMail(m: Omit<MailEntry, "id" | "when"> & { when?: string }): void {
250
+ this._db.recordMail({
251
+ t: Date.now(),
252
+ subject: m.subject,
253
+ recipient: m.to,
254
+ mailer: m.mailer,
255
+ status: m.status,
256
+ ms: m.ms,
257
+ body: m.body,
258
+ });
259
+ }
260
+
261
+ recordDeploy(sha: string): void {
262
+ this._db.recordDeploy({ t: Date.now(), sha: sha.slice(0, 7) });
263
+ }
264
+
265
+ recordJob(job: {
266
+ status: string;
267
+ className?: string;
268
+ queue?: string;
269
+ ms?: number;
270
+ error?: string | null;
271
+ }): void {
272
+ this._db.recordJob({
273
+ t: Date.now(),
274
+ status: job.status,
275
+ className: job.className ?? "",
276
+ queue: job.queue ?? "default",
277
+ ms: Math.max(0, Math.round(job.ms ?? 0)),
278
+ error: job.error ?? null,
279
+ });
280
+ }
281
+
282
+ /** Record a generic framework event (security, transaction, migration, N+1, WS, …). */
283
+ recordEvent(e: {
284
+ kind: string;
285
+ label: string;
286
+ status?: "ok" | "warn" | "bad" | "info";
287
+ route?: string | null;
288
+ data?: Record<string, unknown>;
289
+ }): void {
290
+ this._db.recordEvent({
291
+ t: Date.now(),
292
+ kind: e.kind,
293
+ label: e.label,
294
+ status: e.status ?? "info",
295
+ route: e.route ?? null,
296
+ data: JSON.stringify(e.data ?? {}),
297
+ });
298
+ }
299
+
300
+ // ── Per-request correlation (contributed by feature packages) ──────────────────
301
+ // Feature packages hold only this store (resolved from the container), so these
302
+ // thin methods expose the shared request buffers: a package buffers its per-request
303
+ // signal while the request is in flight, and the request-lifecycle handler drains
304
+ // it on finalise. See recorder/ctxBuffers.ts.
305
+
306
+ /** Buffer one query span against the request context it ran under. */
307
+ bufferQuery(ctx: object, q: RequestQuery): void {
308
+ _bufferQuery(ctx, q);
309
+ }
310
+
311
+ /** Flag the request context as having triggered an N+1 pattern. */
312
+ markNPlus(ctx: object): void {
313
+ _markNPlus(ctx);
314
+ }
315
+
316
+ /** Read and clear the buffered correlation state for a finalised request/action. */
317
+ collectRequestState(ctx: object): RequestState {
318
+ return _collectRequestState(ctx);
319
+ }
320
+
321
+ // ── UI actions ────────────────────────────────────────────────────────────────
322
+
323
+ /** Toggle a queue's paused state. Returns the new paused state. */
324
+ toggleQueuePaused(name: string): boolean {
325
+ this._invalidate();
326
+ if (this._pausedQueues.has(name)) {
327
+ this._pausedQueues.delete(name);
328
+ return false;
329
+ }
330
+ this._pausedQueues.add(name);
331
+ return true;
332
+ }
333
+
334
+ /** Retry a failed job (real if a queue is bound) and drop it from the list. */
335
+ async retryFailed(id: number): Promise<boolean> {
336
+ this._invalidate();
337
+ this._removedFailed.add(id);
338
+ const { retryFailedJob } = await import("./sources/live.ts");
339
+ return retryFailedJob(this._app, id);
340
+ }
341
+
342
+ /** Permanently forget a failed job. */
343
+ async forgetFailed(id: number): Promise<boolean> {
344
+ this._invalidate();
345
+ this._removedFailed.add(id);
346
+ const { forgetFailedJob } = await import("./sources/live.ts");
347
+ return forgetFailedJob(this._app, id);
348
+ }
349
+
350
+ /** Requeue a dead-letter job: re-dispatch it (if a queue is bound) and drop it from the view. */
351
+ async requeueDead(id: number): Promise<boolean> {
352
+ this._invalidate();
353
+ this._removedDead.add(id);
354
+ const { retryFailedJob } = await import("./sources/live.ts");
355
+ return retryFailedJob(this._app, id);
356
+ }
357
+
358
+ // ── Retention / cleanup ──────────────────────────────────────────────────────
359
+
360
+ /** Prune data older than the retention window (delete or archive). Returns rows removed. */
361
+ prune(): number {
362
+ this._invalidate();
363
+ const cutoff = Date.now() - this._opts.retentionDays * DAY_MS;
364
+ return this._db.prune(cutoff, this._opts.retentionMode);
365
+ }
366
+
367
+ /** Wipe all recorded data. Returns rows removed. */
368
+ wipe(): number {
369
+ this._invalidate();
370
+ return this._db.wipe();
371
+ }
372
+
373
+ storageInfo(): StorageInfo {
374
+ return this._db.info();
375
+ }
376
+ retentionDays(): number {
377
+ return this._opts.retentionDays;
378
+ }
379
+ retentionMode(): RetentionMode {
380
+ return this._opts.retentionMode;
381
+ }
382
+
383
+ dispose(): void {
384
+ this._db.dispose();
385
+ }
386
+
387
+ // ── Snapshot ────────────────────────────────────────────────────────────────
388
+
389
+ async snapshot(range: MonitorRange = "live"): Promise<MonitorSnapshot> {
390
+ if (this._snapTtlMs > 0) {
391
+ const cached = this._snapCache.get(range);
392
+ if (cached && Date.now() - cached.at < this._snapTtlMs) return cached.snap;
393
+ }
394
+ const snap = await this._buildSnapshot(range);
395
+ if (this._snapTtlMs > 0) this._snapCache.set(range, { at: Date.now(), snap });
396
+ return snap;
397
+ }
398
+
399
+ /** Clear any cached snapshots so the next read reflects a just-applied change. */
400
+ private _invalidate(): void {
401
+ this._snapCache.clear();
402
+ }
403
+
404
+ private async _buildSnapshot(range: MonitorRange = "live"): Promise<MonitorSnapshot> {
405
+ this._db.flush(); // drain the write buffer so the snapshot sees the latest activity
406
+ const windowMs = this._rangeMs(range);
407
+ const now = Date.now();
408
+ const cutoff = now - windowMs;
409
+
410
+ const reqs = this._db.requestsWithin(cutoff).map((r) => this._toSample(r));
411
+ const priorReqs = this._db
412
+ .requestsBetween(now - windowMs * 2, cutoff)
413
+ .map((r) => this._toSample(r));
414
+ const queryRows = this._db.queriesWithin(cutoff);
415
+ const excRows = this._db.exceptionsWithin(cutoff);
416
+ const httpRows = this._db.httpWithin(cutoff);
417
+ const cacheRows = this._db.cacheWithin(cutoff);
418
+ const mailRows = this._db.mailWithin(cutoff);
419
+ const jobRows = this._db.jobsWithin(cutoff);
420
+ const deployRows = this._db.deploysWithin(cutoff);
421
+ const eventRows = this._db.eventsWithin(cutoff);
422
+
423
+ // Live sources return null when the peer/subsystem isn't installed → empty,
424
+ // never fabricated sample data.
425
+ const liveQ = await liveQueues(this._app);
426
+ const queues: QueueRow[] = (liveQ ?? []).map((q) => ({
427
+ ...q,
428
+ paused: this._pausedQueues.has(q.name) || q.paused,
429
+ }));
430
+ // Real throughput + trend per queue, derived from the job runs the monitor records
431
+ // (the queue manager only reports pending depth; wait time has no source, stays 0).
432
+ this._enrichQueues(queues, jobRows, windowMs);
433
+ const failedJobs = ((await liveFailedJobs(this._app)) ?? []).filter(
434
+ (j) => !this._removedFailed.has(j.id),
435
+ );
436
+ const deadLetter = ((await liveDeadLetter(this._app)) ?? []).filter(
437
+ (d) => !this._removedDead.has(d.id),
438
+ );
439
+ const workers: Worker[] = liveWorkers(this._app) ?? [];
440
+ const scheduledJobs = liveScheduled(this._app) ?? [];
441
+ const checkins = liveCheckins(this._app) ?? [];
442
+ const health = (await liveHealth(this._app)) ?? [];
443
+
444
+ return {
445
+ range,
446
+ alerts: this._alerts(reqs, queues),
447
+ deploys: this._deployMarkers(deployRows),
448
+ pulse: this._pulse(),
449
+ statCards: this._statCards(reqs, priorReqs, windowMs, cacheRows),
450
+ percentiles: this._percentiles(reqs),
451
+ apdex: this._apdex(reqs),
452
+ throughput: this._bucket(reqs, 60, () => 1),
453
+ jobsSeries: this._jobSeries(jobRows, windowMs),
454
+ slowRoutes: this._slowRoutes(reqs),
455
+ outgoingHttp: this._outgoing(httpRows),
456
+ queueStats: this._queueStats(queues, failedJobs),
457
+ workers,
458
+ queues,
459
+ jobTags: this._jobTags(failedJobs),
460
+ failedJobs,
461
+ scheduledJobs,
462
+ deadLetter,
463
+ requests: this._recentRequests(reqs),
464
+ slowRequests: this._slowRequests(reqs),
465
+ topUsers: this._topUsers(reqs),
466
+ topMemory: this._topMemory(reqs),
467
+ exceptions: this._exceptionGroups(excRows),
468
+ dbStats: this._dbStats(queryRows, windowMs),
469
+ slowQueries: this._slowQueries(queryRows),
470
+ transactions: this._transactions(eventRows),
471
+ migrations: this._migrations(eventRows),
472
+ nplusOnes: this._nplusOnes(eventRows),
473
+ security: this._feed(eventRows, "auth", 50),
474
+ alertHistory: this._alertEntries(eventRows),
475
+ logs: this._feed(eventRows, "log", 100),
476
+ realtime: this._realtime(eventRows, windowMs),
477
+ scheduledRuns: this._feed(eventRows, "task", 30),
478
+ slowJobs: this._slowJobs(jobRows),
479
+ cache: this._cache(cacheRows, eventRows),
480
+ mail: this._mailEntries(mailRows),
481
+ notifications: this._notifications(eventRows),
482
+ commands: this._commands(eventRows),
483
+ models: this._models(eventRows),
484
+ recentModels: this._recentModels(eventRows),
485
+ health,
486
+ gauges: systemGauges(),
487
+ uptimeChecks: this._uptimeChecks(health),
488
+ checkins,
489
+ storage: this._db.info(),
490
+ meta: systemMeta({
491
+ ...(this._opts.zerotalVersion ? { zerotal: this._opts.zerotalVersion } : {}),
492
+ ...(this._opts.region ? { region: this._opts.region } : {}),
493
+ ...(this._opts.deploy ? { deploy: this._opts.deploy } : {}),
494
+ }),
495
+ };
496
+ }
497
+
498
+ /**
499
+ * Drill into a single route: latency percentiles, throughput/latency/error
500
+ * series, status-class breakdown, and the recent + slowest requests — all from
501
+ * `mon_requests` filtered to one method+path within the window.
502
+ */
503
+ async routeDetail(
504
+ method: string,
505
+ path: string,
506
+ range: MonitorRange = "live",
507
+ ): Promise<RouteDetail> {
508
+ this._db.flush();
509
+ const windowMs = this._rangeMs(range);
510
+ const cutoff = Date.now() - windowMs;
511
+ const reqs = this._db
512
+ .requestsForRouteWithin(method.toUpperCase(), path, cutoff)
513
+ .map((r) => this._toSample(r));
514
+
515
+ const ms = reqs.map((r) => r.ms);
516
+ const errorCount = reqs.filter((r) => r.status >= 500).length;
517
+ const minutes = Math.max(1, windowMs / 60000);
518
+
519
+ const classes: Record<StatusClassCount["label"], number> = {
520
+ "2xx": 0,
521
+ "3xx": 0,
522
+ "4xx": 0,
523
+ "5xx": 0,
524
+ };
525
+ for (const r of reqs) {
526
+ const cls = `${Math.floor(r.status / 100)}xx` as StatusClassCount["label"];
527
+ if (cls in classes) classes[cls]++;
528
+ }
529
+ const statusDist: StatusClassCount[] = (["2xx", "3xx", "4xx", "5xx"] as const)
530
+ .map((label) => ({ label, count: classes[label] }))
531
+ .filter((c) => c.count > 0);
532
+
533
+ return {
534
+ method: method.toUpperCase(),
535
+ path,
536
+ range,
537
+ total: reqs.length,
538
+ rpm: Math.round((reqs.length / minutes) * 10) / 10,
539
+ errorCount,
540
+ errorRate: reqs.length ? +((errorCount / reqs.length) * 100).toFixed(1) : 0,
541
+ p50: percentile(ms, 50),
542
+ p95: percentile(ms, 95),
543
+ p99: percentile(ms, 99),
544
+ avgMs: ms.length ? Math.round(ms.reduce((a, b) => a + b, 0) / ms.length) : 0,
545
+ maxMs: ms.length ? Math.max(...ms) : 0,
546
+ throughput: this._bucket(reqs, 60, () => 1),
547
+ latency: this._bucket(reqs, 60, (r) => r.ms, true),
548
+ errors: this._bucket(reqs, 60, (r) => (r.status >= 500 ? 1 : 0)),
549
+ statusDist,
550
+ recent: [...reqs]
551
+ .sort((a, b) => b.t - a.t)
552
+ .slice(0, 10)
553
+ .map((r) => this._toEntry(r)),
554
+ slowest: [...reqs]
555
+ .sort((a, b) => b.ms - a.ms)
556
+ .slice(0, 10)
557
+ .map((r) => this._toEntry(r)),
558
+ };
559
+ }
560
+
561
+ private _toSample(r: RequestRow): ReqSample {
562
+ return {
563
+ t: r.t,
564
+ method: r.method,
565
+ path: r.path,
566
+ status: r.status,
567
+ ms: r.ms,
568
+ queries: this._parseQueries(r.queries),
569
+ nplus: r.nplus === 1,
570
+ user: r.user ?? null,
571
+ ip: r.ip ?? null,
572
+ mem: r.mem ?? 0,
573
+ context: this._parseContext(r.context),
574
+ payload: this._parsePayload(r.payload),
575
+ error: r.error ?? null,
576
+ };
577
+ }
578
+
579
+ private _parsePayload(json: string | null | undefined): RequestPayload | null {
580
+ if (!json) return null;
581
+ try {
582
+ const v = JSON.parse(json);
583
+ return v && typeof v === "object" ? (v as RequestPayload) : null;
584
+ } catch {
585
+ return null;
586
+ }
587
+ }
588
+
589
+ private _parseContext(json: string | undefined): Record<string, unknown> {
590
+ if (!json) return {};
591
+ try {
592
+ const v = JSON.parse(json);
593
+ return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
594
+ } catch {
595
+ return {};
596
+ }
597
+ }
598
+
599
+ /** Live "right now" gauges for the Overview pulse row (in-flight, connections, rate). */
600
+ private _pulse(): PulseStats {
601
+ const p = httpPulse();
602
+ return {
603
+ activeRequests: p.activeRequests,
604
+ activeConnections: activeConnections(),
605
+ requestsPerSec: p.requestsPerSec,
606
+ errorRatePct: p.errorRatePct,
607
+ };
608
+ }
609
+
610
+ private _deployMarkers(rows: DeployRow[]): Deploy[] {
611
+ if (rows.length > 0) {
612
+ return rows.map((d, i) => ({ at: 12 + i * 18, sha: d.sha.slice(0, 7), when: ago(d.t) }));
613
+ }
614
+ if (this._opts.deploy) return [{ at: 50, sha: this._opts.deploy.slice(0, 7), when: "current" }];
615
+ return [];
616
+ }
617
+
618
+ private _uptimeChecks(health: HealthEntry[]): UptimeCheck[] {
619
+ return health.map((h) => ({
620
+ name: h.name,
621
+ ok: h.ok,
622
+ ms: 0,
623
+ uptime: h.ok ? "operational" : "down",
624
+ }));
625
+ }
626
+
627
+ // ── Derivations ──────────────────────────────────────────────────────────────
628
+
629
+ private _statCards(
630
+ reqs: ReqSample[],
631
+ priorReqs: ReqSample[],
632
+ windowMs: number,
633
+ cacheRows: CacheRow[],
634
+ ): StatCard[] {
635
+ const minutes = Math.max(1, windowMs / 60000);
636
+ const rpm = reqs.length / minutes;
637
+ const avg = reqs.length ? reqs.reduce((a, r) => a + r.ms, 0) / reqs.length : 0;
638
+ const errs = reqs.filter((r) => r.status >= 500).length;
639
+ const errRate = reqs.length ? (errs / reqs.length) * 100 : 0;
640
+ const cHits = cacheRows.filter((c) => c.hit).length;
641
+ const cacheRate = cacheRows.length ? (cHits / cacheRows.length) * 100 : 0;
642
+
643
+ const priorAvg = priorReqs.length
644
+ ? priorReqs.reduce((a, r) => a + r.ms, 0) / priorReqs.length
645
+ : avg;
646
+ const priorRpm = priorReqs.length / minutes;
647
+
648
+ return [
649
+ {
650
+ label: "Requests / min",
651
+ value: commas(rpm),
652
+ delta: delta(rpm, priorRpm || rpm),
653
+ series: this._bucket(reqs, 20, () => 1),
654
+ },
655
+ {
656
+ label: "Avg response",
657
+ value: `${Math.round(avg)}ms`,
658
+ delta: delta(avg, priorAvg),
659
+ series: this._bucket(reqs, 20, (r) => r.ms, true),
660
+ },
661
+ {
662
+ label: "Error rate",
663
+ value: `${errRate.toFixed(1)}%`,
664
+ delta: 0,
665
+ series: this._bucket(reqs, 20, (r) => (r.status >= 500 ? 1 : 0)),
666
+ },
667
+ {
668
+ label: "Cache hit rate",
669
+ value: `${cacheRate.toFixed(1)}%`,
670
+ delta: 0,
671
+ series: Array.from({ length: 20 }, () => Math.round(cacheRate)),
672
+ },
673
+ ];
674
+ }
675
+
676
+ private _percentiles(reqs: ReqSample[]): Percentile[] {
677
+ const ms = reqs.map((r) => r.ms);
678
+ return [
679
+ { label: "p50", value: percentile(ms, 50) },
680
+ { label: "p95", value: percentile(ms, 95) },
681
+ { label: "p99", value: percentile(ms, 99) },
682
+ ];
683
+ }
684
+
685
+ /** Apdex = (satisfied + tolerating/2) / total, with T = apdexTargetMs. */
686
+ private _apdex(reqs: ReqSample[]): number {
687
+ if (reqs.length === 0) return 1;
688
+ const T = this._opts.apdexTargetMs;
689
+ let satisfied = 0;
690
+ let tolerating = 0;
691
+ for (const r of reqs) {
692
+ if (r.ms <= T) satisfied++;
693
+ else if (r.ms <= 4 * T) tolerating++;
694
+ }
695
+ return +((satisfied + tolerating / 2) / reqs.length).toFixed(2);
696
+ }
697
+
698
+ private _slowRoutes(reqs: ReqSample[]): RouteStat[] {
699
+ const byRoute = new Map<string, { method: string; path: string; total: number; n: number }>();
700
+ for (const r of reqs) {
701
+ const key = `${r.method} ${r.path}`;
702
+ const e = byRoute.get(key) ?? { method: r.method, path: r.path, total: 0, n: 0 };
703
+ e.total += r.ms;
704
+ e.n++;
705
+ byRoute.set(key, e);
706
+ }
707
+ return [...byRoute.values()]
708
+ .map((e) => ({ method: e.method, path: e.path, ms: Math.round(e.total / e.n) }))
709
+ .sort((a, b) => b.ms - a.ms)
710
+ .slice(0, 5);
711
+ }
712
+
713
+ private _outgoing(rows: HttpRow[]): OutgoingHttp[] {
714
+ const byHost = new Map<string, { calls: number; durations: number[]; errors: number }>();
715
+ for (const r of rows) {
716
+ const e = byHost.get(r.host) ?? { calls: 0, durations: [], errors: 0 };
717
+ e.calls++;
718
+ e.durations.push(r.ms);
719
+ if (r.error) e.errors++;
720
+ byHost.set(r.host, e);
721
+ }
722
+ return [...byHost.entries()]
723
+ .map(([host, e]) => ({
724
+ host,
725
+ calls: e.calls,
726
+ p95: percentile(e.durations, 95),
727
+ errRate: +((e.errors / Math.max(1, e.calls)) * 100).toFixed(1),
728
+ }))
729
+ .sort((a, b) => b.p95 - a.p95)
730
+ .slice(0, 8);
731
+ }
732
+
733
+ private _toEntry(r: ReqSample): RequestEntry {
734
+ return {
735
+ id: r.t,
736
+ method: r.method,
737
+ path: r.path,
738
+ status: r.status,
739
+ ms: r.ms,
740
+ when: ago(r.t),
741
+ nplus: r.nplus,
742
+ user: r.user,
743
+ ip: r.ip,
744
+ memKb: r.mem,
745
+ context: r.context,
746
+ payload: r.payload,
747
+ error: r.error,
748
+ spans: this._synthSpans(r.ms, r.queries),
749
+ queries: r.queries,
750
+ logs: [],
751
+ };
752
+ }
753
+
754
+ private _recentRequests(reqs: ReqSample[]): RequestEntry[] {
755
+ return [...reqs]
756
+ .sort((a, b) => b.t - a.t)
757
+ .slice(0, 50) // the table paginates this client-side, 10 per page
758
+ .map((r) => this._toEntry(r));
759
+ }
760
+
761
+ /** Requests at/above the slow threshold, slowest first. */
762
+ private _slowRequests(reqs: ReqSample[]): RequestEntry[] {
763
+ return reqs
764
+ .filter((r) => r.ms >= this._opts.slowRequestMs)
765
+ .sort((a, b) => b.ms - a.ms)
766
+ .slice(0, 10)
767
+ .map((r) => this._toEntry(r));
768
+ }
769
+
770
+ /** Top active users by request volume (Application Usage). */
771
+ private _topUsers(reqs: ReqSample[]): UserUsage[] {
772
+ const counts = new Map<string, number>();
773
+ for (const r of reqs) {
774
+ if (r.user) counts.set(r.user, (counts.get(r.user) ?? 0) + 1);
775
+ }
776
+ return [...counts.entries()]
777
+ .map(([id, requests]) => ({ id, requests }))
778
+ .sort((a, b) => b.requests - a.requests)
779
+ .slice(0, 8);
780
+ }
781
+
782
+ /** Heaviest routes by peak memory (Top Memory Requests). */
783
+ private _topMemory(reqs: ReqSample[]): MemoryRoute[] {
784
+ const byRoute = new Map<string, MemoryRoute>();
785
+ for (const r of reqs) {
786
+ if (r.mem <= 0) continue;
787
+ const key = `${r.method} ${r.path}`;
788
+ const e = byRoute.get(key) ?? { method: r.method, path: r.path, memKb: 0, count: 0 };
789
+ e.memKb = Math.max(e.memKb, r.mem);
790
+ e.count++;
791
+ byRoute.set(key, e);
792
+ }
793
+ return [...byRoute.values()].sort((a, b) => b.memKb - a.memKb).slice(0, 8);
794
+ }
795
+
796
+ private _exceptionGroups(rows: ExceptionRow[]): ExceptionGroup[] {
797
+ const now = Date.now();
798
+ const groups = new Map<string, ExceptionRow[]>();
799
+ for (const r of rows) {
800
+ const key = `${r.type}:${r.location}`;
801
+ let list = groups.get(key);
802
+ if (!list) {
803
+ list = [];
804
+ groups.set(key, list);
805
+ }
806
+ list.push(r);
807
+ }
808
+
809
+ const out: ExceptionGroup[] = [];
810
+ for (const list of groups.values()) {
811
+ list.sort((a, b) => a.t - b.t);
812
+ const last = list[list.length - 1]!;
813
+ const within = (ms: number): number => list.filter((r) => r.t >= now - ms).length;
814
+ // Distinct authenticated users who hit this exception (anonymous hits excluded).
815
+ const affectedUsers = new Set(list.map((r) => r.user).filter((u): u is string => !!u));
816
+
817
+ // 24 hourly buckets over the last 24h (oldest → newest).
818
+ const buckets = Array.from({ length: 24 }, () => 0);
819
+ for (const r of list) {
820
+ const hoursAgo = (now - r.t) / (60 * 60 * 1000);
821
+ if (hoursAgo >= 0 && hoursAgo < 24) {
822
+ const idx = 23 - Math.floor(hoursAgo);
823
+ if (idx >= 0 && idx < 24) buckets[idx] = (buckets[idx] ?? 0) + 1;
824
+ }
825
+ }
826
+
827
+ out.push({
828
+ type: last.type,
829
+ message: last.message,
830
+ location: last.location,
831
+ count: list.length,
832
+ users: affectedUsers.size,
833
+ lastSeen: ago(last.t).replace(" ago", ""),
834
+ d1: within(DAY_MS),
835
+ d7: within(7 * DAY_MS),
836
+ d30: within(30 * DAY_MS),
837
+ series: buckets,
838
+ frames: this._parseFrames(last.frames),
839
+ });
840
+ }
841
+ return out.sort((a, b) => b.count - a.count);
842
+ }
843
+
844
+ private _slowQueries(rows: QueryRow[]): SlowQuery[] {
845
+ const bySql = new Map<string, { sql: string; total: number; n: number; location: string }>();
846
+ for (const q of rows) {
847
+ const e = bySql.get(q.sql) ?? { sql: q.sql, total: 0, n: 0, location: q.location };
848
+ e.total += q.ms;
849
+ e.n++;
850
+ bySql.set(q.sql, e);
851
+ }
852
+ return [...bySql.values()]
853
+ .map((e) => ({
854
+ sql: e.sql,
855
+ ms: Math.round(e.total / e.n),
856
+ callers: e.n,
857
+ location: e.location,
858
+ }))
859
+ .filter((e) => e.ms >= this._opts.slowQueryMs)
860
+ .sort((a, b) => b.ms - a.ms)
861
+ .slice(0, 10);
862
+ }
863
+
864
+ private _cache(rows: CacheRow[], events: EventRow[]): CacheStats {
865
+ let hits = 0;
866
+ let misses = 0;
867
+ const keys = new Map<string, { hits: number; total: number }>();
868
+ for (const r of rows) {
869
+ if (r.hit) hits++;
870
+ else misses++;
871
+ if (r.key) {
872
+ const k = keys.get(r.key) ?? { hits: 0, total: 0 };
873
+ k.total++;
874
+ if (r.hit) k.hits++;
875
+ keys.set(r.key, k);
876
+ }
877
+ }
878
+ const total = hits + misses;
879
+ const keyList = [...keys.entries()]
880
+ .map(([key, v]) => ({
881
+ key,
882
+ hits: v.hits,
883
+ rate: Math.round((v.hits / Math.max(1, v.total)) * 100),
884
+ }))
885
+ .sort((a, b) => b.hits - a.hits)
886
+ .slice(0, 8);
887
+ const evictions = events.filter((e) => e.kind === "cache_evict").length;
888
+ return {
889
+ hitRate: total > 0 ? Math.round((hits / total) * 100) : 0,
890
+ hits,
891
+ misses,
892
+ evictions,
893
+ keys: keyList,
894
+ };
895
+ }
896
+
897
+ // ── Framework-event feeds ──────────────────────────────────────────────────
898
+
899
+ private _eventData(row: EventRow): Record<string, unknown> {
900
+ try {
901
+ const v = JSON.parse(row.data);
902
+ return v && typeof v === "object" ? (v as Record<string, unknown>) : {};
903
+ } catch {
904
+ return {};
905
+ }
906
+ }
907
+
908
+ /** A generic event feed (security, scheduled-task runs, …). */
909
+ private _feed(rows: EventRow[], kind: string, limit: number): FeedEvent[] {
910
+ return rows
911
+ .filter((r) => r.kind === kind)
912
+ .slice(0, limit)
913
+ .map((r) => {
914
+ const d = this._eventData(r);
915
+ const detail =
916
+ typeof d.detail === "string"
917
+ ? d.detail
918
+ : d.ms != null
919
+ ? `${Math.round(Number(d.ms))}ms`
920
+ : d.count != null
921
+ ? `×${d.count}`
922
+ : "";
923
+ return {
924
+ when: ago(r.t).replace(" ago", ""),
925
+ label: r.label,
926
+ detail,
927
+ status: (r.status as FeedEvent["status"]) ?? "info",
928
+ route: r.route,
929
+ };
930
+ });
931
+ }
932
+
933
+ /** Group persisted alert firings by kind, keeping a count, a timeline, and the latest firing's context. */
934
+ private _alertEntries(rows: EventRow[]): AlertEntry[] {
935
+ interface G {
936
+ id: string;
937
+ title: string;
938
+ latest: EventRow;
939
+ firstT: number;
940
+ lastT: number;
941
+ count: number;
942
+ occurrences: { when: string; detail: string; value: number }[];
943
+ }
944
+ const groups = new Map<string, G>();
945
+ // Rows arrive newest-first (eventsWithin orders by t DESC).
946
+ for (const r of rows) {
947
+ if (r.kind !== "alert") continue;
948
+ const d = this._eventData(r);
949
+ const id = String(d.id ?? r.route ?? r.label);
950
+ let g = groups.get(id);
951
+ if (!g) {
952
+ g = { id, title: r.label, latest: r, firstT: r.t, lastT: r.t, count: 0, occurrences: [] };
953
+ groups.set(id, g);
954
+ }
955
+ g.count++;
956
+ if (r.t > g.lastT) {
957
+ g.lastT = r.t;
958
+ g.latest = r;
959
+ }
960
+ if (r.t < g.firstT) g.firstT = r.t;
961
+ if (g.occurrences.length < 20) {
962
+ g.occurrences.push({
963
+ when: ago(r.t).replace(" ago", ""),
964
+ detail: String(d.detail ?? ""),
965
+ value: Number(d.value) || 0,
966
+ });
967
+ }
968
+ }
969
+
970
+ // Critical first, then most-recently fired.
971
+ const list = [...groups.values()].sort((a, b) => {
972
+ const ca = a.latest.status === "bad" ? 0 : 1;
973
+ const cb = b.latest.status === "bad" ? 0 : 1;
974
+ if (ca !== cb) return ca - cb;
975
+ return b.lastT - a.lastT;
976
+ });
977
+
978
+ return list.map((g) => {
979
+ const d = this._eventData(g.latest);
980
+ return {
981
+ id: g.id,
982
+ title: g.title,
983
+ level: d.level === "critical" ? ("critical" as const) : ("warning" as const),
984
+ status: (g.latest.status as AlertEntry["status"]) ?? "warn",
985
+ detail: String(d.detail ?? ""),
986
+ metric: String(d.metric ?? ""),
987
+ value: Number(d.value) || 0,
988
+ threshold: Number(d.threshold) || 0,
989
+ unit: String(d.unit ?? ""),
990
+ count: g.count,
991
+ firstSeen: ago(g.firstT).replace(" ago", ""),
992
+ lastSeen: ago(g.lastT).replace(" ago", ""),
993
+ context: this._alertContext(d.context),
994
+ occurrences: g.occurrences,
995
+ };
996
+ });
997
+ }
998
+
999
+ /** Normalise a stored alert-context blob into a typed AlertContext with safe defaults. */
1000
+ private _alertContext(raw: unknown): AlertContext {
1001
+ const c = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;
1002
+ const num = (v: unknown): number => Number(v) || 0;
1003
+ return {
1004
+ errorRate: num(c.errorRate),
1005
+ rpm: num(c.rpm),
1006
+ p50: num(c.p50),
1007
+ p95: num(c.p95),
1008
+ p99: num(c.p99),
1009
+ apdex: num(c.apdex),
1010
+ pending: num(c.pending),
1011
+ rolledBack: num(c.rolledBack),
1012
+ slowRoutes: Array.isArray(c.slowRoutes) ? (c.slowRoutes as RouteStat[]).slice(0, 3) : [],
1013
+ topException: typeof c.topException === "string" ? c.topException : null,
1014
+ };
1015
+ }
1016
+
1017
+ private _transactions(rows: EventRow[]): TxStats {
1018
+ const tx = rows.filter((r) => r.kind === "tx");
1019
+ const committed = tx.filter((r) => r.label === "committed").length;
1020
+ const rolledBack = tx.filter((r) => r.label === "rolledback").length;
1021
+ const durations = tx.map((r) => Number(this._eventData(r).ms) || 0).filter((m) => m > 0);
1022
+ const avgMs = durations.length
1023
+ ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length)
1024
+ : 0;
1025
+ return { committed, rolledBack, avgMs };
1026
+ }
1027
+
1028
+ private _migrations(rows: EventRow[]): MigrationEntry[] {
1029
+ return rows
1030
+ .filter((r) => r.kind === "migration")
1031
+ .slice(0, 20)
1032
+ .map((r) => {
1033
+ const d = this._eventData(r);
1034
+ return {
1035
+ name: r.label,
1036
+ direction: String(d.direction ?? "up"),
1037
+ ms: Math.round(Number(d.ms) || 0),
1038
+ ok: r.status !== "bad",
1039
+ when: ago(r.t).replace(" ago", ""),
1040
+ };
1041
+ });
1042
+ }
1043
+
1044
+ private _nplusOnes(rows: EventRow[]): NPlusOne[] {
1045
+ const groups = new Map<
1046
+ string,
1047
+ { occurrences: number; worst: number; route: string | null; last: number }
1048
+ >();
1049
+ for (const r of rows.filter((x) => x.kind === "nplus")) {
1050
+ const count = Number(this._eventData(r).count) || 0;
1051
+ const g = groups.get(r.label) ?? { occurrences: 0, worst: 0, route: r.route, last: 0 };
1052
+ g.occurrences++;
1053
+ g.worst = Math.max(g.worst, count);
1054
+ g.last = Math.max(g.last, r.t);
1055
+ if (!g.route) g.route = r.route;
1056
+ groups.set(r.label, g);
1057
+ }
1058
+ return [...groups.entries()]
1059
+ .map(([fingerprint, g]) => ({
1060
+ fingerprint,
1061
+ occurrences: g.occurrences,
1062
+ worstCount: g.worst,
1063
+ route: g.route,
1064
+ lastSeen: ago(g.last).replace(" ago", ""),
1065
+ }))
1066
+ .sort((a, b) => b.occurrences - a.occurrences)
1067
+ .slice(0, 10);
1068
+ }
1069
+
1070
+ private _slowJobs(rows: JobRow[]): JobEntry[] {
1071
+ return rows
1072
+ .filter((r) => r.status === "completed" || r.status === "retried" || r.status === "failed")
1073
+ .sort((a, b) => b.ms - a.ms)
1074
+ .slice(0, 10)
1075
+ .map((r) => ({
1076
+ className: r.className || "(job)",
1077
+ queue: r.queue,
1078
+ status: r.status,
1079
+ ms: r.ms,
1080
+ when: ago(r.t).replace(" ago", ""),
1081
+ error: r.error,
1082
+ }));
1083
+ }
1084
+
1085
+ private _realtime(rows: EventRow[], windowMs: number): RealtimeStats {
1086
+ const ws = rows.filter((r) => r.kind === "ws");
1087
+ const actions = ws.filter((r) => r.label === "action");
1088
+ const opened = ws.filter((r) => r.label === "connect").length;
1089
+ const closed = ws.filter((r) => r.label === "disconnect").length;
1090
+ const minutes = Math.max(1, windowMs / 60000);
1091
+ const durations = actions.map((r) => Number(this._eventData(r).ms) || 0);
1092
+ const avgActionMs = durations.length
1093
+ ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length)
1094
+ : 0;
1095
+
1096
+ const n = 60;
1097
+ const series = Array.from({ length: n }, () => 0);
1098
+ const start = Date.now() - windowMs;
1099
+ for (const r of actions) {
1100
+ const idx = Math.min(
1101
+ n - 1,
1102
+ Math.max(0, Math.floor(((r.t - start) / Math.max(1, windowMs)) * n)),
1103
+ );
1104
+ series[idx] = (series[idx] ?? 0) + 1;
1105
+ }
1106
+
1107
+ const byComp = new Map<string, { actions: number; total: number }>();
1108
+ for (const r of actions) {
1109
+ const d = this._eventData(r);
1110
+ const name = String(d.component ?? "?");
1111
+ const e = byComp.get(name) ?? { actions: 0, total: 0 };
1112
+ e.actions++;
1113
+ e.total += Number(d.ms) || 0;
1114
+ byComp.set(name, e);
1115
+ }
1116
+ const components = [...byComp.entries()]
1117
+ .map(([name, e]) => ({
1118
+ name,
1119
+ actions: e.actions,
1120
+ avgMs: Math.round(e.total / Math.max(1, e.actions)),
1121
+ }))
1122
+ .sort((a, b) => b.actions - a.actions)
1123
+ .slice(0, 8);
1124
+
1125
+ const slowActions = actions
1126
+ .map((r) => {
1127
+ const d = this._eventData(r);
1128
+ return {
1129
+ component: String(d.component ?? "?"),
1130
+ action: String(d.action ?? "?"),
1131
+ ms: Math.round(Number(d.ms) || 0),
1132
+ when: ago(r.t).replace(" ago", ""),
1133
+ };
1134
+ })
1135
+ .sort((a, b) => b.ms - a.ms)
1136
+ .slice(0, 8);
1137
+
1138
+ // Most-recent actions with their full request-scoped context (user/ip/queries/memory).
1139
+ // The table paginates this client-side, like the requests table.
1140
+ const recentActions = actions.slice(0, 50).map((r) => {
1141
+ const d = this._eventData(r);
1142
+ const queries = Array.isArray(d.queries) ? (d.queries as RequestQuery[]) : [];
1143
+ const context =
1144
+ d.context && typeof d.context === "object" && !Array.isArray(d.context)
1145
+ ? (d.context as Record<string, unknown>)
1146
+ : {};
1147
+ return {
1148
+ id: r.t,
1149
+ component: String(d.component ?? "?"),
1150
+ action: String(d.action ?? "?"),
1151
+ ms: Math.round(Number(d.ms) || 0),
1152
+ ok: d.ok !== false,
1153
+ user: typeof d.user === "string" ? d.user : null,
1154
+ ip: typeof d.ip === "string" ? d.ip : null,
1155
+ nplus: d.nplus === true,
1156
+ memKb: Math.max(0, Math.round(Number(d.memKb) || 0)),
1157
+ queries,
1158
+ context,
1159
+ when: ago(r.t).replace(" ago", ""),
1160
+ };
1161
+ });
1162
+
1163
+ const mems = actions.map((r) => Number(this._eventData(r).memKb) || 0).filter((m) => m > 0);
1164
+ const avgMemKb = mems.length ? Math.round(mems.reduce((a, b) => a + b, 0) / mems.length) : 0;
1165
+
1166
+ return {
1167
+ activeConnections: activeConnections(),
1168
+ actionsPerMin: Math.round((actions.length / minutes) * 10) / 10,
1169
+ avgActionMs,
1170
+ avgMemKb,
1171
+ opened,
1172
+ closed,
1173
+ series,
1174
+ components,
1175
+ slowActions,
1176
+ recentActions,
1177
+ clients: this._connectedClients(),
1178
+ };
1179
+ }
1180
+
1181
+ /** Live connected Flow clients (who is online now), newest-active first. */
1182
+ private _connectedClients(): ConnectedClient[] {
1183
+ return connectedClients()
1184
+ .slice(0, 50)
1185
+ .map((c) => ({
1186
+ id: c.id,
1187
+ ip: c.ip,
1188
+ user: c.user,
1189
+ connectedFor: ago(c.connectedAt).replace(" ago", ""),
1190
+ lastActivity: c.actions > 0 ? ago(c.lastActivityAt) : "—",
1191
+ actions: c.actions,
1192
+ }));
1193
+ }
1194
+
1195
+ /** Notification deliveries (one row per channel), newest first. */
1196
+ private _notifications(rows: EventRow[]): NotificationEntry[] {
1197
+ return rows
1198
+ .filter((r) => r.kind === "notification")
1199
+ .slice(0, 50)
1200
+ .map((r) => {
1201
+ const d = this._eventData(r);
1202
+ return {
1203
+ notification: r.label,
1204
+ channel: String(d.channel ?? r.route ?? "?"),
1205
+ recipient: String(d.to ?? ""),
1206
+ status: r.status === "bad" ? ("bad" as const) : ("ok" as const),
1207
+ ms: Math.round(Number(d.ms) || 0),
1208
+ when: ago(r.t).replace(" ago", ""),
1209
+ };
1210
+ });
1211
+ }
1212
+
1213
+ /** Console/Artisan command runs, newest first. */
1214
+ private _commands(rows: EventRow[]): CommandEntry[] {
1215
+ return rows
1216
+ .filter((r) => r.kind === "command")
1217
+ .slice(0, 50)
1218
+ .map((r) => {
1219
+ const d = this._eventData(r);
1220
+ return {
1221
+ name: r.label,
1222
+ status: r.status === "bad" ? ("bad" as const) : ("ok" as const),
1223
+ ms: Math.round(Number(d.ms) || 0),
1224
+ code: Math.round(Number(d.code) || 0),
1225
+ when: ago(r.t).replace(" ago", ""),
1226
+ };
1227
+ });
1228
+ }
1229
+
1230
+ /** Per-model created/updated/deleted counts (the models watcher). */
1231
+ private _models(rows: EventRow[]): ModelStat[] {
1232
+ const groups = new Map<string, ModelStat>();
1233
+ for (const r of rows.filter((x) => x.kind === "model")) {
1234
+ const op = String(this._eventData(r).op ?? r.route ?? "");
1235
+ const table = String(this._eventData(r).table ?? "");
1236
+ const g = groups.get(r.label) ?? {
1237
+ model: r.label,
1238
+ table,
1239
+ created: 0,
1240
+ updated: 0,
1241
+ deleted: 0,
1242
+ total: 0,
1243
+ };
1244
+ if (op === "created") g.created++;
1245
+ else if (op === "updated") g.updated++;
1246
+ else if (op === "deleted") g.deleted++;
1247
+ g.total++;
1248
+ if (!g.table && table) g.table = table;
1249
+ groups.set(r.label, g);
1250
+ }
1251
+ return [...groups.values()].sort((a, b) => b.total - a.total).slice(0, 20);
1252
+ }
1253
+
1254
+ /** Most-recent model changes, newest first (the models-watcher timeline). */
1255
+ private _recentModels(rows: EventRow[]): ModelEvent[] {
1256
+ return rows
1257
+ .filter((r) => r.kind === "model")
1258
+ .slice(0, 30)
1259
+ .map((r) => {
1260
+ const d = this._eventData(r);
1261
+ return {
1262
+ model: r.label,
1263
+ table: String(d.table ?? ""),
1264
+ operation: String(d.op ?? r.route ?? "created") as ModelEvent["operation"],
1265
+ when: ago(r.t).replace(" ago", ""),
1266
+ };
1267
+ });
1268
+ }
1269
+
1270
+ private _dbStats(rows: QueryRow[], windowMs: number): DbStat[] {
1271
+ const ms = rows.map((x) => x.ms);
1272
+ const avg = ms.length ? ms.reduce((a, b) => a + b, 0) / ms.length : 0;
1273
+ const slow = rows.filter((x) => x.ms >= this._opts.slowQueryMs).length;
1274
+ const perMin = rows.length / Math.max(1, windowMs / 60000);
1275
+ return [
1276
+ { label: "Queries / min", value: commas(perMin), sub: "reads + writes" },
1277
+ {
1278
+ label: "Avg query",
1279
+ value: `${avg.toFixed(1)}ms`,
1280
+ sub: ms.length ? `p95 ${percentile(ms, 95)}ms` : "no traffic",
1281
+ },
1282
+ {
1283
+ label: `Slow (>${this._opts.slowQueryMs}ms)`,
1284
+ value: String(slow),
1285
+ sub: "in range",
1286
+ tone: slow > 0 ? ("warn" as const) : ("ok" as const),
1287
+ },
1288
+ { label: "Connections", value: "—", sub: "pool usage" },
1289
+ ];
1290
+ }
1291
+
1292
+ private _mailEntries(rows: MailRow[]): MailEntry[] {
1293
+ return rows.slice(0, 50).map((m) => ({
1294
+ id: m.t,
1295
+ subject: m.subject,
1296
+ to: m.recipient,
1297
+ mailer: m.mailer,
1298
+ status: m.status as MailEntry["status"],
1299
+ when: ago(m.t),
1300
+ ms: m.ms,
1301
+ body: m.body,
1302
+ }));
1303
+ }
1304
+
1305
+ /** Fill each queue's throughput (runs/min) and trend series from recorded job runs. */
1306
+ private _enrichQueues(queues: QueueRow[], jobRows: JobRow[], windowMs: number): void {
1307
+ if (queues.length === 0) return;
1308
+ const minutes = Math.max(1, windowMs / 60000);
1309
+ const n = 24;
1310
+ const start = Date.now() - windowMs;
1311
+ const span = Math.max(1, windowMs);
1312
+ for (const q of queues) {
1313
+ const runs = jobRows.filter(
1314
+ (j) => j.queue === q.name && (j.status === "completed" || j.status === "retried"),
1315
+ );
1316
+ q.throughput = Math.round(runs.length / minutes);
1317
+ const slots = Array.from({ length: n }, () => 0);
1318
+ for (const r of runs) {
1319
+ const idx = Math.min(n - 1, Math.max(0, Math.floor(((r.t - start) / span) * n)));
1320
+ slots[idx] = (slots[idx] ?? 0) + 1;
1321
+ }
1322
+ q.series = slots;
1323
+ }
1324
+ }
1325
+
1326
+ private _queueStats(queues: QueueRow[], failed: { length: number }): QueueMetric[] {
1327
+ const pending = queues.reduce((a, q) => a + q.pending, 0);
1328
+ const stats = liveQueueStats(this._app);
1329
+ const perMin = stats ? stats.processedLast5m / 5 : queues.reduce((a, q) => a + q.throughput, 0);
1330
+ return [
1331
+ { label: "Jobs / min", value: commas(perMin), sub: `across ${queues.length} queues` },
1332
+ {
1333
+ label: "Pending",
1334
+ value: commas(pending),
1335
+ sub: "waiting to run",
1336
+ tone: pending > 200 ? "warn" : "neutral",
1337
+ },
1338
+ {
1339
+ label: "Failed (24h)",
1340
+ value: String(failed.length),
1341
+ sub: "needs attention",
1342
+ tone: failed.length > 0 ? "bad" : "ok",
1343
+ },
1344
+ {
1345
+ label: "Avg wait",
1346
+ value: `${(queues.reduce((a, q) => a + q.wait, 0) / Math.max(1, queues.length)).toFixed(1)}s`,
1347
+ sub: "p95 —",
1348
+ },
1349
+ ];
1350
+ }
1351
+
1352
+ private _jobTags(failed: { tags: string[] }[]): string[] {
1353
+ const tags = new Set<string>(["all"]);
1354
+ for (const j of failed) for (const t of j.tags) tags.add(t);
1355
+ return [...tags];
1356
+ }
1357
+
1358
+ private _jobSeries(rows: JobRow[], windowMs: number): number[] {
1359
+ const done = rows.filter((r) => r.status === "completed" || r.status === "retried");
1360
+ const n = 60;
1361
+ const slots = Array.from({ length: n }, () => 0);
1362
+ if (done.length === 0) return slots;
1363
+ const start = Date.now() - windowMs;
1364
+ const span = Math.max(1, windowMs);
1365
+ for (const r of done) {
1366
+ const idx = Math.min(n - 1, Math.max(0, Math.floor(((r.t - start) / span) * n)));
1367
+ slots[idx] = (slots[idx] ?? 0) + 1;
1368
+ }
1369
+ return slots;
1370
+ }
1371
+
1372
+ private _alerts(reqs: ReqSample[], queues: QueueRow[]): Alert[] {
1373
+ const out: Alert[] = [];
1374
+ if (reqs.length > 20) {
1375
+ const errRate = (reqs.filter((r) => r.status >= 500).length / reqs.length) * 100;
1376
+ if (errRate > 2)
1377
+ out.push({
1378
+ tone: "red",
1379
+ text: `Elevated error rate: ${errRate.toFixed(1)}% of requests are 5xx.`,
1380
+ });
1381
+ }
1382
+ const backed = queues.find((q) => q.pending > 200);
1383
+ if (backed)
1384
+ out.push({
1385
+ tone: "amber",
1386
+ text: `Queue "${backed.name}" backed up: ${backed.pending} pending, p95 wait ${backed.wait}s.`,
1387
+ });
1388
+ return out;
1389
+ }
1390
+
1391
+ // ── helpers ──────────────────────────────────────────────────────────────────
1392
+
1393
+ /** Bucket samples into `n` time slots; `pick` extracts the value, summed (or averaged). */
1394
+ private _bucket(
1395
+ reqs: ReqSample[],
1396
+ n: number,
1397
+ pick: (r: ReqSample) => number,
1398
+ average = false,
1399
+ ): number[] {
1400
+ if (reqs.length === 0) return Array.from({ length: n }, () => 0);
1401
+ const first = reqs.reduce((m, r) => Math.min(m, r.t), Infinity);
1402
+ const last = reqs.reduce((m, r) => Math.max(m, r.t), -Infinity);
1403
+ const span = Math.max(1, last - first);
1404
+ const sums = Array.from({ length: n }, () => 0);
1405
+ const counts = Array.from({ length: n }, () => 0);
1406
+ for (const r of reqs) {
1407
+ const idx = Math.min(n - 1, Math.floor(((r.t - first) / span) * n));
1408
+ sums[idx] = (sums[idx] ?? 0) + pick(r);
1409
+ counts[idx] = (counts[idx] ?? 0) + 1;
1410
+ }
1411
+ return sums.map((s, i) =>
1412
+ average ? (counts[i] ? Math.round(s / (counts[i] as number)) : 0) : s,
1413
+ );
1414
+ }
1415
+
1416
+ private _synthSpans(ms: number, queries: RequestQuery[]): RequestSpan[] {
1417
+ const boot = Math.min(8, Math.round(ms * 0.05));
1418
+ const queryMs = queries.reduce((a, q) => a + q.ms, 0);
1419
+ const handler = Math.max(0, ms - boot - queryMs);
1420
+ const spans: RequestSpan[] = [{ label: "boot", kind: "boot", start: 0, dur: boot }];
1421
+ let cursor = boot;
1422
+ if (queryMs > 0) {
1423
+ spans.push({ label: "queries", kind: "query", start: cursor, dur: queryMs });
1424
+ cursor += queryMs;
1425
+ }
1426
+ spans.push({ label: "handler", kind: "controller", start: cursor, dur: handler });
1427
+ return spans;
1428
+ }
1429
+
1430
+ private _rangeMs(range: MonitorRange): number {
1431
+ switch (range) {
1432
+ case "1h":
1433
+ return 60 * 60 * 1000;
1434
+ case "24h":
1435
+ return 24 * 60 * 60 * 1000;
1436
+ case "7d":
1437
+ return 7 * 24 * 60 * 60 * 1000;
1438
+ case "live":
1439
+ default:
1440
+ return 60 * 1000;
1441
+ }
1442
+ }
1443
+
1444
+ private _parseQueries(json: string): RequestQuery[] {
1445
+ try {
1446
+ const v = JSON.parse(json);
1447
+ return Array.isArray(v) ? (v as RequestQuery[]) : [];
1448
+ } catch {
1449
+ return [];
1450
+ }
1451
+ }
1452
+
1453
+ private _parseFrames(json: string): string[] {
1454
+ try {
1455
+ const v = JSON.parse(json);
1456
+ return Array.isArray(v) ? (v as string[]) : [];
1457
+ } catch {
1458
+ return [];
1459
+ }
1460
+ }
1461
+
1462
+ private _stackFrames(stack?: string): string[] {
1463
+ if (!stack) return [];
1464
+ return stack
1465
+ .split("\n")
1466
+ .slice(1)
1467
+ .map((l) => l.trim().replace(/^at\s+/, ""))
1468
+ .filter(Boolean)
1469
+ .slice(0, 6);
1470
+ }
1471
+ }